zstd-ruby 1.4.0.0 → 1.4.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +35 -0
- data/README.md +2 -2
- data/ext/zstdruby/libzstd/Makefile +274 -107
- data/ext/zstdruby/libzstd/README.md +75 -16
- data/ext/zstdruby/libzstd/common/bitstream.h +59 -51
- data/ext/zstdruby/libzstd/common/compiler.h +154 -5
- data/ext/zstdruby/libzstd/common/cpu.h +1 -3
- data/ext/zstdruby/libzstd/common/debug.c +11 -31
- data/ext/zstdruby/libzstd/common/debug.h +22 -49
- data/ext/zstdruby/libzstd/common/entropy_common.c +201 -75
- data/ext/zstdruby/libzstd/common/error_private.c +3 -1
- data/ext/zstdruby/libzstd/common/error_private.h +7 -3
- data/ext/zstdruby/libzstd/common/fse.h +50 -42
- data/ext/zstdruby/libzstd/common/fse_decompress.c +134 -50
- data/ext/zstdruby/libzstd/common/huf.h +41 -38
- data/ext/zstdruby/libzstd/common/mem.h +68 -22
- data/ext/zstdruby/libzstd/common/pool.c +30 -20
- data/ext/zstdruby/libzstd/common/pool.h +3 -3
- data/ext/zstdruby/libzstd/common/threading.c +51 -4
- data/ext/zstdruby/libzstd/common/threading.h +36 -4
- data/ext/zstdruby/libzstd/common/xxhash.c +39 -89
- data/ext/zstdruby/libzstd/common/xxhash.h +12 -32
- data/ext/zstdruby/libzstd/common/zstd_common.c +10 -10
- data/ext/zstdruby/libzstd/common/zstd_deps.h +111 -0
- data/ext/zstdruby/libzstd/common/zstd_errors.h +3 -1
- data/ext/zstdruby/libzstd/common/zstd_internal.h +231 -72
- data/ext/zstdruby/libzstd/common/zstd_trace.c +42 -0
- data/ext/zstdruby/libzstd/common/zstd_trace.h +152 -0
- data/ext/zstdruby/libzstd/compress/fse_compress.c +47 -63
- data/ext/zstdruby/libzstd/compress/hist.c +41 -63
- data/ext/zstdruby/libzstd/compress/hist.h +13 -33
- data/ext/zstdruby/libzstd/compress/huf_compress.c +288 -172
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +2504 -1626
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +446 -85
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +158 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +29 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +433 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +54 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +849 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +561 -0
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +82 -60
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +106 -80
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +411 -105
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +21 -1
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +296 -207
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +14 -3
- data/ext/zstdruby/libzstd/compress/zstd_ldm_geartab.h +103 -0
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +260 -148
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +153 -440
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +29 -110
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +356 -238
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +20 -16
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +3 -3
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +641 -238
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +600 -371
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +8 -5
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +40 -9
- data/ext/zstdruby/libzstd/deprecated/zbuff.h +9 -8
- data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
- data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
- data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +197 -78
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +52 -7
- data/ext/zstdruby/libzstd/dictBuilder/divsufsort.c +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +84 -66
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +58 -36
- data/ext/zstdruby/libzstd/dictBuilder/zdict.h +60 -31
- data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
- data/ext/zstdruby/libzstd/dll/example/README.md +16 -22
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +8 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +115 -111
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +28 -14
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +28 -14
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +36 -19
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +122 -107
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +2 -2
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +29 -23
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +34 -24
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +1 -1
- data/ext/zstdruby/libzstd/libzstd.pc.in +2 -1
- data/ext/zstdruby/libzstd/zstd.h +655 -118
- data/lib/zstd-ruby/version.rb +1 -1
- data/zstd-ruby.gemspec +1 -1
- metadata +20 -10
- data/.travis.yml +0 -14
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -19,7 +19,7 @@ extern "C" {
|
|
19
19
|
* Dependencies
|
20
20
|
***************************************/
|
21
21
|
#include <stddef.h> /* size_t */
|
22
|
-
#include "mem.h" /* U64, U32 */
|
22
|
+
#include "../common/mem.h" /* U64, U32 */
|
23
23
|
|
24
24
|
|
25
25
|
/* *************************************
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -14,7 +14,7 @@
|
|
14
14
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
15
15
|
#include <string.h> /* memcpy */
|
16
16
|
#include <stdlib.h> /* malloc, free, qsort */
|
17
|
-
#include "error_private.h"
|
17
|
+
#include "../common/error_private.h"
|
18
18
|
|
19
19
|
|
20
20
|
|
@@ -82,7 +82,11 @@ extern "C" {
|
|
82
82
|
* Basic Types
|
83
83
|
*****************************************************************/
|
84
84
|
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
85
|
-
#
|
85
|
+
# if defined(_AIX)
|
86
|
+
# include <inttypes.h>
|
87
|
+
# else
|
88
|
+
# include <stdint.h> /* intptr_t */
|
89
|
+
# endif
|
86
90
|
typedef uint8_t BYTE;
|
87
91
|
typedef uint16_t U16;
|
88
92
|
typedef int16_t S16;
|
@@ -860,7 +864,7 @@ MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
|
|
860
864
|
_BitScanReverse ( &r, val );
|
861
865
|
return (unsigned) r;
|
862
866
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
863
|
-
return
|
867
|
+
return __builtin_clz (val) ^ 31;
|
864
868
|
# else /* Software version */
|
865
869
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
866
870
|
U32 v = val;
|
@@ -1862,7 +1866,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
|
|
1862
1866
|
|
1863
1867
|
if (!srcSize) return ERROR(srcSize_wrong);
|
1864
1868
|
iSize = ip[0];
|
1865
|
-
|
1869
|
+
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
|
1866
1870
|
|
1867
1871
|
if (iSize >= 128) { /* special header */
|
1868
1872
|
if (iSize >= (242)) { /* RLE */
|
@@ -2014,7 +2018,7 @@ size_t HUFv06_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
|
|
2014
2018
|
HUFv06_DEltX2* const dt = (HUFv06_DEltX2*)dtPtr;
|
2015
2019
|
|
2016
2020
|
HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
|
2017
|
-
|
2021
|
+
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
|
2018
2022
|
|
2019
2023
|
iSize = HUFv06_readStats(huffWeight, HUFv06_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
2020
2024
|
if (HUFv06_isError(iSize)) return iSize;
|
@@ -2340,7 +2344,7 @@ size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
2340
2344
|
|
2341
2345
|
HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
|
2342
2346
|
if (memLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
2343
|
-
|
2347
|
+
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
|
2344
2348
|
|
2345
2349
|
iSize = HUFv06_readStats(weightList, HUFv06_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
2346
2350
|
if (HUFv06_isError(iSize)) return iSize;
|
@@ -2664,13 +2668,13 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
|
2664
2668
|
|
2665
2669
|
{ U32 algoNb = 0;
|
2666
2670
|
if (Dtime[1] < Dtime[0]) algoNb = 1;
|
2667
|
-
|
2671
|
+
/* if (Dtime[2] < Dtime[algoNb]) algoNb = 2; */ /* current speed of HUFv06_decompress4X6 is not good */
|
2668
2672
|
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
2669
2673
|
}
|
2670
2674
|
|
2671
|
-
|
2672
|
-
|
2673
|
-
|
2675
|
+
/* return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
|
2676
|
+
/* return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
|
2677
|
+
/* return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams quad-symbols decoding */
|
2674
2678
|
}
|
2675
2679
|
/*
|
2676
2680
|
Common functions of Zstd compression library
|
@@ -3025,7 +3029,7 @@ typedef struct
|
|
3025
3029
|
* Provides the size of compressed block from block header `src` */
|
3026
3030
|
static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
3027
3031
|
{
|
3028
|
-
const BYTE* const in = (const BYTE*
|
3032
|
+
const BYTE* const in = (const BYTE*)src;
|
3029
3033
|
U32 cSize;
|
3030
3034
|
|
3031
3035
|
if (srcSize < ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
|
@@ -3219,7 +3223,7 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
|
3219
3223
|
FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
|
3220
3224
|
const void* src, size_t srcSize)
|
3221
3225
|
{
|
3222
|
-
const BYTE* const istart = (const BYTE*
|
3226
|
+
const BYTE* const istart = (const BYTE*)src;
|
3223
3227
|
const BYTE* const iend = istart + srcSize;
|
3224
3228
|
const BYTE* ip = istart;
|
3225
3229
|
|
@@ -3242,14 +3246,12 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
|
3242
3246
|
}
|
3243
3247
|
|
3244
3248
|
/* FSE table descriptors */
|
3249
|
+
if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
|
3245
3250
|
{ U32 const LLtype = *ip >> 6;
|
3246
3251
|
U32 const Offtype = (*ip >> 4) & 3;
|
3247
3252
|
U32 const MLtype = (*ip >> 2) & 3;
|
3248
3253
|
ip++;
|
3249
3254
|
|
3250
|
-
/* check */
|
3251
|
-
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
3252
|
-
|
3253
3255
|
/* Build DTables */
|
3254
3256
|
{ size_t const bhSize = ZSTDv06_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
3255
3257
|
if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
|
@@ -3443,7 +3445,7 @@ static size_t ZSTDv06_decompressSequences(
|
|
3443
3445
|
{
|
3444
3446
|
const BYTE* ip = (const BYTE*)seqStart;
|
3445
3447
|
const BYTE* const iend = ip + seqSize;
|
3446
|
-
BYTE* const ostart = (BYTE*
|
3448
|
+
BYTE* const ostart = (BYTE*)dst;
|
3447
3449
|
BYTE* const oend = ostart + maxDstSize;
|
3448
3450
|
BYTE* op = ostart;
|
3449
3451
|
const BYTE* litPtr = dctx->litPtr;
|
@@ -3503,8 +3505,10 @@ static size_t ZSTDv06_decompressSequences(
|
|
3503
3505
|
{ size_t const lastLLSize = litEnd - litPtr;
|
3504
3506
|
if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
|
3505
3507
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
3506
|
-
|
3507
|
-
|
3508
|
+
if (lastLLSize > 0) {
|
3509
|
+
memcpy(op, litPtr, lastLLSize);
|
3510
|
+
op += lastLLSize;
|
3511
|
+
}
|
3508
3512
|
}
|
3509
3513
|
|
3510
3514
|
return op-ostart;
|
@@ -3557,7 +3561,7 @@ static size_t ZSTDv06_decompressFrame(ZSTDv06_DCtx* dctx,
|
|
3557
3561
|
{
|
3558
3562
|
const BYTE* ip = (const BYTE*)src;
|
3559
3563
|
const BYTE* const iend = ip + srcSize;
|
3560
|
-
BYTE* const ostart = (BYTE*
|
3564
|
+
BYTE* const ostart = (BYTE*)dst;
|
3561
3565
|
BYTE* op = ostart;
|
3562
3566
|
BYTE* const oend = ostart + dstCapacity;
|
3563
3567
|
size_t remainingSize = srcSize;
|
@@ -3672,7 +3676,7 @@ void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cS
|
|
3672
3676
|
blockProperties_t blockProperties = { bt_compressed, 0 };
|
3673
3677
|
|
3674
3678
|
/* Frame Header */
|
3675
|
-
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src,
|
3679
|
+
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, srcSize);
|
3676
3680
|
if (ZSTDv06_isError(frameHeaderSize)) {
|
3677
3681
|
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
3678
3682
|
return;
|
@@ -4002,7 +4006,9 @@ size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* zbd)
|
|
4002
4006
|
MEM_STATIC size_t ZBUFFv06_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
4003
4007
|
{
|
4004
4008
|
size_t length = MIN(dstCapacity, srcSize);
|
4005
|
-
|
4009
|
+
if (length > 0) {
|
4010
|
+
memcpy(dst, src, length);
|
4011
|
+
}
|
4006
4012
|
return length;
|
4007
4013
|
}
|
4008
4014
|
|
@@ -4111,7 +4117,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|
4111
4117
|
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
4112
4118
|
zbd->outEnd = zbd->outStart + decodedSize;
|
4113
4119
|
zbd->stage = ZBUFFds_flush;
|
4114
|
-
|
4120
|
+
/* break; */ /* ZBUFFds_flush follows */
|
4115
4121
|
}
|
4116
4122
|
}
|
4117
4123
|
/* fall-through */
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -17,14 +17,14 @@
|
|
17
17
|
#ifndef XXH_STATIC_LINKING_ONLY
|
18
18
|
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
19
19
|
#endif
|
20
|
-
#include "xxhash.h" /* XXH64_* */
|
20
|
+
#include "../common/xxhash.h" /* XXH64_* */
|
21
21
|
#include "zstd_v07.h"
|
22
22
|
|
23
23
|
#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
|
24
24
|
#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
|
25
25
|
#define ZSTDv07_STATIC_LINKING_ONLY
|
26
26
|
|
27
|
-
#include "error_private.h"
|
27
|
+
#include "../common/error_private.h"
|
28
28
|
|
29
29
|
|
30
30
|
#ifdef ZSTDv07_STATIC_LINKING_ONLY
|
@@ -242,7 +242,11 @@ extern "C" {
|
|
242
242
|
* Basic Types
|
243
243
|
*****************************************************************/
|
244
244
|
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
245
|
-
#
|
245
|
+
# if defined(_AIX)
|
246
|
+
# include <inttypes.h>
|
247
|
+
# else
|
248
|
+
# include <stdint.h> /* intptr_t */
|
249
|
+
# endif
|
246
250
|
typedef uint8_t BYTE;
|
247
251
|
typedef uint16_t U16;
|
248
252
|
typedef int16_t S16;
|
@@ -530,7 +534,7 @@ MEM_STATIC unsigned BITv07_highbit32 (U32 val)
|
|
530
534
|
_BitScanReverse ( &r, val );
|
531
535
|
return (unsigned) r;
|
532
536
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
533
|
-
return
|
537
|
+
return __builtin_clz (val) ^ 31;
|
534
538
|
# else /* Software version */
|
535
539
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
536
540
|
U32 v = val;
|
@@ -1314,7 +1318,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
|
1314
1318
|
|
1315
1319
|
if (!srcSize) return ERROR(srcSize_wrong);
|
1316
1320
|
iSize = ip[0];
|
1317
|
-
|
1321
|
+
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
|
1318
1322
|
|
1319
1323
|
if (iSize >= 128) { /* special header */
|
1320
1324
|
if (iSize >= (242)) { /* RLE */
|
@@ -1784,7 +1788,7 @@ size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSi
|
|
1784
1788
|
HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr;
|
1785
1789
|
|
1786
1790
|
HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable));
|
1787
|
-
|
1791
|
+
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
|
1788
1792
|
|
1789
1793
|
iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
1790
1794
|
if (HUFv07_isError(iSize)) return iSize;
|
@@ -2148,7 +2152,7 @@ size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSi
|
|
2148
2152
|
|
2149
2153
|
HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */
|
2150
2154
|
if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
|
2151
|
-
|
2155
|
+
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
|
2152
2156
|
|
2153
2157
|
iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
2154
2158
|
if (HUFv07_isError(iSize)) return iSize;
|
@@ -2530,8 +2534,8 @@ size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
|
2530
2534
|
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
2531
2535
|
}
|
2532
2536
|
|
2533
|
-
|
2534
|
-
|
2537
|
+
/* return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
|
2538
|
+
/* return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
|
2535
2539
|
}
|
2536
2540
|
|
2537
2541
|
size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
@@ -3254,7 +3258,7 @@ typedef struct
|
|
3254
3258
|
* Provides the size of compressed block from block header `src` */
|
3255
3259
|
static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
3256
3260
|
{
|
3257
|
-
const BYTE* const in = (const BYTE*
|
3261
|
+
const BYTE* const in = (const BYTE*)src;
|
3258
3262
|
U32 cSize;
|
3259
3263
|
|
3260
3264
|
if (srcSize < ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
|
@@ -3272,7 +3276,9 @@ static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProper
|
|
3272
3276
|
static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
3273
3277
|
{
|
3274
3278
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
3275
|
-
|
3279
|
+
if (srcSize > 0) {
|
3280
|
+
memcpy(dst, src, srcSize);
|
3281
|
+
}
|
3276
3282
|
return srcSize;
|
3277
3283
|
}
|
3278
3284
|
|
@@ -3447,7 +3453,7 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
|
|
3447
3453
|
FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable,
|
3448
3454
|
const void* src, size_t srcSize)
|
3449
3455
|
{
|
3450
|
-
const BYTE* const istart = (const BYTE*
|
3456
|
+
const BYTE* const istart = (const BYTE*)src;
|
3451
3457
|
const BYTE* const iend = istart + srcSize;
|
3452
3458
|
const BYTE* ip = istart;
|
3453
3459
|
|
@@ -3470,14 +3476,12 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
|
|
3470
3476
|
}
|
3471
3477
|
|
3472
3478
|
/* FSE table descriptors */
|
3479
|
+
if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
|
3473
3480
|
{ U32 const LLtype = *ip >> 6;
|
3474
3481
|
U32 const OFtype = (*ip >> 4) & 3;
|
3475
3482
|
U32 const MLtype = (*ip >> 2) & 3;
|
3476
3483
|
ip++;
|
3477
3484
|
|
3478
|
-
/* check */
|
3479
|
-
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
3480
|
-
|
3481
3485
|
/* Build DTables */
|
3482
3486
|
{ size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
3483
3487
|
if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected);
|
@@ -3668,7 +3672,7 @@ static size_t ZSTDv07_decompressSequences(
|
|
3668
3672
|
{
|
3669
3673
|
const BYTE* ip = (const BYTE*)seqStart;
|
3670
3674
|
const BYTE* const iend = ip + seqSize;
|
3671
|
-
BYTE* const ostart = (BYTE*
|
3675
|
+
BYTE* const ostart = (BYTE*)dst;
|
3672
3676
|
BYTE* const oend = ostart + maxDstSize;
|
3673
3677
|
BYTE* op = ostart;
|
3674
3678
|
const BYTE* litPtr = dctx->litPtr;
|
@@ -3714,10 +3718,12 @@ static size_t ZSTDv07_decompressSequences(
|
|
3714
3718
|
|
3715
3719
|
/* last literal segment */
|
3716
3720
|
{ size_t const lastLLSize = litEnd - litPtr;
|
3717
|
-
|
3721
|
+
/* if (litPtr > litEnd) return ERROR(corruption_detected); */ /* too many literals already used */
|
3718
3722
|
if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
|
3719
|
-
|
3720
|
-
|
3723
|
+
if (lastLLSize > 0) {
|
3724
|
+
memcpy(op, litPtr, lastLLSize);
|
3725
|
+
op += lastLLSize;
|
3726
|
+
}
|
3721
3727
|
}
|
3722
3728
|
|
3723
3729
|
return op-ostart;
|
@@ -3778,7 +3784,9 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
|
|
3778
3784
|
static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
3779
3785
|
{
|
3780
3786
|
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
|
3781
|
-
|
3787
|
+
if (length > 0) {
|
3788
|
+
memset(dst, byte, length);
|
3789
|
+
}
|
3782
3790
|
return length;
|
3783
3791
|
}
|
3784
3792
|
|
@@ -3791,7 +3799,7 @@ static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx,
|
|
3791
3799
|
{
|
3792
3800
|
const BYTE* ip = (const BYTE*)src;
|
3793
3801
|
const BYTE* const iend = ip + srcSize;
|
3794
|
-
BYTE* const ostart = (BYTE*
|
3802
|
+
BYTE* const ostart = (BYTE*)dst;
|
3795
3803
|
BYTE* const oend = ostart + dstCapacity;
|
3796
3804
|
BYTE* op = ostart;
|
3797
3805
|
size_t remainingSize = srcSize;
|
@@ -3918,7 +3926,7 @@ void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cS
|
|
3918
3926
|
}
|
3919
3927
|
|
3920
3928
|
/* Frame Header */
|
3921
|
-
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src,
|
3929
|
+
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, srcSize);
|
3922
3930
|
if (ZSTDv07_isError(frameHeaderSize)) {
|
3923
3931
|
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
|
3924
3932
|
return;
|
@@ -4380,7 +4388,9 @@ size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd)
|
|
4380
4388
|
MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
4381
4389
|
{
|
4382
4390
|
size_t const length = MIN(dstCapacity, srcSize);
|
4383
|
-
|
4391
|
+
if (length > 0) {
|
4392
|
+
memcpy(dst, src, length);
|
4393
|
+
}
|
4384
4394
|
return length;
|
4385
4395
|
}
|
4386
4396
|
|
@@ -3,8 +3,9 @@
|
|
3
3
|
# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
4
4
|
|
5
5
|
prefix=@PREFIX@
|
6
|
-
|
6
|
+
exec_prefix=@EXEC_PREFIX@
|
7
7
|
includedir=@INCLUDEDIR@
|
8
|
+
libdir=@LIBDIR@
|
8
9
|
|
9
10
|
Name: zstd
|
10
11
|
Description: fast lossless compression algorithm library
|
data/ext/zstdruby/libzstd/zstd.h
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -15,6 +15,7 @@ extern "C" {
|
|
15
15
|
#define ZSTD_H_235446
|
16
16
|
|
17
17
|
/* ====== Dependency ======*/
|
18
|
+
#include <limits.h> /* INT_MAX */
|
18
19
|
#include <stddef.h> /* size_t */
|
19
20
|
|
20
21
|
|
@@ -71,27 +72,32 @@ extern "C" {
|
|
71
72
|
/*------ Version ------*/
|
72
73
|
#define ZSTD_VERSION_MAJOR 1
|
73
74
|
#define ZSTD_VERSION_MINOR 4
|
74
|
-
#define ZSTD_VERSION_RELEASE
|
75
|
-
|
75
|
+
#define ZSTD_VERSION_RELEASE 9
|
76
76
|
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
77
|
-
|
77
|
+
|
78
|
+
/*! ZSTD_versionNumber() :
|
79
|
+
* Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */
|
80
|
+
ZSTDLIB_API unsigned ZSTD_versionNumber(void);
|
78
81
|
|
79
82
|
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
|
80
83
|
#define ZSTD_QUOTE(str) #str
|
81
84
|
#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
|
82
85
|
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
|
83
|
-
ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */
|
84
86
|
|
85
|
-
|
86
|
-
*
|
87
|
-
|
87
|
+
/*! ZSTD_versionString() :
|
88
|
+
* Return runtime library version, like "1.4.5". Requires v1.3.0+. */
|
89
|
+
ZSTDLIB_API const char* ZSTD_versionString(void);
|
90
|
+
|
91
|
+
/* *************************************
|
92
|
+
* Default constant
|
93
|
+
***************************************/
|
88
94
|
#ifndef ZSTD_CLEVEL_DEFAULT
|
89
95
|
# define ZSTD_CLEVEL_DEFAULT 3
|
90
96
|
#endif
|
91
97
|
|
92
|
-
|
93
|
-
* Constants
|
94
|
-
***************************************/
|
98
|
+
/* *************************************
|
99
|
+
* Constants
|
100
|
+
***************************************/
|
95
101
|
|
96
102
|
/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
|
97
103
|
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
|
@@ -183,17 +189,26 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres
|
|
183
189
|
***************************************/
|
184
190
|
/*= Compression context
|
185
191
|
* When compressing many times,
|
186
|
-
* it is recommended to allocate a context just once,
|
192
|
+
* it is recommended to allocate a context just once,
|
193
|
+
* and re-use it for each successive compression operation.
|
187
194
|
* This will make workload friendlier for system's memory.
|
188
|
-
*
|
195
|
+
* Note : re-using context is just a speed / resource optimization.
|
196
|
+
* It doesn't change the compression ratio, which remains identical.
|
197
|
+
* Note 2 : In multi-threaded environments,
|
198
|
+
* use one different context per thread for parallel execution.
|
199
|
+
*/
|
189
200
|
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
|
190
201
|
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
|
191
202
|
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
|
192
203
|
|
193
204
|
/*! ZSTD_compressCCtx() :
|
194
|
-
* Same as ZSTD_compress(), using an explicit ZSTD_CCtx
|
195
|
-
*
|
196
|
-
*
|
205
|
+
* Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
|
206
|
+
* Important : in order to behave similarly to `ZSTD_compress()`,
|
207
|
+
* this function compresses at requested compression level,
|
208
|
+
* __ignoring any other parameter__ .
|
209
|
+
* If any advanced parameter was set using the advanced API,
|
210
|
+
* they will all be reset. Only `compressionLevel` remains.
|
211
|
+
*/
|
197
212
|
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
|
198
213
|
void* dst, size_t dstCapacity,
|
199
214
|
const void* src, size_t srcSize,
|
@@ -228,7 +243,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
|
|
228
243
|
* using ZSTD_CCtx_set*() functions.
|
229
244
|
* Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
|
230
245
|
* "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
|
231
|
-
*
|
246
|
+
* __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
|
232
247
|
*
|
233
248
|
* It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
|
234
249
|
*
|
@@ -256,18 +271,29 @@ typedef enum {
|
|
256
271
|
|
257
272
|
/* compression parameters
|
258
273
|
* Note: When compressing with a ZSTD_CDict these parameters are superseded
|
259
|
-
* by the parameters used to construct the ZSTD_CDict.
|
260
|
-
* for more info (superseded-by-cdict). */
|
261
|
-
ZSTD_c_compressionLevel=100, /*
|
274
|
+
* by the parameters used to construct the ZSTD_CDict.
|
275
|
+
* See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
|
276
|
+
ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
|
277
|
+
* Note that exact compression parameters are dynamically determined,
|
278
|
+
* depending on both compression level and srcSize (when known).
|
262
279
|
* Default level is ZSTD_CLEVEL_DEFAULT==3.
|
263
280
|
* Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
|
264
281
|
* Note 1 : it's possible to pass a negative compression level.
|
265
|
-
* Note 2 : setting a level
|
282
|
+
* Note 2 : setting a level does not automatically set all other compression parameters
|
283
|
+
* to default. Setting this will however eventually dynamically impact the compression
|
284
|
+
* parameters which have not been manually set. The manually set
|
285
|
+
* ones will 'stick'. */
|
286
|
+
/* Advanced compression parameters :
|
287
|
+
* It's possible to pin down compression parameters to some specific values.
|
288
|
+
* In which case, these values are no longer dynamically selected by the compressor */
|
266
289
|
ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
|
290
|
+
* This will set a memory budget for streaming decompression,
|
291
|
+
* with larger values requiring more memory
|
292
|
+
* and typically compressing more.
|
267
293
|
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
|
268
294
|
* Special: value 0 means "use default windowLog".
|
269
295
|
* Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
|
270
|
-
* requires explicitly allowing such
|
296
|
+
* requires explicitly allowing such size at streaming decompression stage. */
|
271
297
|
ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
|
272
298
|
* Resulting memory usage is (1 << (hashLog+2)).
|
273
299
|
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
|
@@ -278,13 +304,13 @@ typedef enum {
|
|
278
304
|
* Resulting memory usage is (1 << (chainLog+2)).
|
279
305
|
* Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
|
280
306
|
* Larger tables result in better and slower compression.
|
281
|
-
* This parameter is useless
|
307
|
+
* This parameter is useless for "fast" strategy.
|
282
308
|
* It's still useful when using "dfast" strategy,
|
283
309
|
* in which case it defines a secondary probe table.
|
284
310
|
* Special: value 0 means "use default chainLog". */
|
285
311
|
ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
|
286
312
|
* More attempts result in better and slower compression.
|
287
|
-
* This parameter is useless
|
313
|
+
* This parameter is useless for "fast" and "dFast" strategies.
|
288
314
|
* Special: value 0 means "use default searchLog". */
|
289
315
|
ZSTD_c_minMatch=105, /* Minimum size of searched matches.
|
290
316
|
* Note that Zstandard can still find matches of smaller size,
|
@@ -313,7 +339,9 @@ typedef enum {
|
|
313
339
|
* for large inputs, by finding large matches at long distance.
|
314
340
|
* It increases memory usage and window size.
|
315
341
|
* Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
|
316
|
-
* except when expressly set to a different value.
|
342
|
+
* except when expressly set to a different value.
|
343
|
+
* Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
|
344
|
+
* compression strategy >= ZSTD_btopt (== compression level 16+) */
|
317
345
|
ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
|
318
346
|
* Larger values increase memory usage and compression ratio,
|
319
347
|
* but decrease compression speed.
|
@@ -339,26 +367,30 @@ typedef enum {
|
|
339
367
|
ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
|
340
368
|
* Content size must be known at the beginning of compression.
|
341
369
|
* This is automatically the case when using ZSTD_compress2(),
|
342
|
-
* For streaming
|
370
|
+
* For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
|
343
371
|
ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
|
344
372
|
ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
|
345
373
|
|
346
374
|
/* multi-threading parameters */
|
347
|
-
/* These parameters are only
|
348
|
-
*
|
375
|
+
/* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
|
376
|
+
* Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
|
377
|
+
* In a situation where it's unknown if the linked library supports multi-threading or not,
|
378
|
+
* setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
|
379
|
+
*/
|
349
380
|
ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
|
350
|
-
* When nbWorkers >= 1, triggers asynchronous mode when
|
381
|
+
* When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
|
351
382
|
* ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
|
352
|
-
* while compression
|
383
|
+
* while compression is performed in parallel, within worker thread(s).
|
353
384
|
* (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
|
354
385
|
* in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
|
355
386
|
* More workers improve speed, but also increase memory usage.
|
356
|
-
* Default value is `0`, aka "single-threaded mode" : no worker is spawned,
|
387
|
+
* Default value is `0`, aka "single-threaded mode" : no worker is spawned,
|
388
|
+
* compression is performed inside Caller's thread, and all invocations are blocking */
|
357
389
|
ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
|
358
390
|
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
|
359
391
|
* 0 means default, which is dynamically determined based on compression parameters.
|
360
392
|
* Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
|
361
|
-
* The minimum size is automatically and transparently enforced */
|
393
|
+
* The minimum size is automatically and transparently enforced. */
|
362
394
|
ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
|
363
395
|
* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
|
364
396
|
* It helps preserve compression ratio, while each job is compressed in parallel.
|
@@ -380,6 +412,13 @@ typedef enum {
|
|
380
412
|
* ZSTD_c_forceMaxWindow
|
381
413
|
* ZSTD_c_forceAttachDict
|
382
414
|
* ZSTD_c_literalCompressionMode
|
415
|
+
* ZSTD_c_targetCBlockSize
|
416
|
+
* ZSTD_c_srcSizeHint
|
417
|
+
* ZSTD_c_enableDedicatedDictSearch
|
418
|
+
* ZSTD_c_stableInBuffer
|
419
|
+
* ZSTD_c_stableOutBuffer
|
420
|
+
* ZSTD_c_blockDelimiters
|
421
|
+
* ZSTD_c_validateSequences
|
383
422
|
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
|
384
423
|
* note : never ever use experimentalParam? names directly;
|
385
424
|
* also, the enums values themselves are unstable and can still change.
|
@@ -389,6 +428,13 @@ typedef enum {
|
|
389
428
|
ZSTD_c_experimentalParam3=1000,
|
390
429
|
ZSTD_c_experimentalParam4=1001,
|
391
430
|
ZSTD_c_experimentalParam5=1002,
|
431
|
+
ZSTD_c_experimentalParam6=1003,
|
432
|
+
ZSTD_c_experimentalParam7=1004,
|
433
|
+
ZSTD_c_experimentalParam8=1005,
|
434
|
+
ZSTD_c_experimentalParam9=1006,
|
435
|
+
ZSTD_c_experimentalParam10=1007,
|
436
|
+
ZSTD_c_experimentalParam11=1008,
|
437
|
+
ZSTD_c_experimentalParam12=1009
|
392
438
|
} ZSTD_cParameter;
|
393
439
|
|
394
440
|
typedef struct {
|
@@ -497,11 +543,17 @@ typedef enum {
|
|
497
543
|
/* note : additional experimental parameters are also available
|
498
544
|
* within the experimental section of the API.
|
499
545
|
* At the time of this writing, they include :
|
500
|
-
*
|
546
|
+
* ZSTD_d_format
|
547
|
+
* ZSTD_d_stableOutBuffer
|
548
|
+
* ZSTD_d_forceIgnoreChecksum
|
549
|
+
* ZSTD_d_refMultipleDDicts
|
501
550
|
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
|
502
551
|
* note : never ever use experimentalParam? names directly
|
503
552
|
*/
|
504
|
-
ZSTD_d_experimentalParam1=1000
|
553
|
+
ZSTD_d_experimentalParam1=1000,
|
554
|
+
ZSTD_d_experimentalParam2=1001,
|
555
|
+
ZSTD_d_experimentalParam3=1002,
|
556
|
+
ZSTD_d_experimentalParam4=1003
|
505
557
|
|
506
558
|
} ZSTD_dParameter;
|
507
559
|
|
@@ -637,8 +689,9 @@ typedef enum {
|
|
637
689
|
* - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
|
638
690
|
* - output->pos must be <= dstCapacity, input->pos must be <= srcSize
|
639
691
|
* - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
692
|
+
* - endOp must be a valid directive
|
640
693
|
* - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
|
641
|
-
* - When nbWorkers>=1, function is non-blocking : it
|
694
|
+
* - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
|
642
695
|
* and then immediately returns, just indicating that there is some data remaining to be flushed.
|
643
696
|
* The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
|
644
697
|
* - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
|
@@ -657,17 +710,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
|
|
657
710
|
ZSTD_inBuffer* input,
|
658
711
|
ZSTD_EndDirective endOp);
|
659
712
|
|
713
|
+
|
714
|
+
/* These buffer sizes are softly recommended.
|
715
|
+
* They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
|
716
|
+
* Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
|
717
|
+
* reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
|
718
|
+
*
|
719
|
+
* However, note that these recommendations are from the perspective of a C caller program.
|
720
|
+
* If the streaming interface is invoked from some other language,
|
721
|
+
* especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
|
722
|
+
* a major performance rule is to reduce crossing such interface to an absolute minimum.
|
723
|
+
* It's not rare that performance ends being spent more into the interface, rather than compression itself.
|
724
|
+
* In which cases, prefer using large buffers, as large as practical,
|
725
|
+
* for both input and output, to reduce the nb of roundtrips.
|
726
|
+
*/
|
660
727
|
ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
|
661
|
-
ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block
|
728
|
+
ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
|
662
729
|
|
663
|
-
|
664
|
-
|
665
|
-
*
|
730
|
+
|
731
|
+
/* *****************************************************************************
|
732
|
+
* This following is a legacy streaming API.
|
733
|
+
* It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
|
734
|
+
* It is redundant, but remains fully supported.
|
666
735
|
* Advanced parameters and dictionary compression can only be used through the
|
667
736
|
* new API.
|
668
737
|
******************************************************************************/
|
669
738
|
|
670
|
-
|
739
|
+
/*!
|
671
740
|
* Equivalent to:
|
672
741
|
*
|
673
742
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
@@ -675,16 +744,16 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
|
|
675
744
|
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
676
745
|
*/
|
677
746
|
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
|
678
|
-
|
747
|
+
/*!
|
679
748
|
* Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
|
680
749
|
* NOTE: The return value is different. ZSTD_compressStream() returns a hint for
|
681
750
|
* the next read size (if non-zero and not an error). ZSTD_compressStream2()
|
682
|
-
* returns the
|
751
|
+
* returns the minimum nb of bytes left to flush (if non-zero and not an error).
|
683
752
|
*/
|
684
753
|
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
685
|
-
|
754
|
+
/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
|
686
755
|
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
687
|
-
|
756
|
+
/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
|
688
757
|
ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
689
758
|
|
690
759
|
|
@@ -725,7 +794,7 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
|
725
794
|
|
726
795
|
/* This function is redundant with the advanced API and equivalent to:
|
727
796
|
*
|
728
|
-
* ZSTD_DCtx_reset(zds);
|
797
|
+
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
729
798
|
* ZSTD_DCtx_refDDict(zds, NULL);
|
730
799
|
*/
|
731
800
|
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
@@ -770,12 +839,17 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
|
770
839
|
typedef struct ZSTD_CDict_s ZSTD_CDict;
|
771
840
|
|
772
841
|
/*! ZSTD_createCDict() :
|
773
|
-
* When compressing multiple messages
|
774
|
-
*
|
842
|
+
* When compressing multiple messages or blocks using the same dictionary,
|
843
|
+
* it's recommended to digest the dictionary only once, since it's a costly operation.
|
844
|
+
* ZSTD_createCDict() will create a state from digesting a dictionary.
|
845
|
+
* The resulting state can be used for future compression operations with very limited startup cost.
|
775
846
|
* ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
776
|
-
*
|
777
|
-
* Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate
|
778
|
-
* Note : A ZSTD_CDict can be created from an empty dictBuffer,
|
847
|
+
* @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
|
848
|
+
* Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
|
849
|
+
* Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
|
850
|
+
* in which case the only thing that it transports is the @compressionLevel.
|
851
|
+
* This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
|
852
|
+
* expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
|
779
853
|
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
|
780
854
|
int compressionLevel);
|
781
855
|
|
@@ -876,7 +950,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
|
|
876
950
|
* Reference a prepared dictionary, to be used for all next compressed frames.
|
877
951
|
* Note that compression parameters are enforced from within CDict,
|
878
952
|
* and supersede any compression parameter previously set within CCtx.
|
879
|
-
* The parameters ignored are
|
953
|
+
* The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
|
880
954
|
* The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
|
881
955
|
* The dictionary will remain valid for future compressed frames using same CCtx.
|
882
956
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
@@ -902,7 +976,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
|
|
902
976
|
* Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
|
903
977
|
* It's a CPU consuming operation, with non-negligible impact on latency.
|
904
978
|
* If there is a need to use the same prefix multiple times, consider loadDictionary instead.
|
905
|
-
* Note 4 : By default, the prefix is interpreted as raw content (
|
979
|
+
* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
|
906
980
|
* Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
|
907
981
|
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
|
908
982
|
const void* prefix, size_t prefixSize);
|
@@ -927,6 +1001,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
|
|
927
1001
|
/*! ZSTD_DCtx_refDDict() :
|
928
1002
|
* Reference a prepared dictionary, to be used to decompress next frames.
|
929
1003
|
* The dictionary remains active for decompression of future frames using same DCtx.
|
1004
|
+
*
|
1005
|
+
* If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
|
1006
|
+
* will store the DDict references in a table, and the DDict used for decompression
|
1007
|
+
* will be determined at decompression time, as per the dict ID in the frame.
|
1008
|
+
* The memory for the table is allocated on the first call to refDDict, and can be
|
1009
|
+
* freed with ZSTD_freeDCtx().
|
1010
|
+
*
|
930
1011
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
931
1012
|
* Note 1 : Currently, only one dictionary can be managed.
|
932
1013
|
* Referencing a new dictionary effectively "discards" any previous one.
|
@@ -946,7 +1027,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
|
946
1027
|
* Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
|
947
1028
|
* Prefix buffer must remain unmodified up to the end of frame,
|
948
1029
|
* reached when ZSTD_decompressStream() returns 0.
|
949
|
-
* Note 3 : By default, the prefix is treated as raw content (
|
1030
|
+
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
|
950
1031
|
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
|
951
1032
|
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
|
952
1033
|
* A full dictionary is more costly, as it requires building tables.
|
@@ -969,7 +1050,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
969
1050
|
#endif /* ZSTD_H_235446 */
|
970
1051
|
|
971
1052
|
|
972
|
-
|
1053
|
+
/* **************************************************************************************
|
973
1054
|
* ADVANCED AND EXPERIMENTAL FUNCTIONS
|
974
1055
|
****************************************************************************************
|
975
1056
|
* The definitions in the following section are considered experimental.
|
@@ -991,8 +1072,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
991
1072
|
* Some of them might be removed in the future (especially when redundant with existing stable functions)
|
992
1073
|
* ***************************************************************************************/
|
993
1074
|
|
994
|
-
#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */
|
995
|
-
#define ZSTD_FRAMEHEADERSIZE_MIN 6
|
1075
|
+
#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
|
1076
|
+
#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
|
996
1077
|
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
|
997
1078
|
#define ZSTD_SKIPPABLEHEADERSIZE 8
|
998
1079
|
|
@@ -1037,6 +1118,12 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
1037
1118
|
#define ZSTD_LDM_HASHRATELOG_MIN 0
|
1038
1119
|
#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
|
1039
1120
|
|
1121
|
+
/* Advanced parameter bounds */
|
1122
|
+
#define ZSTD_TARGETCBLOCKSIZE_MIN 64
|
1123
|
+
#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
|
1124
|
+
#define ZSTD_SRCSIZEHINT_MIN 0
|
1125
|
+
#define ZSTD_SRCSIZEHINT_MAX INT_MAX
|
1126
|
+
|
1040
1127
|
/* internal */
|
1041
1128
|
#define ZSTD_HASHLOG3_MAX 17
|
1042
1129
|
|
@@ -1045,6 +1132,43 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
1045
1132
|
|
1046
1133
|
typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
|
1047
1134
|
|
1135
|
+
typedef struct {
|
1136
|
+
unsigned int offset; /* The offset of the match. (NOT the same as the offset code)
|
1137
|
+
* If offset == 0 and matchLength == 0, this sequence represents the last
|
1138
|
+
* literals in the block of litLength size.
|
1139
|
+
*/
|
1140
|
+
|
1141
|
+
unsigned int litLength; /* Literal length of the sequence. */
|
1142
|
+
unsigned int matchLength; /* Match length of the sequence. */
|
1143
|
+
|
1144
|
+
/* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
|
1145
|
+
* In this case, we will treat the sequence as a marker for a block boundary.
|
1146
|
+
*/
|
1147
|
+
|
1148
|
+
unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'.
|
1149
|
+
* Ranges from [0, 3].
|
1150
|
+
*
|
1151
|
+
* Repeat offsets are essentially previous offsets from previous sequences sorted in
|
1152
|
+
* recency order. For more detail, see doc/zstd_compression_format.md
|
1153
|
+
*
|
1154
|
+
* If rep == 0, then 'offset' does not contain a repeat offset.
|
1155
|
+
* If rep > 0:
|
1156
|
+
* If litLength != 0:
|
1157
|
+
* rep == 1 --> offset == repeat_offset_1
|
1158
|
+
* rep == 2 --> offset == repeat_offset_2
|
1159
|
+
* rep == 3 --> offset == repeat_offset_3
|
1160
|
+
* If litLength == 0:
|
1161
|
+
* rep == 1 --> offset == repeat_offset_2
|
1162
|
+
* rep == 2 --> offset == repeat_offset_3
|
1163
|
+
* rep == 3 --> offset == repeat_offset_1 - 1
|
1164
|
+
*
|
1165
|
+
* Note: This field is optional. ZSTD_generateSequences() will calculate the value of
|
1166
|
+
* 'rep', but repeat offsets do not necessarily need to be calculated from an external
|
1167
|
+
* sequence provider's perspective. For example, ZSTD_compressSequences() does not
|
1168
|
+
* use this 'rep' field at all (as of now).
|
1169
|
+
*/
|
1170
|
+
} ZSTD_Sequence;
|
1171
|
+
|
1048
1172
|
typedef struct {
|
1049
1173
|
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
|
1050
1174
|
unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
|
@@ -1074,32 +1198,35 @@ typedef enum {
|
|
1074
1198
|
|
1075
1199
|
typedef enum {
|
1076
1200
|
ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
|
1077
|
-
ZSTD_dlm_byRef = 1
|
1201
|
+
ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
|
1078
1202
|
} ZSTD_dictLoadMethod_e;
|
1079
1203
|
|
1080
1204
|
typedef enum {
|
1081
|
-
/* Opened question : should we have a format ZSTD_f_auto ?
|
1082
|
-
* Today, it would mean exactly the same as ZSTD_f_zstd1.
|
1083
|
-
* But, in the future, should several formats become supported,
|
1084
|
-
* on the compression side, it would mean "default format".
|
1085
|
-
* On the decompression side, it would mean "automatic format detection",
|
1086
|
-
* so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
|
1087
|
-
* Since meaning is a little different, another option could be to define different enums for compression and decompression.
|
1088
|
-
* This question could be kept for later, when there are actually multiple formats to support,
|
1089
|
-
* but there is also the question of pinning enum values, and pinning value `0` is especially important */
|
1090
1205
|
ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
|
1091
|
-
ZSTD_f_zstd1_magicless = 1
|
1206
|
+
ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
|
1092
1207
|
* Useful to save 4 bytes per generated frame.
|
1093
1208
|
* Decoder cannot recognise automatically this format, requiring this instruction. */
|
1094
1209
|
} ZSTD_format_e;
|
1095
1210
|
|
1211
|
+
typedef enum {
|
1212
|
+
/* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
|
1213
|
+
ZSTD_d_validateChecksum = 0,
|
1214
|
+
ZSTD_d_ignoreChecksum = 1
|
1215
|
+
} ZSTD_forceIgnoreChecksum_e;
|
1216
|
+
|
1217
|
+
typedef enum {
|
1218
|
+
/* Note: this enum controls ZSTD_d_refMultipleDDicts */
|
1219
|
+
ZSTD_rmd_refSingleDDict = 0,
|
1220
|
+
ZSTD_rmd_refMultipleDDicts = 1
|
1221
|
+
} ZSTD_refMultipleDDicts_e;
|
1222
|
+
|
1096
1223
|
typedef enum {
|
1097
1224
|
/* Note: this enum and the behavior it controls are effectively internal
|
1098
1225
|
* implementation details of the compressor. They are expected to continue
|
1099
1226
|
* to evolve and should be considered only in the context of extremely
|
1100
1227
|
* advanced performance tuning.
|
1101
1228
|
*
|
1102
|
-
* Zstd currently supports the use of a CDict in
|
1229
|
+
* Zstd currently supports the use of a CDict in three ways:
|
1103
1230
|
*
|
1104
1231
|
* - The contents of the CDict can be copied into the working context. This
|
1105
1232
|
* means that the compression can search both the dictionary and input
|
@@ -1115,6 +1242,12 @@ typedef enum {
|
|
1115
1242
|
* working context's tables can be reused). For small inputs, this can be
|
1116
1243
|
* faster than copying the CDict's tables.
|
1117
1244
|
*
|
1245
|
+
* - The CDict's tables are not used at all, and instead we use the working
|
1246
|
+
* context alone to reload the dictionary and use params based on the source
|
1247
|
+
* size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
|
1248
|
+
* This method is effective when the dictionary sizes are very small relative
|
1249
|
+
* to the input size, and the input size is fairly large to begin with.
|
1250
|
+
*
|
1118
1251
|
* Zstd has a simple internal heuristic that selects which strategy to use
|
1119
1252
|
* at the beginning of a compression. However, if experimentation shows that
|
1120
1253
|
* Zstd is making poor choices, it is possible to override that choice with
|
@@ -1123,6 +1256,7 @@ typedef enum {
|
|
1123
1256
|
ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
|
1124
1257
|
ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
|
1125
1258
|
ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
|
1259
|
+
ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
|
1126
1260
|
} ZSTD_dictAttachPref_e;
|
1127
1261
|
|
1128
1262
|
typedef enum {
|
@@ -1131,7 +1265,7 @@ typedef enum {
|
|
1131
1265
|
* levels will be compressed. */
|
1132
1266
|
ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
|
1133
1267
|
* emitted if Huffman compression is not profitable. */
|
1134
|
-
ZSTD_lcm_uncompressed = 2
|
1268
|
+
ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
|
1135
1269
|
} ZSTD_literalCompressionMode_e;
|
1136
1270
|
|
1137
1271
|
|
@@ -1162,12 +1296,12 @@ typedef enum {
|
|
1162
1296
|
* however it does mean that all frame data must be present and valid. */
|
1163
1297
|
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
|
1164
1298
|
|
1165
|
-
|
1299
|
+
/*! ZSTD_decompressBound() :
|
1166
1300
|
* `src` should point to the start of a series of ZSTD encoded and/or skippable frames
|
1167
1301
|
* `srcSize` must be the _exact_ size of this series
|
1168
1302
|
* (i.e. there should be a frame boundary at `src + srcSize`)
|
1169
1303
|
* @return : - upper-bound for the decompressed size of all data in all successive frames
|
1170
|
-
* - if an error
|
1304
|
+
* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
|
1171
1305
|
*
|
1172
1306
|
* note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
|
1173
1307
|
* note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
|
@@ -1183,6 +1317,92 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS
|
|
1183
1317
|
* or an error code (if srcSize is too small) */
|
1184
1318
|
ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
|
1185
1319
|
|
1320
|
+
typedef enum {
|
1321
|
+
ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
|
1322
|
+
ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
|
1323
|
+
} ZSTD_sequenceFormat_e;
|
1324
|
+
|
1325
|
+
/*! ZSTD_generateSequences() :
|
1326
|
+
* Generate sequences using ZSTD_compress2, given a source buffer.
|
1327
|
+
*
|
1328
|
+
* Each block will end with a dummy sequence
|
1329
|
+
* with offset == 0, matchLength == 0, and litLength == length of last literals.
|
1330
|
+
* litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
|
1331
|
+
* simply acts as a block delimiter.
|
1332
|
+
*
|
1333
|
+
* zc can be used to insert custom compression params.
|
1334
|
+
* This function invokes ZSTD_compress2
|
1335
|
+
*
|
1336
|
+
* The output of this function can be fed into ZSTD_compressSequences() with CCtx
|
1337
|
+
* setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
|
1338
|
+
* @return : number of sequences generated
|
1339
|
+
*/
|
1340
|
+
|
1341
|
+
ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
|
1342
|
+
size_t outSeqsSize, const void* src, size_t srcSize);
|
1343
|
+
|
1344
|
+
/*! ZSTD_mergeBlockDelimiters() :
|
1345
|
+
* Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
|
1346
|
+
* by merging them into into the literals of the next sequence.
|
1347
|
+
*
|
1348
|
+
* As such, the final generated result has no explicit representation of block boundaries,
|
1349
|
+
* and the final last literals segment is not represented in the sequences.
|
1350
|
+
*
|
1351
|
+
* The output of this function can be fed into ZSTD_compressSequences() with CCtx
|
1352
|
+
* setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
|
1353
|
+
* @return : number of sequences left after merging
|
1354
|
+
*/
|
1355
|
+
ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
|
1356
|
+
|
1357
|
+
/*! ZSTD_compressSequences() :
|
1358
|
+
* Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
|
1359
|
+
* If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
|
1360
|
+
* The entire source is compressed into a single frame.
|
1361
|
+
*
|
1362
|
+
* The compression behavior changes based on cctx params. In particular:
|
1363
|
+
* If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
|
1364
|
+
* no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
|
1365
|
+
* the block size derived from the cctx, and sequences may be split. This is the default setting.
|
1366
|
+
*
|
1367
|
+
* If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
|
1368
|
+
* block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
|
1369
|
+
*
|
1370
|
+
* If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
|
1371
|
+
* behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
|
1372
|
+
* specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
|
1373
|
+
*
|
1374
|
+
* In addition to the two adjustable experimental params, there are other important cctx params.
|
1375
|
+
* - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
|
1376
|
+
* - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
|
1377
|
+
* - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
|
1378
|
+
* is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
|
1379
|
+
*
|
1380
|
+
* Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
|
1381
|
+
* Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
|
1382
|
+
* and cannot emit an RLE block that disagrees with the repcode history
|
1383
|
+
* @return : final compressed size or a ZSTD error.
|
1384
|
+
*/
|
1385
|
+
ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
|
1386
|
+
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
1387
|
+
const void* src, size_t srcSize);
|
1388
|
+
|
1389
|
+
|
1390
|
+
/*! ZSTD_writeSkippableFrame() :
|
1391
|
+
* Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
|
1392
|
+
*
|
1393
|
+
* Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
|
1394
|
+
* ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
|
1395
|
+
* As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
|
1396
|
+
* the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
|
1397
|
+
*
|
1398
|
+
* Returns an error if destination buffer is not large enough, if the source size is not representable
|
1399
|
+
* with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
|
1400
|
+
*
|
1401
|
+
* @return : number of bytes written or a ZSTD error.
|
1402
|
+
*/
|
1403
|
+
ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
|
1404
|
+
const void* src, size_t srcSize, unsigned magicVariant);
|
1405
|
+
|
1186
1406
|
|
1187
1407
|
/***************************************
|
1188
1408
|
* Memory management
|
@@ -1191,12 +1411,26 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
|
|
1191
1411
|
/*! ZSTD_estimate*() :
|
1192
1412
|
* These functions make it possible to estimate memory usage
|
1193
1413
|
* of a future {D,C}Ctx, before its creation.
|
1194
|
-
*
|
1195
|
-
*
|
1196
|
-
*
|
1197
|
-
*
|
1198
|
-
*
|
1199
|
-
*
|
1414
|
+
*
|
1415
|
+
* ZSTD_estimateCCtxSize() will provide a memory budget large enough
|
1416
|
+
* for any compression level up to selected one.
|
1417
|
+
* Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
|
1418
|
+
* does not include space for a window buffer.
|
1419
|
+
* Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
|
1420
|
+
* The estimate will assume the input may be arbitrarily large,
|
1421
|
+
* which is the worst case.
|
1422
|
+
*
|
1423
|
+
* When srcSize can be bound by a known and rather "small" value,
|
1424
|
+
* this fact can be used to provide a tighter estimation
|
1425
|
+
* because the CCtx compression context will need less memory.
|
1426
|
+
* This tighter estimation can be provided by more advanced functions
|
1427
|
+
* ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
|
1428
|
+
* and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
|
1429
|
+
* Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
|
1430
|
+
*
|
1431
|
+
* Note 2 : only single-threaded compression is supported.
|
1432
|
+
* ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
|
1433
|
+
*/
|
1200
1434
|
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
|
1201
1435
|
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
|
1202
1436
|
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
|
@@ -1279,7 +1513,11 @@ ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict(
|
|
1279
1513
|
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
1280
1514
|
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
1281
1515
|
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
1282
|
-
static
|
1516
|
+
static
|
1517
|
+
#ifdef __GNUC__
|
1518
|
+
__attribute__((__unused__))
|
1519
|
+
#endif
|
1520
|
+
ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
|
1283
1521
|
|
1284
1522
|
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
1285
1523
|
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
@@ -1292,11 +1530,37 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
|
|
1292
1530
|
ZSTD_compressionParameters cParams,
|
1293
1531
|
ZSTD_customMem customMem);
|
1294
1532
|
|
1295
|
-
|
1296
|
-
|
1297
|
-
|
1298
|
-
|
1533
|
+
/* ! Thread pool :
|
1534
|
+
* These prototypes make it possible to share a thread pool among multiple compression contexts.
|
1535
|
+
* This can limit resources for applications with multiple threads where each one uses
|
1536
|
+
* a threaded compression mode (via ZSTD_c_nbWorkers parameter).
|
1537
|
+
* ZSTD_createThreadPool creates a new thread pool with a given number of threads.
|
1538
|
+
* Note that the lifetime of such pool must exist while being used.
|
1539
|
+
* ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
|
1540
|
+
* to use an internal thread pool).
|
1541
|
+
* ZSTD_freeThreadPool frees a thread pool.
|
1542
|
+
*/
|
1543
|
+
typedef struct POOL_ctx_s ZSTD_threadPool;
|
1544
|
+
ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
|
1545
|
+
ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);
|
1546
|
+
ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
|
1547
|
+
|
1299
1548
|
|
1549
|
+
/*
|
1550
|
+
* This API is temporary and is expected to change or disappear in the future!
|
1551
|
+
*/
|
1552
|
+
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
|
1553
|
+
const void* dict, size_t dictSize,
|
1554
|
+
ZSTD_dictLoadMethod_e dictLoadMethod,
|
1555
|
+
ZSTD_dictContentType_e dictContentType,
|
1556
|
+
const ZSTD_CCtx_params* cctxParams,
|
1557
|
+
ZSTD_customMem customMem);
|
1558
|
+
|
1559
|
+
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(
|
1560
|
+
const void* dict, size_t dictSize,
|
1561
|
+
ZSTD_dictLoadMethod_e dictLoadMethod,
|
1562
|
+
ZSTD_dictContentType_e dictContentType,
|
1563
|
+
ZSTD_customMem customMem);
|
1300
1564
|
|
1301
1565
|
|
1302
1566
|
/***************************************
|
@@ -1307,9 +1571,16 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS
|
|
1307
1571
|
* Create a digested dictionary for compression
|
1308
1572
|
* Dictionary content is just referenced, not duplicated.
|
1309
1573
|
* As a consequence, `dictBuffer` **must** outlive CDict,
|
1310
|
-
* and its content must remain unmodified throughout the lifetime of CDict.
|
1574
|
+
* and its content must remain unmodified throughout the lifetime of CDict.
|
1575
|
+
* note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
|
1311
1576
|
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
|
1312
1577
|
|
1578
|
+
/*! ZSTD_getDictID_fromCDict() :
|
1579
|
+
* Provides the dictID of the dictionary loaded into `cdict`.
|
1580
|
+
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
|
1581
|
+
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
|
1582
|
+
ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
|
1583
|
+
|
1313
1584
|
/*! ZSTD_getCParams() :
|
1314
1585
|
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
|
1315
1586
|
* `estimatedSrcSize` value is optional, select 0 if not known */
|
@@ -1334,7 +1605,9 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
|
1334
1605
|
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
|
1335
1606
|
|
1336
1607
|
/*! ZSTD_compress_advanced() :
|
1337
|
-
*
|
1608
|
+
* Note : this function is now DEPRECATED.
|
1609
|
+
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
|
1610
|
+
* This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */
|
1338
1611
|
ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
|
1339
1612
|
void* dst, size_t dstCapacity,
|
1340
1613
|
const void* src, size_t srcSize,
|
@@ -1342,7 +1615,9 @@ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
|
|
1342
1615
|
ZSTD_parameters params);
|
1343
1616
|
|
1344
1617
|
/*! ZSTD_compress_usingCDict_advanced() :
|
1345
|
-
*
|
1618
|
+
* Note : this function is now REDUNDANT.
|
1619
|
+
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
|
1620
|
+
* This prototype will be marked as deprecated and generate compilation warning in some future version */
|
1346
1621
|
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
|
1347
1622
|
void* dst, size_t dstCapacity,
|
1348
1623
|
const void* src, size_t srcSize,
|
@@ -1409,12 +1684,160 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
|
|
1409
1684
|
*/
|
1410
1685
|
#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
|
1411
1686
|
|
1687
|
+
/* Tries to fit compressed block size to be around targetCBlockSize.
|
1688
|
+
* No target when targetCBlockSize == 0.
|
1689
|
+
* There is no guarantee on compressed block size (default:0) */
|
1690
|
+
#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
|
1691
|
+
|
1692
|
+
/* User's best guess of source size.
|
1693
|
+
* Hint is not valid when srcSizeHint == 0.
|
1694
|
+
* There is no guarantee that hint is close to actual source size,
|
1695
|
+
* but compression ratio may regress significantly if guess considerably underestimates */
|
1696
|
+
#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
|
1697
|
+
|
1698
|
+
/* Controls whether the new and experimental "dedicated dictionary search
|
1699
|
+
* structure" can be used. This feature is still rough around the edges, be
|
1700
|
+
* prepared for surprising behavior!
|
1701
|
+
*
|
1702
|
+
* How to use it:
|
1703
|
+
*
|
1704
|
+
* When using a CDict, whether to use this feature or not is controlled at
|
1705
|
+
* CDict creation, and it must be set in a CCtxParams set passed into that
|
1706
|
+
* construction (via ZSTD_createCDict_advanced2()). A compression will then
|
1707
|
+
* use the feature or not based on how the CDict was constructed; the value of
|
1708
|
+
* this param, set in the CCtx, will have no effect.
|
1709
|
+
*
|
1710
|
+
* However, when a dictionary buffer is passed into a CCtx, such as via
|
1711
|
+
* ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
|
1712
|
+
* whether the CDict that is created internally can use the feature or not.
|
1713
|
+
*
|
1714
|
+
* What it does:
|
1715
|
+
*
|
1716
|
+
* Normally, the internal data structures of the CDict are analogous to what
|
1717
|
+
* would be stored in a CCtx after compressing the contents of a dictionary.
|
1718
|
+
* To an approximation, a compression using a dictionary can then use those
|
1719
|
+
* data structures to simply continue what is effectively a streaming
|
1720
|
+
* compression where the simulated compression of the dictionary left off.
|
1721
|
+
* Which is to say, the search structures in the CDict are normally the same
|
1722
|
+
* format as in the CCtx.
|
1723
|
+
*
|
1724
|
+
* It is possible to do better, since the CDict is not like a CCtx: the search
|
1725
|
+
* structures are written once during CDict creation, and then are only read
|
1726
|
+
* after that, while the search structures in the CCtx are both read and
|
1727
|
+
* written as the compression goes along. This means we can choose a search
|
1728
|
+
* structure for the dictionary that is read-optimized.
|
1729
|
+
*
|
1730
|
+
* This feature enables the use of that different structure.
|
1731
|
+
*
|
1732
|
+
* Note that some of the members of the ZSTD_compressionParameters struct have
|
1733
|
+
* different semantics and constraints in the dedicated search structure. It is
|
1734
|
+
* highly recommended that you simply set a compression level in the CCtxParams
|
1735
|
+
* you pass into the CDict creation call, and avoid messing with the cParams
|
1736
|
+
* directly.
|
1737
|
+
*
|
1738
|
+
* Effects:
|
1739
|
+
*
|
1740
|
+
* This will only have any effect when the selected ZSTD_strategy
|
1741
|
+
* implementation supports this feature. Currently, that's limited to
|
1742
|
+
* ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
|
1743
|
+
*
|
1744
|
+
* Note that this means that the CDict tables can no longer be copied into the
|
1745
|
+
* CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
|
1746
|
+
* useable. The dictionary can only be attached or reloaded.
|
1747
|
+
*
|
1748
|
+
* In general, you should expect compression to be faster--sometimes very much
|
1749
|
+
* so--and CDict creation to be slightly slower. Eventually, we will probably
|
1750
|
+
* make this mode the default.
|
1751
|
+
*/
|
1752
|
+
#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
|
1753
|
+
|
1754
|
+
/* ZSTD_c_stableInBuffer
|
1755
|
+
* Experimental parameter.
|
1756
|
+
* Default is 0 == disabled. Set to 1 to enable.
|
1757
|
+
*
|
1758
|
+
* Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same
|
1759
|
+
* between calls, except for the modifications that zstd makes to pos (the
|
1760
|
+
* caller must not modify pos). This is checked by the compressor, and
|
1761
|
+
* compression will fail if it ever changes. This means the only flush
|
1762
|
+
* mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end
|
1763
|
+
* is not used. The data in the ZSTD_inBuffer in the range [src, src + pos)
|
1764
|
+
* MUST not be modified during compression or you will get data corruption.
|
1765
|
+
*
|
1766
|
+
* When this flag is enabled zstd won't allocate an input window buffer,
|
1767
|
+
* because the user guarantees it can reference the ZSTD_inBuffer until
|
1768
|
+
* the frame is complete. But, it will still allocate an output buffer
|
1769
|
+
* large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
|
1770
|
+
* avoid the memcpy() from the input buffer to the input window buffer.
|
1771
|
+
*
|
1772
|
+
* NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
|
1773
|
+
* That means this flag cannot be used with ZSTD_compressStream().
|
1774
|
+
*
|
1775
|
+
* NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
|
1776
|
+
* this flag is ALWAYS memory safe, and will never access out-of-bounds
|
1777
|
+
* memory. However, compression WILL fail if you violate the preconditions.
|
1778
|
+
*
|
1779
|
+
* WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST
|
1780
|
+
* not be modified during compression or you will get data corruption. This
|
1781
|
+
* is because zstd needs to reference data in the ZSTD_inBuffer to find
|
1782
|
+
* matches. Normally zstd maintains its own window buffer for this purpose,
|
1783
|
+
* but passing this flag tells zstd to use the user provided buffer.
|
1784
|
+
*/
|
1785
|
+
#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
|
1786
|
+
|
1787
|
+
/* ZSTD_c_stableOutBuffer
|
1788
|
+
* Experimental parameter.
|
1789
|
+
* Default is 0 == disabled. Set to 1 to enable.
|
1790
|
+
*
|
1791
|
+
* Tells he compressor that the ZSTD_outBuffer will not be resized between
|
1792
|
+
* calls. Specifically: (out.size - out.pos) will never grow. This gives the
|
1793
|
+
* compressor the freedom to say: If the compressed data doesn't fit in the
|
1794
|
+
* output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
|
1795
|
+
* always decompress directly into the output buffer, instead of decompressing
|
1796
|
+
* into an internal buffer and copying to the output buffer.
|
1797
|
+
*
|
1798
|
+
* When this flag is enabled zstd won't allocate an output buffer, because
|
1799
|
+
* it can write directly to the ZSTD_outBuffer. It will still allocate the
|
1800
|
+
* input window buffer (see ZSTD_c_stableInBuffer).
|
1801
|
+
*
|
1802
|
+
* Zstd will check that (out.size - out.pos) never grows and return an error
|
1803
|
+
* if it does. While not strictly necessary, this should prevent surprises.
|
1804
|
+
*/
|
1805
|
+
#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
|
1806
|
+
|
1807
|
+
/* ZSTD_c_blockDelimiters
|
1808
|
+
* Default is 0 == ZSTD_sf_noBlockDelimiters.
|
1809
|
+
*
|
1810
|
+
* For use with sequence compression API: ZSTD_compressSequences().
|
1811
|
+
*
|
1812
|
+
* Designates whether or not the given array of ZSTD_Sequence contains block delimiters
|
1813
|
+
* and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
|
1814
|
+
* See the definition of ZSTD_Sequence for more specifics.
|
1815
|
+
*/
|
1816
|
+
#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
|
1817
|
+
|
1818
|
+
/* ZSTD_c_validateSequences
|
1819
|
+
* Default is 0 == disabled. Set to 1 to enable sequence validation.
|
1820
|
+
*
|
1821
|
+
* For use with sequence compression API: ZSTD_compressSequences().
|
1822
|
+
* Designates whether or not we validate sequences provided to ZSTD_compressSequences()
|
1823
|
+
* during function execution.
|
1824
|
+
*
|
1825
|
+
* Without validation, providing a sequence that does not conform to the zstd spec will cause
|
1826
|
+
* undefined behavior, and may produce a corrupted block.
|
1827
|
+
*
|
1828
|
+
* With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for
|
1829
|
+
* specifics regarding offset/matchlength requirements) then the function will bail out and
|
1830
|
+
* return an error.
|
1831
|
+
*
|
1832
|
+
*/
|
1833
|
+
#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
|
1834
|
+
|
1412
1835
|
/*! ZSTD_CCtx_getParameter() :
|
1413
1836
|
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
|
1414
1837
|
* and store it into int* value.
|
1415
1838
|
* @return : 0, or an error code (which can be tested with ZSTD_isError()).
|
1416
1839
|
*/
|
1417
|
-
ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
|
1840
|
+
ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
|
1418
1841
|
|
1419
1842
|
|
1420
1843
|
/*! ZSTD_CCtx_params :
|
@@ -1457,8 +1880,10 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, Z
|
|
1457
1880
|
/*! ZSTD_CCtxParams_setParameter() :
|
1458
1881
|
* Similar to ZSTD_CCtx_setParameter.
|
1459
1882
|
* Set one compression parameter, selected by enum ZSTD_cParameter.
|
1460
|
-
* Parameters must be applied to a ZSTD_CCtx using
|
1461
|
-
*
|
1883
|
+
* Parameters must be applied to a ZSTD_CCtx using
|
1884
|
+
* ZSTD_CCtx_setParametersUsingCCtxParams().
|
1885
|
+
* @result : a code representing success or failure (which can be tested with
|
1886
|
+
* ZSTD_isError()).
|
1462
1887
|
*/
|
1463
1888
|
ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
|
1464
1889
|
|
@@ -1467,7 +1892,7 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_c
|
|
1467
1892
|
* Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
|
1468
1893
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
1469
1894
|
*/
|
1470
|
-
ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
|
1895
|
+
ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
|
1471
1896
|
|
1472
1897
|
/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
|
1473
1898
|
* Apply a set of ZSTD_CCtx_params to the compression context.
|
@@ -1538,11 +1963,84 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre
|
|
1538
1963
|
*/
|
1539
1964
|
ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
|
1540
1965
|
|
1966
|
+
/*! ZSTD_DCtx_getParameter() :
|
1967
|
+
* Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
|
1968
|
+
* and store it into int* value.
|
1969
|
+
* @return : 0, or an error code (which can be tested with ZSTD_isError()).
|
1970
|
+
*/
|
1971
|
+
ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
|
1972
|
+
|
1541
1973
|
/* ZSTD_d_format
|
1542
1974
|
* experimental parameter,
|
1543
1975
|
* allowing selection between ZSTD_format_e input compression formats
|
1544
1976
|
*/
|
1545
1977
|
#define ZSTD_d_format ZSTD_d_experimentalParam1
|
1978
|
+
/* ZSTD_d_stableOutBuffer
|
1979
|
+
* Experimental parameter.
|
1980
|
+
* Default is 0 == disabled. Set to 1 to enable.
|
1981
|
+
*
|
1982
|
+
* Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
|
1983
|
+
* between calls, except for the modifications that zstd makes to pos (the
|
1984
|
+
* caller must not modify pos). This is checked by the decompressor, and
|
1985
|
+
* decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
|
1986
|
+
* MUST be large enough to fit the entire decompressed frame. This will be
|
1987
|
+
* checked when the frame content size is known. The data in the ZSTD_outBuffer
|
1988
|
+
* in the range [dst, dst + pos) MUST not be modified during decompression
|
1989
|
+
* or you will get data corruption.
|
1990
|
+
*
|
1991
|
+
* When this flags is enabled zstd won't allocate an output buffer, because
|
1992
|
+
* it can write directly to the ZSTD_outBuffer, but it will still allocate
|
1993
|
+
* an input buffer large enough to fit any compressed block. This will also
|
1994
|
+
* avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
|
1995
|
+
* If you need to avoid the input buffer allocation use the buffer-less
|
1996
|
+
* streaming API.
|
1997
|
+
*
|
1998
|
+
* NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
|
1999
|
+
* this flag is ALWAYS memory safe, and will never access out-of-bounds
|
2000
|
+
* memory. However, decompression WILL fail if you violate the preconditions.
|
2001
|
+
*
|
2002
|
+
* WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
|
2003
|
+
* not be modified during decompression or you will get data corruption. This
|
2004
|
+
* is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
|
2005
|
+
* matches. Normally zstd maintains its own buffer for this purpose, but passing
|
2006
|
+
* this flag tells zstd to use the user provided buffer.
|
2007
|
+
*/
|
2008
|
+
#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
|
2009
|
+
|
2010
|
+
/* ZSTD_d_forceIgnoreChecksum
|
2011
|
+
* Experimental parameter.
|
2012
|
+
* Default is 0 == disabled. Set to 1 to enable
|
2013
|
+
*
|
2014
|
+
* Tells the decompressor to skip checksum validation during decompression, regardless
|
2015
|
+
* of whether checksumming was specified during compression. This offers some
|
2016
|
+
* slight performance benefits, and may be useful for debugging.
|
2017
|
+
* Param has values of type ZSTD_forceIgnoreChecksum_e
|
2018
|
+
*/
|
2019
|
+
#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
|
2020
|
+
|
2021
|
+
/* ZSTD_d_refMultipleDDicts
|
2022
|
+
* Experimental parameter.
|
2023
|
+
* Default is 0 == disabled. Set to 1 to enable
|
2024
|
+
*
|
2025
|
+
* If enabled and dctx is allocated on the heap, then additional memory will be allocated
|
2026
|
+
* to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
|
2027
|
+
* using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
|
2028
|
+
* store all references. At decompression time, the appropriate dictID is selected
|
2029
|
+
* from the set of DDicts based on the dictID in the frame.
|
2030
|
+
*
|
2031
|
+
* Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
|
2032
|
+
*
|
2033
|
+
* Param has values of byte ZSTD_refMultipleDDicts_e
|
2034
|
+
*
|
2035
|
+
* WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
|
2036
|
+
* allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
|
2037
|
+
* Memory is allocated as per ZSTD_DCtx::customMem.
|
2038
|
+
*
|
2039
|
+
* Although this function allocates memory for the table, the user is still responsible for
|
2040
|
+
* memory management of the underlying ZSTD_DDict* themselves.
|
2041
|
+
*/
|
2042
|
+
#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
|
2043
|
+
|
1546
2044
|
|
1547
2045
|
/*! ZSTD_DCtx_setFormat() :
|
1548
2046
|
* Instruct the decoder context about what kind of data to decode next.
|
@@ -1571,7 +2069,8 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
|
|
1571
2069
|
********************************************************************/
|
1572
2070
|
|
1573
2071
|
/*===== Advanced Streaming compression functions =====*/
|
1574
|
-
|
2072
|
+
|
2073
|
+
/*! ZSTD_initCStream_srcSize() :
|
1575
2074
|
* This function is deprecated, and equivalent to:
|
1576
2075
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
1577
2076
|
* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
|
@@ -1581,9 +2080,14 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
|
|
1581
2080
|
* pledgedSrcSize must be correct. If it is not known at init time, use
|
1582
2081
|
* ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
|
1583
2082
|
* "0" also disables frame content size field. It may be enabled in the future.
|
2083
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
1584
2084
|
*/
|
1585
|
-
ZSTDLIB_API size_t
|
1586
|
-
|
2085
|
+
ZSTDLIB_API size_t
|
2086
|
+
ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
|
2087
|
+
int compressionLevel,
|
2088
|
+
unsigned long long pledgedSrcSize);
|
2089
|
+
|
2090
|
+
/*! ZSTD_initCStream_usingDict() :
|
1587
2091
|
* This function is deprecated, and is equivalent to:
|
1588
2092
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
1589
2093
|
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
@@ -1591,42 +2095,66 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
|
|
1591
2095
|
*
|
1592
2096
|
* Creates of an internal CDict (incompatible with static CCtx), except if
|
1593
2097
|
* dict == NULL or dictSize < 8, in which case no dict is used.
|
1594
|
-
* Note: dict is loaded with
|
2098
|
+
* Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
|
1595
2099
|
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
|
2100
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
1596
2101
|
*/
|
1597
|
-
ZSTDLIB_API size_t
|
1598
|
-
|
2102
|
+
ZSTDLIB_API size_t
|
2103
|
+
ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
|
2104
|
+
const void* dict, size_t dictSize,
|
2105
|
+
int compressionLevel);
|
2106
|
+
|
2107
|
+
/*! ZSTD_initCStream_advanced() :
|
1599
2108
|
* This function is deprecated, and is approximately equivalent to:
|
1600
2109
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
1601
|
-
*
|
2110
|
+
* // Pseudocode: Set each zstd parameter and leave the rest as-is.
|
2111
|
+
* for ((param, value) : params) {
|
2112
|
+
* ZSTD_CCtx_setParameter(zcs, param, value);
|
2113
|
+
* }
|
1602
2114
|
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
1603
2115
|
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
|
1604
2116
|
*
|
1605
|
-
*
|
1606
|
-
*
|
2117
|
+
* dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
|
2118
|
+
* pledgedSrcSize must be correct.
|
2119
|
+
* If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
|
2120
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
1607
2121
|
*/
|
1608
|
-
ZSTDLIB_API size_t
|
1609
|
-
|
1610
|
-
|
2122
|
+
ZSTDLIB_API size_t
|
2123
|
+
ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
2124
|
+
const void* dict, size_t dictSize,
|
2125
|
+
ZSTD_parameters params,
|
2126
|
+
unsigned long long pledgedSrcSize);
|
2127
|
+
|
2128
|
+
/*! ZSTD_initCStream_usingCDict() :
|
1611
2129
|
* This function is deprecated, and equivalent to:
|
1612
2130
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
1613
2131
|
* ZSTD_CCtx_refCDict(zcs, cdict);
|
1614
2132
|
*
|
1615
2133
|
* note : cdict will just be referenced, and must outlive compression session
|
2134
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
1616
2135
|
*/
|
1617
2136
|
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
|
1618
|
-
|
1619
|
-
|
2137
|
+
|
2138
|
+
/*! ZSTD_initCStream_usingCDict_advanced() :
|
2139
|
+
* This function is DEPRECATED, and is approximately equivalent to:
|
1620
2140
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
1621
|
-
*
|
2141
|
+
* // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
|
2142
|
+
* for ((fParam, value) : fParams) {
|
2143
|
+
* ZSTD_CCtx_setParameter(zcs, fParam, value);
|
2144
|
+
* }
|
1622
2145
|
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
1623
2146
|
* ZSTD_CCtx_refCDict(zcs, cdict);
|
1624
2147
|
*
|
1625
2148
|
* same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
|
1626
2149
|
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
|
1627
2150
|
* value ZSTD_CONTENTSIZE_UNKNOWN.
|
2151
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
1628
2152
|
*/
|
1629
|
-
ZSTDLIB_API size_t
|
2153
|
+
ZSTDLIB_API size_t
|
2154
|
+
ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
|
2155
|
+
const ZSTD_CDict* cdict,
|
2156
|
+
ZSTD_frameParameters fParams,
|
2157
|
+
unsigned long long pledgedSrcSize);
|
1630
2158
|
|
1631
2159
|
/*! ZSTD_resetCStream() :
|
1632
2160
|
* This function is deprecated, and is equivalent to:
|
@@ -1641,6 +2169,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
|
|
1641
2169
|
* For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
|
1642
2170
|
* but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
|
1643
2171
|
* @return : 0, or an error code (which can be tested using ZSTD_isError())
|
2172
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
1644
2173
|
*/
|
1645
2174
|
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
|
1646
2175
|
|
@@ -1679,30 +2208,36 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
|
|
1679
2208
|
|
1680
2209
|
|
1681
2210
|
/*===== Advanced Streaming decompression functions =====*/
|
1682
|
-
|
2211
|
+
|
2212
|
+
/*!
|
1683
2213
|
* This function is deprecated, and is equivalent to:
|
1684
2214
|
*
|
1685
2215
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
1686
2216
|
* ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
|
1687
2217
|
*
|
1688
2218
|
* note: no dictionary will be used if dict == NULL or dictSize < 8
|
2219
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
1689
2220
|
*/
|
1690
2221
|
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
|
1691
|
-
|
2222
|
+
|
2223
|
+
/*!
|
1692
2224
|
* This function is deprecated, and is equivalent to:
|
1693
2225
|
*
|
1694
2226
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
1695
2227
|
* ZSTD_DCtx_refDDict(zds, ddict);
|
1696
2228
|
*
|
1697
2229
|
* note : ddict is referenced, it must outlive decompression session
|
2230
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
1698
2231
|
*/
|
1699
2232
|
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
|
1700
|
-
|
2233
|
+
|
2234
|
+
/*!
|
1701
2235
|
* This function is deprecated, and is equivalent to:
|
1702
2236
|
*
|
1703
2237
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
1704
2238
|
*
|
1705
2239
|
* re-use decompression parameters from previous init; saves dictionary loading
|
2240
|
+
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
1706
2241
|
*/
|
1707
2242
|
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
|
1708
2243
|
|
@@ -1758,7 +2293,7 @@ ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstC
|
|
1758
2293
|
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
1759
2294
|
|
1760
2295
|
|
1761
|
-
|
2296
|
+
/**
|
1762
2297
|
Buffer-less streaming decompression (synchronous mode)
|
1763
2298
|
|
1764
2299
|
A ZSTD_DCtx object is required to track streaming operations.
|
@@ -1843,7 +2378,7 @@ typedef struct {
|
|
1843
2378
|
unsigned checksumFlag;
|
1844
2379
|
} ZSTD_frameHeader;
|
1845
2380
|
|
1846
|
-
|
2381
|
+
/*! ZSTD_getFrameHeader() :
|
1847
2382
|
* decode Frame Header, or requires larger `srcSize`.
|
1848
2383
|
* @return : 0, `zfhPtr` is correctly filled,
|
1849
2384
|
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
@@ -1876,8 +2411,8 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
|
1876
2411
|
|
1877
2412
|
/*!
|
1878
2413
|
Block functions produce and decode raw zstd blocks, without frame metadata.
|
1879
|
-
Frame metadata cost is typically ~
|
1880
|
-
|
2414
|
+
Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
2415
|
+
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
|
1881
2416
|
|
1882
2417
|
A few rules to respect :
|
1883
2418
|
- Compressing and decompressing require a context structure
|
@@ -1888,12 +2423,14 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
|
1888
2423
|
+ copyCCtx() and copyDCtx() can be used too
|
1889
2424
|
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
|
1890
2425
|
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
|
1891
|
-
+ For inputs larger than a single block,
|
1892
|
-
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
|
1893
|
-
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero
|
1894
|
-
In which case, nothing is produced into `dst` !
|
1895
|
-
+ User
|
1896
|
-
+
|
2426
|
+
+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.
|
2427
|
+
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
|
2428
|
+
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
|
2429
|
+
===> In which case, nothing is produced into `dst` !
|
2430
|
+
+ User __must__ test for such outcome and deal directly with uncompressed data
|
2431
|
+
+ A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
|
2432
|
+
Doing so would mess up with statistics history, leading to potential data corruption.
|
2433
|
+
+ ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
|
1897
2434
|
+ In case of multiple successive blocks, should some of them be uncompressed,
|
1898
2435
|
decoder must be informed of their existence in order to follow proper history.
|
1899
2436
|
Use ZSTD_insertBlock() for such a case.
|