extzstd 0.3.2 → 0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -3
- data/contrib/zstd/CHANGELOG +225 -1
- data/contrib/zstd/CONTRIBUTING.md +158 -75
- data/contrib/zstd/LICENSE +4 -4
- data/contrib/zstd/Makefile +106 -69
- data/contrib/zstd/Package.swift +36 -0
- data/contrib/zstd/README.md +64 -36
- data/contrib/zstd/SECURITY.md +15 -0
- data/contrib/zstd/TESTING.md +2 -3
- data/contrib/zstd/lib/BUCK +5 -7
- data/contrib/zstd/lib/Makefile +117 -199
- data/contrib/zstd/lib/README.md +37 -7
- data/contrib/zstd/lib/common/allocations.h +55 -0
- data/contrib/zstd/lib/common/bits.h +200 -0
- data/contrib/zstd/lib/common/bitstream.h +80 -86
- data/contrib/zstd/lib/common/compiler.h +225 -63
- data/contrib/zstd/lib/common/cpu.h +37 -1
- data/contrib/zstd/lib/common/debug.c +7 -1
- data/contrib/zstd/lib/common/debug.h +21 -12
- data/contrib/zstd/lib/common/entropy_common.c +15 -37
- data/contrib/zstd/lib/common/error_private.c +9 -2
- data/contrib/zstd/lib/common/error_private.h +93 -5
- data/contrib/zstd/lib/common/fse.h +12 -87
- data/contrib/zstd/lib/common/fse_decompress.c +37 -117
- data/contrib/zstd/lib/common/huf.h +97 -172
- data/contrib/zstd/lib/common/mem.h +58 -58
- data/contrib/zstd/lib/common/pool.c +38 -17
- data/contrib/zstd/lib/common/pool.h +10 -4
- data/contrib/zstd/lib/common/portability_macros.h +158 -0
- data/contrib/zstd/lib/common/threading.c +74 -14
- data/contrib/zstd/lib/common/threading.h +5 -10
- data/contrib/zstd/lib/common/xxhash.c +6 -814
- data/contrib/zstd/lib/common/xxhash.h +6930 -195
- data/contrib/zstd/lib/common/zstd_common.c +1 -36
- data/contrib/zstd/lib/common/zstd_deps.h +1 -1
- data/contrib/zstd/lib/common/zstd_internal.h +68 -154
- data/contrib/zstd/lib/common/zstd_trace.h +163 -0
- data/contrib/zstd/lib/compress/clevels.h +134 -0
- data/contrib/zstd/lib/compress/fse_compress.c +75 -155
- data/contrib/zstd/lib/compress/hist.c +1 -1
- data/contrib/zstd/lib/compress/hist.h +1 -1
- data/contrib/zstd/lib/compress/huf_compress.c +810 -259
- data/contrib/zstd/lib/compress/zstd_compress.c +2864 -919
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +523 -192
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +117 -40
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +16 -6
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +28 -19
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +1 -1
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +251 -412
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +1 -1
- data/contrib/zstd/lib/compress/zstd_cwksp.h +284 -97
- data/contrib/zstd/lib/compress/zstd_double_fast.c +382 -133
- data/contrib/zstd/lib/compress/zstd_double_fast.h +14 -2
- data/contrib/zstd/lib/compress/zstd_fast.c +732 -260
- data/contrib/zstd/lib/compress/zstd_fast.h +3 -2
- data/contrib/zstd/lib/compress/zstd_lazy.c +1177 -390
- data/contrib/zstd/lib/compress/zstd_lazy.h +129 -14
- data/contrib/zstd/lib/compress/zstd_ldm.c +280 -210
- data/contrib/zstd/lib/compress/zstd_ldm.h +3 -2
- data/contrib/zstd/lib/compress/zstd_ldm_geartab.h +106 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +516 -285
- data/contrib/zstd/lib/compress/zstd_opt.h +32 -8
- data/contrib/zstd/lib/compress/zstdmt_compress.c +202 -131
- data/contrib/zstd/lib/compress/zstdmt_compress.h +9 -6
- data/contrib/zstd/lib/decompress/huf_decompress.c +1149 -555
- data/contrib/zstd/lib/decompress/huf_decompress_amd64.S +595 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +4 -4
- data/contrib/zstd/lib/decompress/zstd_ddict.h +1 -1
- data/contrib/zstd/lib/decompress/zstd_decompress.c +583 -106
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1054 -379
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +14 -3
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +56 -6
- data/contrib/zstd/lib/deprecated/zbuff.h +1 -1
- data/contrib/zstd/lib/deprecated/zbuff_common.c +1 -1
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +24 -4
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +3 -1
- data/contrib/zstd/lib/dictBuilder/cover.c +60 -44
- data/contrib/zstd/lib/dictBuilder/cover.h +6 -11
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +1 -1
- data/contrib/zstd/lib/dictBuilder/fastcover.c +26 -18
- data/contrib/zstd/lib/dictBuilder/zdict.c +100 -101
- data/contrib/zstd/lib/legacy/zstd_legacy.h +38 -1
- data/contrib/zstd/lib/legacy/zstd_v01.c +18 -53
- data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v02.c +28 -85
- data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v03.c +29 -88
- data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v04.c +27 -80
- data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v05.c +36 -85
- data/contrib/zstd/lib/legacy/zstd_v05.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v06.c +44 -96
- data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v07.c +37 -92
- data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
- data/contrib/zstd/lib/libzstd.mk +237 -0
- data/contrib/zstd/lib/libzstd.pc.in +4 -3
- data/contrib/zstd/lib/module.modulemap +35 -0
- data/contrib/zstd/lib/{dictBuilder/zdict.h → zdict.h} +202 -33
- data/contrib/zstd/lib/zstd.h +1030 -332
- data/contrib/zstd/lib/{common/zstd_errors.h → zstd_errors.h} +27 -8
- data/ext/extconf.rb +26 -7
- data/ext/extzstd.c +51 -24
- data/ext/extzstd.h +33 -6
- data/ext/extzstd_stream.c +74 -31
- data/ext/libzstd_conf.h +0 -1
- data/ext/zstd_decompress_asm.S +1 -0
- metadata +17 -7
- data/contrib/zstd/appveyor.yml +0 -292
- data/ext/depend +0 -2
data/contrib/zstd/lib/zstd.h
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c)
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -14,27 +14,61 @@ extern "C" {
|
|
|
14
14
|
#ifndef ZSTD_H_235446
|
|
15
15
|
#define ZSTD_H_235446
|
|
16
16
|
|
|
17
|
-
/* ======
|
|
17
|
+
/* ====== Dependencies ======*/
|
|
18
18
|
#include <limits.h> /* INT_MAX */
|
|
19
19
|
#include <stddef.h> /* size_t */
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
/* ===== ZSTDLIB_API : control library symbols visibility ===== */
|
|
23
|
-
#ifndef
|
|
24
|
-
|
|
25
|
-
#
|
|
23
|
+
#ifndef ZSTDLIB_VISIBLE
|
|
24
|
+
/* Backwards compatibility with old macro name */
|
|
25
|
+
# ifdef ZSTDLIB_VISIBILITY
|
|
26
|
+
# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
|
|
27
|
+
# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
|
|
28
|
+
# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
|
|
26
29
|
# else
|
|
27
|
-
# define
|
|
30
|
+
# define ZSTDLIB_VISIBLE
|
|
28
31
|
# endif
|
|
29
32
|
#endif
|
|
33
|
+
|
|
34
|
+
#ifndef ZSTDLIB_HIDDEN
|
|
35
|
+
# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
|
|
36
|
+
# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
|
|
37
|
+
# else
|
|
38
|
+
# define ZSTDLIB_HIDDEN
|
|
39
|
+
# endif
|
|
40
|
+
#endif
|
|
41
|
+
|
|
30
42
|
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
|
31
|
-
# define ZSTDLIB_API __declspec(dllexport)
|
|
43
|
+
# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
|
|
32
44
|
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
|
33
|
-
# define ZSTDLIB_API __declspec(dllimport)
|
|
45
|
+
# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
|
34
46
|
#else
|
|
35
|
-
# define ZSTDLIB_API
|
|
47
|
+
# define ZSTDLIB_API ZSTDLIB_VISIBLE
|
|
36
48
|
#endif
|
|
37
49
|
|
|
50
|
+
/* Deprecation warnings :
|
|
51
|
+
* Should these warnings be a problem, it is generally possible to disable them,
|
|
52
|
+
* typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
|
|
53
|
+
* Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
|
|
54
|
+
*/
|
|
55
|
+
#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
|
|
56
|
+
# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */
|
|
57
|
+
#else
|
|
58
|
+
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
|
59
|
+
# define ZSTD_DEPRECATED(message) [[deprecated(message)]]
|
|
60
|
+
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
|
|
61
|
+
# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
|
|
62
|
+
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
|
63
|
+
# define ZSTD_DEPRECATED(message) __attribute__((deprecated))
|
|
64
|
+
# elif defined(_MSC_VER)
|
|
65
|
+
# define ZSTD_DEPRECATED(message) __declspec(deprecated(message))
|
|
66
|
+
# else
|
|
67
|
+
# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
|
|
68
|
+
# define ZSTD_DEPRECATED(message)
|
|
69
|
+
# endif
|
|
70
|
+
#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
|
|
71
|
+
|
|
38
72
|
|
|
39
73
|
/*******************************************************************************
|
|
40
74
|
Introduction
|
|
@@ -71,8 +105,8 @@ extern "C" {
|
|
|
71
105
|
|
|
72
106
|
/*------ Version ------*/
|
|
73
107
|
#define ZSTD_VERSION_MAJOR 1
|
|
74
|
-
#define ZSTD_VERSION_MINOR
|
|
75
|
-
#define ZSTD_VERSION_RELEASE
|
|
108
|
+
#define ZSTD_VERSION_MINOR 5
|
|
109
|
+
#define ZSTD_VERSION_RELEASE 6
|
|
76
110
|
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
|
77
111
|
|
|
78
112
|
/*! ZSTD_versionNumber() :
|
|
@@ -109,13 +143,13 @@ ZSTDLIB_API const char* ZSTD_versionString(void);
|
|
|
109
143
|
#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
|
|
110
144
|
|
|
111
145
|
|
|
112
|
-
|
|
113
146
|
/***************************************
|
|
114
147
|
* Simple API
|
|
115
148
|
***************************************/
|
|
116
149
|
/*! ZSTD_compress() :
|
|
117
150
|
* Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
|
|
118
|
-
*
|
|
151
|
+
* NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
|
|
152
|
+
* enough space to successfully compress the data.
|
|
119
153
|
* @return : compressed size written into `dst` (<= `dstCapacity),
|
|
120
154
|
* or an error code if it fails (which can be tested using ZSTD_isError()). */
|
|
121
155
|
ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
|
|
@@ -164,9 +198,11 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
|
|
|
164
198
|
* "empty", "unknown" and "error" results to the same return value (0),
|
|
165
199
|
* while ZSTD_getFrameContentSize() gives them separate return values.
|
|
166
200
|
* @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
|
|
167
|
-
|
|
201
|
+
ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
|
|
202
|
+
ZSTDLIB_API
|
|
203
|
+
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
|
|
168
204
|
|
|
169
|
-
/*! ZSTD_findFrameCompressedSize() :
|
|
205
|
+
/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
|
|
170
206
|
* `src` should point to the start of a ZSTD frame or skippable frame.
|
|
171
207
|
* `srcSize` must be >= first frame size
|
|
172
208
|
* @return : the compressed size of the first frame starting at `src`,
|
|
@@ -176,12 +212,35 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
|
|
|
176
212
|
|
|
177
213
|
|
|
178
214
|
/*====== Helper functions ======*/
|
|
179
|
-
|
|
180
|
-
|
|
215
|
+
/* ZSTD_compressBound() :
|
|
216
|
+
* maximum compressed size in worst case single-pass scenario.
|
|
217
|
+
* When invoking `ZSTD_compress()` or any other one-pass compression function,
|
|
218
|
+
* it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
|
|
219
|
+
* as it eliminates one potential failure scenario,
|
|
220
|
+
* aka not enough room in dst buffer to write the compressed frame.
|
|
221
|
+
* Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
|
|
222
|
+
* In which case, ZSTD_compressBound() will return an error code
|
|
223
|
+
* which can be tested using ZSTD_isError().
|
|
224
|
+
*
|
|
225
|
+
* ZSTD_COMPRESSBOUND() :
|
|
226
|
+
* same as ZSTD_compressBound(), but as a macro.
|
|
227
|
+
* It can be used to produce constants, which can be useful for static allocation,
|
|
228
|
+
* for example to size a static array on stack.
|
|
229
|
+
* Will produce constant value 0 if srcSize too large.
|
|
230
|
+
*/
|
|
231
|
+
#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U)
|
|
232
|
+
#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
|
|
233
|
+
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
|
|
234
|
+
/* ZSTD_isError() :
|
|
235
|
+
* Most ZSTD_* functions returning a size_t value can be tested for error,
|
|
236
|
+
* using ZSTD_isError().
|
|
237
|
+
* @return 1 if error, 0 otherwise
|
|
238
|
+
*/
|
|
181
239
|
ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
|
182
240
|
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
|
|
183
|
-
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */
|
|
241
|
+
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
|
|
184
242
|
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
|
|
243
|
+
ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
|
|
185
244
|
|
|
186
245
|
|
|
187
246
|
/***************************************
|
|
@@ -190,7 +249,7 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres
|
|
|
190
249
|
/*= Compression context
|
|
191
250
|
* When compressing many times,
|
|
192
251
|
* it is recommended to allocate a context just once,
|
|
193
|
-
* and
|
|
252
|
+
* and reuse it for each successive compression operation.
|
|
194
253
|
* This will make workload friendlier for system's memory.
|
|
195
254
|
* Note : re-using context is just a speed / resource optimization.
|
|
196
255
|
* It doesn't change the compression ratio, which remains identical.
|
|
@@ -199,13 +258,13 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres
|
|
|
199
258
|
*/
|
|
200
259
|
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
|
|
201
260
|
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
|
|
202
|
-
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
|
|
261
|
+
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */
|
|
203
262
|
|
|
204
263
|
/*! ZSTD_compressCCtx() :
|
|
205
264
|
* Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
|
|
206
|
-
* Important : in order to
|
|
207
|
-
* this function compresses at requested compression level,
|
|
208
|
-
* __ignoring any other parameter__ .
|
|
265
|
+
* Important : in order to mirror `ZSTD_compress()` behavior,
|
|
266
|
+
* this function compresses at the requested compression level,
|
|
267
|
+
* __ignoring any other advanced parameter__ .
|
|
209
268
|
* If any advanced parameter was set using the advanced API,
|
|
210
269
|
* they will all be reset. Only `compressionLevel` remains.
|
|
211
270
|
*/
|
|
@@ -217,38 +276,38 @@ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
|
|
|
217
276
|
/*= Decompression context
|
|
218
277
|
* When decompressing many times,
|
|
219
278
|
* it is recommended to allocate a context only once,
|
|
220
|
-
* and
|
|
279
|
+
* and reuse it for each successive compression operation.
|
|
221
280
|
* This will make workload friendlier for system's memory.
|
|
222
281
|
* Use one context per thread for parallel execution. */
|
|
223
282
|
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
|
224
283
|
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
|
|
225
|
-
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
|
284
|
+
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */
|
|
226
285
|
|
|
227
286
|
/*! ZSTD_decompressDCtx() :
|
|
228
287
|
* Same as ZSTD_decompress(),
|
|
229
288
|
* requires an allocated ZSTD_DCtx.
|
|
230
|
-
* Compatible with sticky parameters.
|
|
289
|
+
* Compatible with sticky parameters (see below).
|
|
231
290
|
*/
|
|
232
291
|
ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
|
|
233
292
|
void* dst, size_t dstCapacity,
|
|
234
293
|
const void* src, size_t srcSize);
|
|
235
294
|
|
|
236
295
|
|
|
237
|
-
|
|
238
|
-
* Advanced compression API
|
|
239
|
-
|
|
296
|
+
/*********************************************
|
|
297
|
+
* Advanced compression API (Requires v1.4.0+)
|
|
298
|
+
**********************************************/
|
|
240
299
|
|
|
241
300
|
/* API design :
|
|
242
301
|
* Parameters are pushed one by one into an existing context,
|
|
243
302
|
* using ZSTD_CCtx_set*() functions.
|
|
244
303
|
* Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
|
|
245
304
|
* "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
|
|
246
|
-
* __They do not apply to
|
|
305
|
+
* __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ .
|
|
247
306
|
*
|
|
248
307
|
* It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
|
|
249
308
|
*
|
|
250
|
-
* This API
|
|
251
|
-
* In the future, we expect to remove
|
|
309
|
+
* This API supersedes all other "advanced" API entry points in the experimental section.
|
|
310
|
+
* In the future, we expect to remove API entry points from experimental which are redundant with this API.
|
|
252
311
|
*/
|
|
253
312
|
|
|
254
313
|
|
|
@@ -266,7 +325,6 @@ typedef enum { ZSTD_fast=1,
|
|
|
266
325
|
Only the order (from fast to strong) is guaranteed */
|
|
267
326
|
} ZSTD_strategy;
|
|
268
327
|
|
|
269
|
-
|
|
270
328
|
typedef enum {
|
|
271
329
|
|
|
272
330
|
/* compression parameters
|
|
@@ -333,6 +391,18 @@ typedef enum {
|
|
|
333
391
|
* resulting in stronger and slower compression.
|
|
334
392
|
* Special: value 0 means "use default strategy". */
|
|
335
393
|
|
|
394
|
+
ZSTD_c_targetCBlockSize=130, /* v1.5.6+
|
|
395
|
+
* Attempts to fit compressed block size into approximatively targetCBlockSize.
|
|
396
|
+
* Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
|
|
397
|
+
* Note that it's not a guarantee, just a convergence target (default:0).
|
|
398
|
+
* No target when targetCBlockSize == 0.
|
|
399
|
+
* This is helpful in low bandwidth streaming environments to improve end-to-end latency,
|
|
400
|
+
* when a client can make use of partial documents (a prominent example being Chrome).
|
|
401
|
+
* Note: this parameter is stable since v1.5.6.
|
|
402
|
+
* It was present as an experimental parameter in earlier versions,
|
|
403
|
+
* but it's not recommended using it with earlier library versions
|
|
404
|
+
* due to massive performance regressions.
|
|
405
|
+
*/
|
|
336
406
|
/* LDM mode parameters */
|
|
337
407
|
ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
|
|
338
408
|
* This parameter is designed to improve compression ratio
|
|
@@ -389,7 +459,7 @@ typedef enum {
|
|
|
389
459
|
ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
|
|
390
460
|
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
|
|
391
461
|
* 0 means default, which is dynamically determined based on compression parameters.
|
|
392
|
-
* Job size must be a minimum of overlap size, or
|
|
462
|
+
* Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
|
|
393
463
|
* The minimum size is automatically and transparently enforced. */
|
|
394
464
|
ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
|
|
395
465
|
* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
|
|
@@ -412,13 +482,17 @@ typedef enum {
|
|
|
412
482
|
* ZSTD_c_forceMaxWindow
|
|
413
483
|
* ZSTD_c_forceAttachDict
|
|
414
484
|
* ZSTD_c_literalCompressionMode
|
|
415
|
-
* ZSTD_c_targetCBlockSize
|
|
416
485
|
* ZSTD_c_srcSizeHint
|
|
417
486
|
* ZSTD_c_enableDedicatedDictSearch
|
|
418
487
|
* ZSTD_c_stableInBuffer
|
|
419
488
|
* ZSTD_c_stableOutBuffer
|
|
420
489
|
* ZSTD_c_blockDelimiters
|
|
421
490
|
* ZSTD_c_validateSequences
|
|
491
|
+
* ZSTD_c_useBlockSplitter
|
|
492
|
+
* ZSTD_c_useRowMatchFinder
|
|
493
|
+
* ZSTD_c_prefetchCDictTables
|
|
494
|
+
* ZSTD_c_enableSeqProducerFallback
|
|
495
|
+
* ZSTD_c_maxBlockSize
|
|
422
496
|
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
|
|
423
497
|
* note : never ever use experimentalParam? names directly;
|
|
424
498
|
* also, the enums values themselves are unstable and can still change.
|
|
@@ -428,13 +502,20 @@ typedef enum {
|
|
|
428
502
|
ZSTD_c_experimentalParam3=1000,
|
|
429
503
|
ZSTD_c_experimentalParam4=1001,
|
|
430
504
|
ZSTD_c_experimentalParam5=1002,
|
|
431
|
-
ZSTD_c_experimentalParam6=1003
|
|
505
|
+
/* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */
|
|
432
506
|
ZSTD_c_experimentalParam7=1004,
|
|
433
507
|
ZSTD_c_experimentalParam8=1005,
|
|
434
508
|
ZSTD_c_experimentalParam9=1006,
|
|
435
509
|
ZSTD_c_experimentalParam10=1007,
|
|
436
510
|
ZSTD_c_experimentalParam11=1008,
|
|
437
|
-
ZSTD_c_experimentalParam12=1009
|
|
511
|
+
ZSTD_c_experimentalParam12=1009,
|
|
512
|
+
ZSTD_c_experimentalParam13=1010,
|
|
513
|
+
ZSTD_c_experimentalParam14=1011,
|
|
514
|
+
ZSTD_c_experimentalParam15=1012,
|
|
515
|
+
ZSTD_c_experimentalParam16=1013,
|
|
516
|
+
ZSTD_c_experimentalParam17=1014,
|
|
517
|
+
ZSTD_c_experimentalParam18=1015,
|
|
518
|
+
ZSTD_c_experimentalParam19=1016
|
|
438
519
|
} ZSTD_cParameter;
|
|
439
520
|
|
|
440
521
|
typedef struct {
|
|
@@ -497,7 +578,7 @@ typedef enum {
|
|
|
497
578
|
* They will be used to compress next frame.
|
|
498
579
|
* Resetting session never fails.
|
|
499
580
|
* - The parameters : changes all parameters back to "default".
|
|
500
|
-
* This removes any reference to any dictionary
|
|
581
|
+
* This also removes any reference to any dictionary or external sequence producer.
|
|
501
582
|
* Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
|
|
502
583
|
* otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
|
|
503
584
|
* - Both : similar to resetting the session, followed by resetting parameters.
|
|
@@ -506,11 +587,13 @@ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
|
|
|
506
587
|
|
|
507
588
|
/*! ZSTD_compress2() :
|
|
508
589
|
* Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
|
|
590
|
+
* (note that this entry point doesn't even expose a compression level parameter).
|
|
509
591
|
* ZSTD_compress2() always starts a new frame.
|
|
510
592
|
* Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
|
|
511
593
|
* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
|
|
512
594
|
* - The function is always blocking, returns when compression is completed.
|
|
513
|
-
*
|
|
595
|
+
* NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
|
|
596
|
+
* enough space to successfully compress the data, though it is possible it fails for other reasons.
|
|
514
597
|
* @return : compressed size written into `dst` (<= `dstCapacity),
|
|
515
598
|
* or an error code if it fails (which can be tested using ZSTD_isError()).
|
|
516
599
|
*/
|
|
@@ -519,9 +602,9 @@ ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
|
|
|
519
602
|
const void* src, size_t srcSize);
|
|
520
603
|
|
|
521
604
|
|
|
522
|
-
|
|
523
|
-
* Advanced decompression API
|
|
524
|
-
|
|
605
|
+
/***********************************************
|
|
606
|
+
* Advanced decompression API (Requires v1.4.0+)
|
|
607
|
+
************************************************/
|
|
525
608
|
|
|
526
609
|
/* The advanced API pushes parameters one by one into an existing DCtx context.
|
|
527
610
|
* Parameters are sticky, and remain valid for all following frames
|
|
@@ -546,12 +629,18 @@ typedef enum {
|
|
|
546
629
|
* ZSTD_d_format
|
|
547
630
|
* ZSTD_d_stableOutBuffer
|
|
548
631
|
* ZSTD_d_forceIgnoreChecksum
|
|
632
|
+
* ZSTD_d_refMultipleDDicts
|
|
633
|
+
* ZSTD_d_disableHuffmanAssembly
|
|
634
|
+
* ZSTD_d_maxBlockSize
|
|
549
635
|
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
|
|
550
636
|
* note : never ever use experimentalParam? names directly
|
|
551
637
|
*/
|
|
552
638
|
ZSTD_d_experimentalParam1=1000,
|
|
553
639
|
ZSTD_d_experimentalParam2=1001,
|
|
554
|
-
ZSTD_d_experimentalParam3=1002
|
|
640
|
+
ZSTD_d_experimentalParam3=1002,
|
|
641
|
+
ZSTD_d_experimentalParam4=1003,
|
|
642
|
+
ZSTD_d_experimentalParam5=1004,
|
|
643
|
+
ZSTD_d_experimentalParam6=1005
|
|
555
644
|
|
|
556
645
|
} ZSTD_dParameter;
|
|
557
646
|
|
|
@@ -606,14 +695,14 @@ typedef struct ZSTD_outBuffer_s {
|
|
|
606
695
|
* A ZSTD_CStream object is required to track streaming operation.
|
|
607
696
|
* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
|
|
608
697
|
* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
|
|
609
|
-
* It is recommended to
|
|
698
|
+
* It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
|
|
610
699
|
*
|
|
611
700
|
* For parallel execution, use one separate ZSTD_CStream per thread.
|
|
612
701
|
*
|
|
613
702
|
* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
|
|
614
703
|
*
|
|
615
704
|
* Parameters are sticky : when starting a new compression on the same context,
|
|
616
|
-
* it will
|
|
705
|
+
* it will reuse the same sticky parameters as previous compression session.
|
|
617
706
|
* When in doubt, it's recommended to fully initialize the context before usage.
|
|
618
707
|
* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
|
|
619
708
|
* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
|
|
@@ -665,7 +754,7 @@ typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same
|
|
|
665
754
|
/* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
|
|
666
755
|
/*===== ZSTD_CStream management functions =====*/
|
|
667
756
|
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
|
|
668
|
-
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
|
|
757
|
+
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */
|
|
669
758
|
|
|
670
759
|
/*===== Streaming compression functions =====*/
|
|
671
760
|
typedef enum {
|
|
@@ -681,7 +770,7 @@ typedef enum {
|
|
|
681
770
|
: note : multithreaded compression will block to flush as much output as possible. */
|
|
682
771
|
} ZSTD_EndDirective;
|
|
683
772
|
|
|
684
|
-
/*! ZSTD_compressStream2() :
|
|
773
|
+
/*! ZSTD_compressStream2() : Requires v1.4.0+
|
|
685
774
|
* Behaves about the same as ZSTD_compressStream, with additional control on end directive.
|
|
686
775
|
* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
|
|
687
776
|
* - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
|
|
@@ -702,6 +791,11 @@ typedef enum {
|
|
|
702
791
|
* only ZSTD_e_end or ZSTD_e_flush operations are allowed.
|
|
703
792
|
* Before starting a new compression job, or changing compression parameters,
|
|
704
793
|
* it is required to fully flush internal buffers.
|
|
794
|
+
* - note: if an operation ends with an error, it may leave @cctx in an undefined state.
|
|
795
|
+
* Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state.
|
|
796
|
+
* In order to be re-employed after an error, a state must be reset,
|
|
797
|
+
* which can be done explicitly (ZSTD_CCtx_reset()),
|
|
798
|
+
* or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx())
|
|
705
799
|
*/
|
|
706
800
|
ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
|
|
707
801
|
ZSTD_outBuffer* output,
|
|
@@ -727,11 +821,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
|
|
|
727
821
|
|
|
728
822
|
|
|
729
823
|
/* *****************************************************************************
|
|
730
|
-
* This following is a legacy streaming API.
|
|
824
|
+
* This following is a legacy streaming API, available since v1.0+ .
|
|
731
825
|
* It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
|
|
732
826
|
* It is redundant, but remains fully supported.
|
|
733
|
-
* Advanced parameters and dictionary compression can only be used through the
|
|
734
|
-
* new API.
|
|
735
827
|
******************************************************************************/
|
|
736
828
|
|
|
737
829
|
/*!
|
|
@@ -740,6 +832,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
|
|
|
740
832
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
741
833
|
* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
|
|
742
834
|
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
|
835
|
+
*
|
|
836
|
+
* Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
|
|
837
|
+
* to compress with a dictionary.
|
|
743
838
|
*/
|
|
744
839
|
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
|
|
745
840
|
/*!
|
|
@@ -760,7 +855,7 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
|
|
|
760
855
|
*
|
|
761
856
|
* A ZSTD_DStream object is required to track streaming operations.
|
|
762
857
|
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
|
|
763
|
-
* ZSTD_DStream objects can be
|
|
858
|
+
* ZSTD_DStream objects can be reused multiple times.
|
|
764
859
|
*
|
|
765
860
|
* Use ZSTD_initDStream() to start a new decompression operation.
|
|
766
861
|
* @return : recommended first input size
|
|
@@ -786,17 +881,41 @@ typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same
|
|
|
786
881
|
/* For compatibility with versions <= v1.2.0, prefer differentiating them. */
|
|
787
882
|
/*===== ZSTD_DStream management functions =====*/
|
|
788
883
|
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
|
|
789
|
-
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
|
884
|
+
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */
|
|
790
885
|
|
|
791
886
|
/*===== Streaming decompression functions =====*/
|
|
792
887
|
|
|
793
|
-
|
|
888
|
+
/*! ZSTD_initDStream() :
|
|
889
|
+
* Initialize/reset DStream state for new decompression operation.
|
|
890
|
+
* Call before new decompression operation using same DStream.
|
|
794
891
|
*
|
|
892
|
+
* Note : This function is redundant with the advanced API and equivalent to:
|
|
795
893
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
|
796
894
|
* ZSTD_DCtx_refDDict(zds, NULL);
|
|
797
895
|
*/
|
|
798
896
|
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
|
799
897
|
|
|
898
|
+
/*! ZSTD_decompressStream() :
|
|
899
|
+
* Streaming decompression function.
|
|
900
|
+
* Call repetitively to consume full input updating it as necessary.
|
|
901
|
+
* Function will update both input and output `pos` fields exposing current state via these fields:
|
|
902
|
+
* - `input.pos < input.size`, some input remaining and caller should provide remaining input
|
|
903
|
+
* on the next call.
|
|
904
|
+
* - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
|
|
905
|
+
* - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
|
|
906
|
+
* call ZSTD_decompressStream() again to flush remaining data to output.
|
|
907
|
+
* Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
|
|
908
|
+
*
|
|
909
|
+
* @return : 0 when a frame is completely decoded and fully flushed,
|
|
910
|
+
* or an error code, which can be tested using ZSTD_isError(),
|
|
911
|
+
* or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
|
|
912
|
+
*
|
|
913
|
+
* Note: when an operation returns with an error code, the @zds state may be left in undefined state.
|
|
914
|
+
* It's UB to invoke `ZSTD_decompressStream()` on such a state.
|
|
915
|
+
* In order to re-use such a state, it must be first reset,
|
|
916
|
+
* which can be done explicitly (`ZSTD_DCtx_reset()`),
|
|
917
|
+
* or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`)
|
|
918
|
+
*/
|
|
800
919
|
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
|
801
920
|
|
|
802
921
|
ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
|
|
@@ -809,7 +928,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
|
|
|
809
928
|
/*! ZSTD_compress_usingDict() :
|
|
810
929
|
* Compression at an explicit compression level using a Dictionary.
|
|
811
930
|
* A dictionary can be any arbitrary data segment (also called a prefix),
|
|
812
|
-
* or a buffer with specified information (see
|
|
931
|
+
* or a buffer with specified information (see zdict.h).
|
|
813
932
|
* Note : This function loads the dictionary, resulting in significant startup delay.
|
|
814
933
|
* It's intended for a dictionary used only once.
|
|
815
934
|
* Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
|
|
@@ -852,7 +971,8 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize
|
|
|
852
971
|
int compressionLevel);
|
|
853
972
|
|
|
854
973
|
/*! ZSTD_freeCDict() :
|
|
855
|
-
* Function frees memory allocated by ZSTD_createCDict().
|
|
974
|
+
* Function frees memory allocated by ZSTD_createCDict().
|
|
975
|
+
* If a NULL pointer is passed, no operation is performed. */
|
|
856
976
|
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
|
|
857
977
|
|
|
858
978
|
/*! ZSTD_compress_usingCDict() :
|
|
@@ -874,7 +994,8 @@ typedef struct ZSTD_DDict_s ZSTD_DDict;
|
|
|
874
994
|
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
|
|
875
995
|
|
|
876
996
|
/*! ZSTD_freeDDict() :
|
|
877
|
-
* Function frees memory allocated with ZSTD_createDDict()
|
|
997
|
+
* Function frees memory allocated with ZSTD_createDDict()
|
|
998
|
+
* If a NULL pointer is passed, no operation is performed. */
|
|
878
999
|
ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
|
|
879
1000
|
|
|
880
1001
|
/*! ZSTD_decompress_usingDDict() :
|
|
@@ -890,24 +1011,30 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
|
|
|
890
1011
|
* Dictionary helper functions
|
|
891
1012
|
*******************************/
|
|
892
1013
|
|
|
893
|
-
/*! ZSTD_getDictID_fromDict() :
|
|
1014
|
+
/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+
|
|
894
1015
|
* Provides the dictID stored within dictionary.
|
|
895
1016
|
* if @return == 0, the dictionary is not conformant with Zstandard specification.
|
|
896
1017
|
* It can still be loaded, but as a content-only dictionary. */
|
|
897
1018
|
ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
|
|
898
1019
|
|
|
899
|
-
/*!
|
|
1020
|
+
/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+
|
|
1021
|
+
* Provides the dictID of the dictionary loaded into `cdict`.
|
|
1022
|
+
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
|
|
1023
|
+
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
|
|
1024
|
+
ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
|
|
1025
|
+
|
|
1026
|
+
/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+
|
|
900
1027
|
* Provides the dictID of the dictionary loaded into `ddict`.
|
|
901
1028
|
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
|
|
902
1029
|
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
|
|
903
1030
|
ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
|
|
904
1031
|
|
|
905
|
-
/*! ZSTD_getDictID_fromFrame() :
|
|
1032
|
+
/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+
|
|
906
1033
|
* Provides the dictID required to decompressed the frame stored within `src`.
|
|
907
1034
|
* If @return == 0, the dictID could not be decoded.
|
|
908
1035
|
* This could for one of the following reasons :
|
|
909
1036
|
* - The frame does not require a dictionary to be decoded (most common case).
|
|
910
|
-
* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
|
|
1037
|
+
* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information.
|
|
911
1038
|
* Note : this use case also happens when using a non-conformant dictionary.
|
|
912
1039
|
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
|
|
913
1040
|
* - This is not a Zstandard frame.
|
|
@@ -916,23 +1043,26 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
|
|
916
1043
|
|
|
917
1044
|
|
|
918
1045
|
/*******************************************************************************
|
|
919
|
-
* Advanced dictionary and prefix API
|
|
1046
|
+
* Advanced dictionary and prefix API (Requires v1.4.0+)
|
|
920
1047
|
*
|
|
921
1048
|
* This API allows dictionaries to be used with ZSTD_compress2(),
|
|
922
|
-
* ZSTD_compressStream2(), and
|
|
923
|
-
*
|
|
924
|
-
*
|
|
1049
|
+
* ZSTD_compressStream2(), and ZSTD_decompressDCtx().
|
|
1050
|
+
* Dictionaries are sticky, they remain valid when same context is reused,
|
|
1051
|
+
* they only reset when the context is reset
|
|
1052
|
+
* with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters.
|
|
1053
|
+
* In contrast, Prefixes are single-use.
|
|
925
1054
|
******************************************************************************/
|
|
926
1055
|
|
|
927
1056
|
|
|
928
|
-
/*! ZSTD_CCtx_loadDictionary() :
|
|
1057
|
+
/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+
|
|
929
1058
|
* Create an internal CDict from `dict` buffer.
|
|
930
1059
|
* Decompression will have to use same dictionary.
|
|
931
1060
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
932
1061
|
* Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
|
|
933
1062
|
* meaning "return to no-dictionary mode".
|
|
934
|
-
* Note 1 : Dictionary is sticky, it will be used for all future compressed frames
|
|
935
|
-
*
|
|
1063
|
+
* Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
|
|
1064
|
+
* until parameters are reset, a new dictionary is loaded, or the dictionary
|
|
1065
|
+
* is explicitly invalidated by loading a NULL dictionary.
|
|
936
1066
|
* Note 2 : Loading a dictionary involves building tables.
|
|
937
1067
|
* It's also a CPU consuming operation, with non-negligible impact on latency.
|
|
938
1068
|
* Tables are dependent on compression parameters, and for this reason,
|
|
@@ -941,14 +1071,18 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
|
|
941
1071
|
* Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
|
|
942
1072
|
* In such a case, dictionary buffer must outlive its users.
|
|
943
1073
|
* Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
|
|
944
|
-
* to precisely select how dictionary content must be interpreted.
|
|
1074
|
+
* to precisely select how dictionary content must be interpreted.
|
|
1075
|
+
* Note 5 : This method does not benefit from LDM (long distance mode).
|
|
1076
|
+
* If you want to employ LDM on some large dictionary content,
|
|
1077
|
+
* prefer employing ZSTD_CCtx_refPrefix() described below.
|
|
1078
|
+
*/
|
|
945
1079
|
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
|
|
946
1080
|
|
|
947
|
-
/*! ZSTD_CCtx_refCDict() :
|
|
948
|
-
* Reference a prepared dictionary, to be used for all
|
|
1081
|
+
/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
|
|
1082
|
+
* Reference a prepared dictionary, to be used for all future compressed frames.
|
|
949
1083
|
* Note that compression parameters are enforced from within CDict,
|
|
950
1084
|
* and supersede any compression parameter previously set within CCtx.
|
|
951
|
-
* The parameters ignored are
|
|
1085
|
+
* The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
|
|
952
1086
|
* The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
|
|
953
1087
|
* The dictionary will remain valid for future compressed frames using same CCtx.
|
|
954
1088
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
@@ -958,12 +1092,13 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
|
|
|
958
1092
|
* Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
|
|
959
1093
|
ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
|
|
960
1094
|
|
|
961
|
-
/*! ZSTD_CCtx_refPrefix() :
|
|
1095
|
+
/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+
|
|
962
1096
|
* Reference a prefix (single-usage dictionary) for next compressed frame.
|
|
963
1097
|
* A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
|
|
964
1098
|
* Decompression will need same prefix to properly regenerate data.
|
|
965
1099
|
* Compressing with a prefix is similar in outcome as performing a diff and compressing it,
|
|
966
1100
|
* but performs much faster, especially during decompression (compression speed is tunable with compression level).
|
|
1101
|
+
* This method is compatible with LDM (long distance mode).
|
|
967
1102
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
968
1103
|
* Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
|
|
969
1104
|
* Note 1 : Prefix buffer is referenced. It **must** outlive compression.
|
|
@@ -979,10 +1114,10 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
|
|
|
979
1114
|
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
|
|
980
1115
|
const void* prefix, size_t prefixSize);
|
|
981
1116
|
|
|
982
|
-
/*! ZSTD_DCtx_loadDictionary() :
|
|
983
|
-
* Create an internal DDict from dict buffer,
|
|
984
|
-
*
|
|
985
|
-
*
|
|
1117
|
+
/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
|
|
1118
|
+
* Create an internal DDict from dict buffer, to be used to decompress all future frames.
|
|
1119
|
+
* The dictionary remains valid for all future frames, until explicitly invalidated, or
|
|
1120
|
+
* a new dictionary is loaded.
|
|
986
1121
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
987
1122
|
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
|
|
988
1123
|
* meaning "return to no-dictionary mode".
|
|
@@ -996,18 +1131,26 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
|
|
|
996
1131
|
*/
|
|
997
1132
|
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
|
998
1133
|
|
|
999
|
-
/*! ZSTD_DCtx_refDDict() :
|
|
1134
|
+
/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+
|
|
1000
1135
|
* Reference a prepared dictionary, to be used to decompress next frames.
|
|
1001
1136
|
* The dictionary remains active for decompression of future frames using same DCtx.
|
|
1137
|
+
*
|
|
1138
|
+
* If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
|
|
1139
|
+
* will store the DDict references in a table, and the DDict used for decompression
|
|
1140
|
+
* will be determined at decompression time, as per the dict ID in the frame.
|
|
1141
|
+
* The memory for the table is allocated on the first call to refDDict, and can be
|
|
1142
|
+
* freed with ZSTD_freeDCtx().
|
|
1143
|
+
*
|
|
1144
|
+
* If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
|
|
1145
|
+
* will be managed, and referencing a dictionary effectively "discards" any previous one.
|
|
1146
|
+
*
|
|
1002
1147
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1003
|
-
* Note 1 : Currently, only one dictionary can be managed.
|
|
1004
|
-
* Referencing a new dictionary effectively "discards" any previous one.
|
|
1005
1148
|
* Special: referencing a NULL DDict means "return to no-dictionary mode".
|
|
1006
1149
|
* Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
|
|
1007
1150
|
*/
|
|
1008
1151
|
ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
|
1009
1152
|
|
|
1010
|
-
/*! ZSTD_DCtx_refPrefix() :
|
|
1153
|
+
/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+
|
|
1011
1154
|
* Reference a prefix (single-usage dictionary) to decompress next frame.
|
|
1012
1155
|
* This is the reverse operation of ZSTD_CCtx_refPrefix(),
|
|
1013
1156
|
* and must use the same prefix as the one used during compression.
|
|
@@ -1028,7 +1171,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
|
|
|
1028
1171
|
|
|
1029
1172
|
/* === Memory management === */
|
|
1030
1173
|
|
|
1031
|
-
/*! ZSTD_sizeof_*() :
|
|
1174
|
+
/*! ZSTD_sizeof_*() : Requires v1.4.0+
|
|
1032
1175
|
* These functions give the _current_ memory usage of selected object.
|
|
1033
1176
|
* Note that object memory usage can evolve (increase or decrease) over time. */
|
|
1034
1177
|
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
|
|
@@ -1053,6 +1196,17 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
|
1053
1196
|
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
|
|
1054
1197
|
#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
|
|
1055
1198
|
|
|
1199
|
+
/* This can be overridden externally to hide static symbols. */
|
|
1200
|
+
#ifndef ZSTDLIB_STATIC_API
|
|
1201
|
+
# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
|
1202
|
+
# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE
|
|
1203
|
+
# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
|
1204
|
+
# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE
|
|
1205
|
+
# else
|
|
1206
|
+
# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE
|
|
1207
|
+
# endif
|
|
1208
|
+
#endif
|
|
1209
|
+
|
|
1056
1210
|
/****************************************************************************************
|
|
1057
1211
|
* experimental API (static linking only)
|
|
1058
1212
|
****************************************************************************************
|
|
@@ -1087,6 +1241,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
|
1087
1241
|
#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
|
|
1088
1242
|
#define ZSTD_STRATEGY_MIN ZSTD_fast
|
|
1089
1243
|
#define ZSTD_STRATEGY_MAX ZSTD_btultra2
|
|
1244
|
+
#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
|
|
1090
1245
|
|
|
1091
1246
|
|
|
1092
1247
|
#define ZSTD_OVERLAPLOG_MIN 0
|
|
@@ -1110,14 +1265,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
|
|
1110
1265
|
#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
|
|
1111
1266
|
|
|
1112
1267
|
/* Advanced parameter bounds */
|
|
1113
|
-
#define ZSTD_TARGETCBLOCKSIZE_MIN
|
|
1268
|
+
#define ZSTD_TARGETCBLOCKSIZE_MIN 1340 /* suitable to fit into an ethernet / wifi / 4G transport frame */
|
|
1114
1269
|
#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
|
|
1115
1270
|
#define ZSTD_SRCSIZEHINT_MIN 0
|
|
1116
1271
|
#define ZSTD_SRCSIZEHINT_MAX INT_MAX
|
|
1117
1272
|
|
|
1118
|
-
/* internal */
|
|
1119
|
-
#define ZSTD_HASHLOG3_MAX 17
|
|
1120
|
-
|
|
1121
1273
|
|
|
1122
1274
|
/* --- Advanced types --- */
|
|
1123
1275
|
|
|
@@ -1205,6 +1357,12 @@ typedef enum {
|
|
|
1205
1357
|
ZSTD_d_ignoreChecksum = 1
|
|
1206
1358
|
} ZSTD_forceIgnoreChecksum_e;
|
|
1207
1359
|
|
|
1360
|
+
typedef enum {
|
|
1361
|
+
/* Note: this enum controls ZSTD_d_refMultipleDDicts */
|
|
1362
|
+
ZSTD_rmd_refSingleDDict = 0,
|
|
1363
|
+
ZSTD_rmd_refMultipleDDicts = 1
|
|
1364
|
+
} ZSTD_refMultipleDDicts_e;
|
|
1365
|
+
|
|
1208
1366
|
typedef enum {
|
|
1209
1367
|
/* Note: this enum and the behavior it controls are effectively internal
|
|
1210
1368
|
* implementation details of the compressor. They are expected to continue
|
|
@@ -1253,9 +1411,18 @@ typedef enum {
|
|
|
1253
1411
|
ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
|
|
1254
1412
|
} ZSTD_literalCompressionMode_e;
|
|
1255
1413
|
|
|
1414
|
+
typedef enum {
|
|
1415
|
+
/* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
|
|
1416
|
+
* decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
|
|
1417
|
+
* or ZSTD_ps_disable allow for a force enable/disable the feature.
|
|
1418
|
+
*/
|
|
1419
|
+
ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */
|
|
1420
|
+
ZSTD_ps_enable = 1, /* Force-enable the feature */
|
|
1421
|
+
ZSTD_ps_disable = 2 /* Do not use the feature */
|
|
1422
|
+
} ZSTD_paramSwitch_e;
|
|
1256
1423
|
|
|
1257
1424
|
/***************************************
|
|
1258
|
-
* Frame size functions
|
|
1425
|
+
* Frame header and size functions
|
|
1259
1426
|
***************************************/
|
|
1260
1427
|
|
|
1261
1428
|
/*! ZSTD_findDecompressedSize() :
|
|
@@ -1279,14 +1446,14 @@ typedef enum {
|
|
|
1279
1446
|
* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
|
|
1280
1447
|
* read each contained frame header. This is fast as most of the data is skipped,
|
|
1281
1448
|
* however it does mean that all frame data must be present and valid. */
|
|
1282
|
-
|
|
1449
|
+
ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
|
|
1283
1450
|
|
|
1284
1451
|
/*! ZSTD_decompressBound() :
|
|
1285
1452
|
* `src` should point to the start of a series of ZSTD encoded and/or skippable frames
|
|
1286
1453
|
* `srcSize` must be the _exact_ size of this series
|
|
1287
1454
|
* (i.e. there should be a frame boundary at `src + srcSize`)
|
|
1288
1455
|
* @return : - upper-bound for the decompressed size of all data in all successive frames
|
|
1289
|
-
* - if an error
|
|
1456
|
+
* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
|
|
1290
1457
|
*
|
|
1291
1458
|
* note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
|
|
1292
1459
|
* note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
|
|
@@ -1294,41 +1461,134 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
|
|
|
1294
1461
|
* note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
|
|
1295
1462
|
* upper-bound = # blocks * min(128 KB, Window_Size)
|
|
1296
1463
|
*/
|
|
1297
|
-
|
|
1464
|
+
ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
|
|
1298
1465
|
|
|
1299
1466
|
/*! ZSTD_frameHeaderSize() :
|
|
1300
1467
|
* srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
|
|
1301
1468
|
* @return : size of the Frame Header,
|
|
1302
1469
|
* or an error code (if srcSize is too small) */
|
|
1303
|
-
|
|
1470
|
+
ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
|
|
1471
|
+
|
|
1472
|
+
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
|
|
1473
|
+
typedef struct {
|
|
1474
|
+
unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
|
|
1475
|
+
unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
|
|
1476
|
+
unsigned blockSizeMax;
|
|
1477
|
+
ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
|
|
1478
|
+
unsigned headerSize;
|
|
1479
|
+
unsigned dictID;
|
|
1480
|
+
unsigned checksumFlag;
|
|
1481
|
+
unsigned _reserved1;
|
|
1482
|
+
unsigned _reserved2;
|
|
1483
|
+
} ZSTD_frameHeader;
|
|
1484
|
+
|
|
1485
|
+
/*! ZSTD_getFrameHeader() :
|
|
1486
|
+
* decode Frame Header, or requires larger `srcSize`.
|
|
1487
|
+
* @return : 0, `zfhPtr` is correctly filled,
|
|
1488
|
+
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
|
1489
|
+
* or an error code, which can be tested using ZSTD_isError() */
|
|
1490
|
+
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
|
1491
|
+
/*! ZSTD_getFrameHeader_advanced() :
|
|
1492
|
+
* same as ZSTD_getFrameHeader(),
|
|
1493
|
+
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
|
|
1494
|
+
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
|
|
1495
|
+
|
|
1496
|
+
/*! ZSTD_decompressionMargin() :
|
|
1497
|
+
* Zstd supports in-place decompression, where the input and output buffers overlap.
|
|
1498
|
+
* In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
|
|
1499
|
+
* and the input buffer must be at the end of the output buffer.
|
|
1500
|
+
*
|
|
1501
|
+
* _______________________ Output Buffer ________________________
|
|
1502
|
+
* | |
|
|
1503
|
+
* | ____ Input Buffer ____|
|
|
1504
|
+
* | | |
|
|
1505
|
+
* v v v
|
|
1506
|
+
* |---------------------------------------|-----------|----------|
|
|
1507
|
+
* ^ ^ ^
|
|
1508
|
+
* |___________________ Output_Size ___________________|_ Margin _|
|
|
1509
|
+
*
|
|
1510
|
+
* NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
|
|
1511
|
+
* NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
|
|
1512
|
+
* ZSTD_decompressDCtx().
|
|
1513
|
+
* NOTE: This function supports multi-frame input.
|
|
1514
|
+
*
|
|
1515
|
+
* @param src The compressed frame(s)
|
|
1516
|
+
* @param srcSize The size of the compressed frame(s)
|
|
1517
|
+
* @returns The decompression margin or an error that can be checked with ZSTD_isError().
|
|
1518
|
+
*/
|
|
1519
|
+
ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
|
|
1520
|
+
|
|
1521
|
+
/*! ZSTD_DECOMPRESS_MARGIN() :
|
|
1522
|
+
* Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
|
|
1523
|
+
* the compressed frame, compute it from the original size and the blockSizeLog.
|
|
1524
|
+
* See ZSTD_decompressionMargin() for details.
|
|
1525
|
+
*
|
|
1526
|
+
* WARNING: This macro does not support multi-frame input, the input must be a single
|
|
1527
|
+
* zstd frame. If you need that support use the function, or implement it yourself.
|
|
1528
|
+
*
|
|
1529
|
+
* @param originalSize The original uncompressed size of the data.
|
|
1530
|
+
* @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
|
|
1531
|
+
* Unless you explicitly set the windowLog smaller than
|
|
1532
|
+
* ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
|
|
1533
|
+
*/
|
|
1534
|
+
#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \
|
|
1535
|
+
ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \
|
|
1536
|
+
4 /* checksum */ + \
|
|
1537
|
+
((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
|
|
1538
|
+
(blockSize) /* One block of margin */ \
|
|
1539
|
+
))
|
|
1304
1540
|
|
|
1305
1541
|
typedef enum {
|
|
1306
1542
|
ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
|
|
1307
1543
|
ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
|
|
1308
1544
|
} ZSTD_sequenceFormat_e;
|
|
1309
1545
|
|
|
1546
|
+
/*! ZSTD_sequenceBound() :
|
|
1547
|
+
* `srcSize` : size of the input buffer
|
|
1548
|
+
* @return : upper-bound for the number of sequences that can be generated
|
|
1549
|
+
* from a buffer of srcSize bytes
|
|
1550
|
+
*
|
|
1551
|
+
* note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
|
|
1552
|
+
*/
|
|
1553
|
+
ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
|
|
1554
|
+
|
|
1310
1555
|
/*! ZSTD_generateSequences() :
|
|
1311
|
-
*
|
|
1556
|
+
* WARNING: This function is meant for debugging and informational purposes ONLY!
|
|
1557
|
+
* Its implementation is flawed, and it will be deleted in a future version.
|
|
1558
|
+
* It is not guaranteed to succeed, as there are several cases where it will give
|
|
1559
|
+
* up and fail. You should NOT use this function in production code.
|
|
1560
|
+
*
|
|
1561
|
+
* This function is deprecated, and will be removed in a future version.
|
|
1562
|
+
*
|
|
1563
|
+
* Generate sequences using ZSTD_compress2(), given a source buffer.
|
|
1564
|
+
*
|
|
1565
|
+
* @param zc The compression context to be used for ZSTD_compress2(). Set any
|
|
1566
|
+
* compression parameters you need on this context.
|
|
1567
|
+
* @param outSeqs The output sequences buffer of size @p outSeqsSize
|
|
1568
|
+
* @param outSeqsSize The size of the output sequences buffer.
|
|
1569
|
+
* ZSTD_sequenceBound(srcSize) is an upper bound on the number
|
|
1570
|
+
* of sequences that can be generated.
|
|
1571
|
+
* @param src The source buffer to generate sequences from of size @p srcSize.
|
|
1572
|
+
* @param srcSize The size of the source buffer.
|
|
1312
1573
|
*
|
|
1313
1574
|
* Each block will end with a dummy sequence
|
|
1314
1575
|
* with offset == 0, matchLength == 0, and litLength == length of last literals.
|
|
1315
1576
|
* litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
|
|
1316
1577
|
* simply acts as a block delimiter.
|
|
1317
1578
|
*
|
|
1318
|
-
*
|
|
1319
|
-
*
|
|
1320
|
-
*
|
|
1321
|
-
* The output of this function can be fed into ZSTD_compressSequences() with CCtx
|
|
1322
|
-
* setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
|
|
1323
|
-
* @return : number of sequences generated
|
|
1579
|
+
* @returns The number of sequences generated, necessarily less than
|
|
1580
|
+
* ZSTD_sequenceBound(srcSize), or an error code that can be checked
|
|
1581
|
+
* with ZSTD_isError().
|
|
1324
1582
|
*/
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1583
|
+
ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()")
|
|
1584
|
+
ZSTDLIB_STATIC_API size_t
|
|
1585
|
+
ZSTD_generateSequences(ZSTD_CCtx* zc,
|
|
1586
|
+
ZSTD_Sequence* outSeqs, size_t outSeqsSize,
|
|
1587
|
+
const void* src, size_t srcSize);
|
|
1328
1588
|
|
|
1329
1589
|
/*! ZSTD_mergeBlockDelimiters() :
|
|
1330
1590
|
* Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
|
|
1331
|
-
* by merging them into
|
|
1591
|
+
* by merging them into the literals of the next sequence.
|
|
1332
1592
|
*
|
|
1333
1593
|
* As such, the final generated result has no explicit representation of block boundaries,
|
|
1334
1594
|
* and the final last literals segment is not represented in the sequences.
|
|
@@ -1337,10 +1597,12 @@ ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
|
|
|
1337
1597
|
* setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
|
|
1338
1598
|
* @return : number of sequences left after merging
|
|
1339
1599
|
*/
|
|
1340
|
-
|
|
1600
|
+
ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
|
|
1341
1601
|
|
|
1342
1602
|
/*! ZSTD_compressSequences() :
|
|
1343
|
-
* Compress an array of ZSTD_Sequence,
|
|
1603
|
+
* Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
|
|
1604
|
+
* @src contains the entire input (not just the literals).
|
|
1605
|
+
* If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
|
|
1344
1606
|
* If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
|
|
1345
1607
|
* The entire source is compressed into a single frame.
|
|
1346
1608
|
*
|
|
@@ -1365,11 +1627,49 @@ ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t se
|
|
|
1365
1627
|
* Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
|
|
1366
1628
|
* Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
|
|
1367
1629
|
* and cannot emit an RLE block that disagrees with the repcode history
|
|
1368
|
-
* @return : final compressed size or a ZSTD error.
|
|
1630
|
+
* @return : final compressed size, or a ZSTD error code.
|
|
1369
1631
|
*/
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1632
|
+
ZSTDLIB_STATIC_API size_t
|
|
1633
|
+
ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
|
|
1634
|
+
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
|
1635
|
+
const void* src, size_t srcSize);
|
|
1636
|
+
|
|
1637
|
+
|
|
1638
|
+
/*! ZSTD_writeSkippableFrame() :
|
|
1639
|
+
* Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
|
|
1640
|
+
*
|
|
1641
|
+
* Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
|
|
1642
|
+
* ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
|
|
1643
|
+
* As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
|
|
1644
|
+
* the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
|
|
1645
|
+
*
|
|
1646
|
+
* Returns an error if destination buffer is not large enough, if the source size is not representable
|
|
1647
|
+
* with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
|
|
1648
|
+
*
|
|
1649
|
+
* @return : number of bytes written or a ZSTD error.
|
|
1650
|
+
*/
|
|
1651
|
+
ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
|
|
1652
|
+
const void* src, size_t srcSize, unsigned magicVariant);
|
|
1653
|
+
|
|
1654
|
+
/*! ZSTD_readSkippableFrame() :
|
|
1655
|
+
* Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
|
|
1656
|
+
*
|
|
1657
|
+
* The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
|
|
1658
|
+
* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
|
|
1659
|
+
* in the magicVariant.
|
|
1660
|
+
*
|
|
1661
|
+
* Returns an error if destination buffer is not large enough, or if the frame is not skippable.
|
|
1662
|
+
*
|
|
1663
|
+
* @return : number of bytes written or a ZSTD error.
|
|
1664
|
+
*/
|
|
1665
|
+
ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
|
|
1666
|
+
const void* src, size_t srcSize);
|
|
1667
|
+
|
|
1668
|
+
/*! ZSTD_isSkippableFrame() :
|
|
1669
|
+
* Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
|
|
1670
|
+
*/
|
|
1671
|
+
ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
|
|
1672
|
+
|
|
1373
1673
|
|
|
1374
1674
|
|
|
1375
1675
|
/***************************************
|
|
@@ -1379,58 +1679,69 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size
|
|
|
1379
1679
|
/*! ZSTD_estimate*() :
|
|
1380
1680
|
* These functions make it possible to estimate memory usage
|
|
1381
1681
|
* of a future {D,C}Ctx, before its creation.
|
|
1682
|
+
* This is useful in combination with ZSTD_initStatic(),
|
|
1683
|
+
* which makes it possible to employ a static buffer for ZSTD_CCtx* state.
|
|
1382
1684
|
*
|
|
1383
1685
|
* ZSTD_estimateCCtxSize() will provide a memory budget large enough
|
|
1384
|
-
*
|
|
1385
|
-
*
|
|
1386
|
-
* does not include space for a window buffer.
|
|
1387
|
-
* Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
|
|
1686
|
+
* to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2()
|
|
1687
|
+
* associated with any compression level up to max specified one.
|
|
1388
1688
|
* The estimate will assume the input may be arbitrarily large,
|
|
1389
1689
|
* which is the worst case.
|
|
1390
1690
|
*
|
|
1691
|
+
* Note that the size estimation is specific for one-shot compression,
|
|
1692
|
+
* it is not valid for streaming (see ZSTD_estimateCStreamSize*())
|
|
1693
|
+
* nor other potential ways of using a ZSTD_CCtx* state.
|
|
1694
|
+
*
|
|
1391
1695
|
* When srcSize can be bound by a known and rather "small" value,
|
|
1392
|
-
* this
|
|
1393
|
-
* because the
|
|
1394
|
-
* This tighter estimation can be provided by more advanced functions
|
|
1696
|
+
* this knowledge can be used to provide a tighter budget estimation
|
|
1697
|
+
* because the ZSTD_CCtx* state will need less memory for small inputs.
|
|
1698
|
+
* This tighter estimation can be provided by employing more advanced functions
|
|
1395
1699
|
* ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
|
|
1396
1700
|
* and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
|
|
1397
1701
|
* Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
|
|
1398
1702
|
*
|
|
1399
|
-
* Note
|
|
1703
|
+
* Note : only single-threaded compression is supported.
|
|
1400
1704
|
* ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
|
|
1401
1705
|
*/
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1706
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel);
|
|
1707
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
|
|
1708
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
|
|
1709
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
|
|
1406
1710
|
|
|
1407
1711
|
/*! ZSTD_estimateCStreamSize() :
|
|
1408
|
-
* ZSTD_estimateCStreamSize() will provide a budget large enough for
|
|
1409
|
-
*
|
|
1712
|
+
* ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression
|
|
1713
|
+
* using any compression level up to the max specified one.
|
|
1714
|
+
* It will also consider src size to be arbitrarily "large", which is a worst case scenario.
|
|
1410
1715
|
* If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
|
|
1411
1716
|
* ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
|
|
1412
1717
|
* ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
|
|
1413
1718
|
* Note : CStream size estimation is only correct for single-threaded compression.
|
|
1414
|
-
*
|
|
1719
|
+
* ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
|
|
1720
|
+
* Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
|
|
1721
|
+
* Size estimates assume that no external sequence producer is registered.
|
|
1722
|
+
*
|
|
1723
|
+
* ZSTD_DStream memory budget depends on frame's window Size.
|
|
1415
1724
|
* This information can be passed manually, using ZSTD_estimateDStreamSize,
|
|
1416
1725
|
* or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
|
|
1726
|
+
* Any frame requesting a window size larger than max specified one will be rejected.
|
|
1417
1727
|
* Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
|
|
1418
1728
|
* an internal ?Dict will be created, which additional size is not estimated here.
|
|
1419
|
-
* In this case, get total size by adding ZSTD_estimate?DictSize
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1729
|
+
* In this case, get total size by adding ZSTD_estimate?DictSize
|
|
1730
|
+
*/
|
|
1731
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel);
|
|
1732
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
|
|
1733
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
|
|
1734
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize);
|
|
1735
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
|
|
1425
1736
|
|
|
1426
1737
|
/*! ZSTD_estimate?DictSize() :
|
|
1427
1738
|
* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
|
|
1428
1739
|
* ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
|
|
1429
1740
|
* Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
|
|
1430
1741
|
*/
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1742
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
|
|
1743
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
|
|
1744
|
+
ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
|
|
1434
1745
|
|
|
1435
1746
|
/*! ZSTD_initStatic*() :
|
|
1436
1747
|
* Initialize an object using a pre-allocated fixed-size buffer.
|
|
@@ -1453,20 +1764,20 @@ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e
|
|
|
1453
1764
|
* Limitation 2 : static cctx currently not compatible with multi-threading.
|
|
1454
1765
|
* Limitation 3 : static dctx is incompatible with legacy support.
|
|
1455
1766
|
*/
|
|
1456
|
-
|
|
1457
|
-
|
|
1767
|
+
ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
|
|
1768
|
+
ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
|
|
1458
1769
|
|
|
1459
|
-
|
|
1460
|
-
|
|
1770
|
+
ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
|
|
1771
|
+
ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
|
|
1461
1772
|
|
|
1462
|
-
|
|
1773
|
+
ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict(
|
|
1463
1774
|
void* workspace, size_t workspaceSize,
|
|
1464
1775
|
const void* dict, size_t dictSize,
|
|
1465
1776
|
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
1466
1777
|
ZSTD_dictContentType_e dictContentType,
|
|
1467
1778
|
ZSTD_compressionParameters cParams);
|
|
1468
1779
|
|
|
1469
|
-
|
|
1780
|
+
ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict(
|
|
1470
1781
|
void* workspace, size_t workspaceSize,
|
|
1471
1782
|
const void* dict, size_t dictSize,
|
|
1472
1783
|
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
@@ -1487,46 +1798,49 @@ __attribute__((__unused__))
|
|
|
1487
1798
|
#endif
|
|
1488
1799
|
ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
|
|
1489
1800
|
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1801
|
+
ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
|
1802
|
+
ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
|
1803
|
+
ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
|
|
1804
|
+
ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
|
1494
1805
|
|
|
1495
|
-
|
|
1806
|
+
ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
|
|
1496
1807
|
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
1497
1808
|
ZSTD_dictContentType_e dictContentType,
|
|
1498
1809
|
ZSTD_compressionParameters cParams,
|
|
1499
1810
|
ZSTD_customMem customMem);
|
|
1500
1811
|
|
|
1501
|
-
|
|
1502
|
-
*
|
|
1503
|
-
*
|
|
1504
|
-
*
|
|
1505
|
-
*
|
|
1506
|
-
*
|
|
1507
|
-
*
|
|
1508
|
-
*
|
|
1509
|
-
*
|
|
1812
|
+
/*! Thread pool :
|
|
1813
|
+
* These prototypes make it possible to share a thread pool among multiple compression contexts.
|
|
1814
|
+
* This can limit resources for applications with multiple threads where each one uses
|
|
1815
|
+
* a threaded compression mode (via ZSTD_c_nbWorkers parameter).
|
|
1816
|
+
* ZSTD_createThreadPool creates a new thread pool with a given number of threads.
|
|
1817
|
+
* Note that the lifetime of such pool must exist while being used.
|
|
1818
|
+
* ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
|
|
1819
|
+
* to use an internal thread pool).
|
|
1820
|
+
* ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
|
|
1510
1821
|
*/
|
|
1511
1822
|
typedef struct POOL_ctx_s ZSTD_threadPool;
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1823
|
+
ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
|
|
1824
|
+
ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */
|
|
1825
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
|
|
1826
|
+
|
|
1515
1827
|
|
|
1516
1828
|
/*
|
|
1517
1829
|
* This API is temporary and is expected to change or disappear in the future!
|
|
1518
1830
|
*/
|
|
1519
|
-
|
|
1831
|
+
ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2(
|
|
1520
1832
|
const void* dict, size_t dictSize,
|
|
1521
1833
|
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
1522
1834
|
ZSTD_dictContentType_e dictContentType,
|
|
1523
1835
|
const ZSTD_CCtx_params* cctxParams,
|
|
1524
1836
|
ZSTD_customMem customMem);
|
|
1525
1837
|
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1838
|
+
ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced(
|
|
1839
|
+
const void* dict, size_t dictSize,
|
|
1840
|
+
ZSTD_dictLoadMethod_e dictLoadMethod,
|
|
1841
|
+
ZSTD_dictContentType_e dictContentType,
|
|
1842
|
+
ZSTD_customMem customMem);
|
|
1843
|
+
|
|
1530
1844
|
|
|
1531
1845
|
/***************************************
|
|
1532
1846
|
* Advanced compression functions
|
|
@@ -1538,28 +1852,22 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS
|
|
|
1538
1852
|
* As a consequence, `dictBuffer` **must** outlive CDict,
|
|
1539
1853
|
* and its content must remain unmodified throughout the lifetime of CDict.
|
|
1540
1854
|
* note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
/*! ZSTD_getDictID_fromCDict() :
|
|
1544
|
-
* Provides the dictID of the dictionary loaded into `cdict`.
|
|
1545
|
-
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
|
|
1546
|
-
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
|
|
1547
|
-
ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
|
|
1855
|
+
ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
|
|
1548
1856
|
|
|
1549
1857
|
/*! ZSTD_getCParams() :
|
|
1550
1858
|
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
|
|
1551
1859
|
* `estimatedSrcSize` value is optional, select 0 if not known */
|
|
1552
|
-
|
|
1860
|
+
ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
|
|
1553
1861
|
|
|
1554
1862
|
/*! ZSTD_getParams() :
|
|
1555
1863
|
* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
|
|
1556
1864
|
* All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
|
|
1557
|
-
|
|
1865
|
+
ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
|
|
1558
1866
|
|
|
1559
1867
|
/*! ZSTD_checkCParams() :
|
|
1560
1868
|
* Ensure param values remain within authorized range.
|
|
1561
1869
|
* @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
|
|
1562
|
-
|
|
1870
|
+
ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
|
1563
1871
|
|
|
1564
1872
|
/*! ZSTD_adjustCParams() :
|
|
1565
1873
|
* optimize params for a given `srcSize` and `dictSize`.
|
|
@@ -1567,23 +1875,48 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
|
|
1567
1875
|
* `dictSize` must be `0` when there is no dictionary.
|
|
1568
1876
|
* cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
|
|
1569
1877
|
* This function never fails (wide contract) */
|
|
1570
|
-
|
|
1878
|
+
ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
|
|
1879
|
+
|
|
1880
|
+
/*! ZSTD_CCtx_setCParams() :
|
|
1881
|
+
* Set all parameters provided within @p cparams into the working @p cctx.
|
|
1882
|
+
* Note : if modifying parameters during compression (MT mode only),
|
|
1883
|
+
* note that changes to the .windowLog parameter will be ignored.
|
|
1884
|
+
* @return 0 on success, or an error code (can be checked with ZSTD_isError()).
|
|
1885
|
+
* On failure, no parameters are updated.
|
|
1886
|
+
*/
|
|
1887
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
|
|
1888
|
+
|
|
1889
|
+
/*! ZSTD_CCtx_setFParams() :
|
|
1890
|
+
* Set all parameters provided within @p fparams into the working @p cctx.
|
|
1891
|
+
* @return 0 on success, or an error code (can be checked with ZSTD_isError()).
|
|
1892
|
+
*/
|
|
1893
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams);
|
|
1894
|
+
|
|
1895
|
+
/*! ZSTD_CCtx_setParams() :
|
|
1896
|
+
* Set all parameters provided within @p params into the working @p cctx.
|
|
1897
|
+
* @return 0 on success, or an error code (can be checked with ZSTD_isError()).
|
|
1898
|
+
*/
|
|
1899
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params);
|
|
1571
1900
|
|
|
1572
1901
|
/*! ZSTD_compress_advanced() :
|
|
1573
1902
|
* Note : this function is now DEPRECATED.
|
|
1574
1903
|
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
|
|
1575
|
-
* This prototype will
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1904
|
+
* This prototype will generate compilation warnings. */
|
|
1905
|
+
ZSTD_DEPRECATED("use ZSTD_compress2")
|
|
1906
|
+
ZSTDLIB_STATIC_API
|
|
1907
|
+
size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
|
|
1908
|
+
void* dst, size_t dstCapacity,
|
|
1909
|
+
const void* src, size_t srcSize,
|
|
1910
|
+
const void* dict,size_t dictSize,
|
|
1911
|
+
ZSTD_parameters params);
|
|
1581
1912
|
|
|
1582
1913
|
/*! ZSTD_compress_usingCDict_advanced() :
|
|
1583
|
-
* Note : this function is now
|
|
1914
|
+
* Note : this function is now DEPRECATED.
|
|
1584
1915
|
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
|
|
1585
|
-
* This prototype will
|
|
1586
|
-
|
|
1916
|
+
* This prototype will generate compilation warnings. */
|
|
1917
|
+
ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
|
|
1918
|
+
ZSTDLIB_STATIC_API
|
|
1919
|
+
size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
|
|
1587
1920
|
void* dst, size_t dstCapacity,
|
|
1588
1921
|
const void* src, size_t srcSize,
|
|
1589
1922
|
const ZSTD_CDict* cdict,
|
|
@@ -1593,18 +1926,18 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
|
|
|
1593
1926
|
/*! ZSTD_CCtx_loadDictionary_byReference() :
|
|
1594
1927
|
* Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
|
|
1595
1928
|
* It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
|
|
1596
|
-
|
|
1929
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
|
|
1597
1930
|
|
|
1598
1931
|
/*! ZSTD_CCtx_loadDictionary_advanced() :
|
|
1599
1932
|
* Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
|
|
1600
1933
|
* how to load the dictionary (by copy ? by reference ?)
|
|
1601
1934
|
* and how to interpret it (automatic ? force raw mode ? full mode only ?) */
|
|
1602
|
-
|
|
1935
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
|
|
1603
1936
|
|
|
1604
1937
|
/*! ZSTD_CCtx_refPrefix_advanced() :
|
|
1605
1938
|
* Same as ZSTD_CCtx_refPrefix(), but gives finer control over
|
|
1606
1939
|
* how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
|
|
1607
|
-
|
|
1940
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
|
|
1608
1941
|
|
|
1609
1942
|
/* === experimental parameters === */
|
|
1610
1943
|
/* these parameters can be used with ZSTD_setParameter()
|
|
@@ -1643,17 +1976,18 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
|
|
|
1643
1976
|
* See the comments on that enum for an explanation of the feature. */
|
|
1644
1977
|
#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
|
|
1645
1978
|
|
|
1646
|
-
/*
|
|
1647
|
-
*
|
|
1648
|
-
*
|
|
1979
|
+
/* Controlled with ZSTD_paramSwitch_e enum.
|
|
1980
|
+
* Default is ZSTD_ps_auto.
|
|
1981
|
+
* Set to ZSTD_ps_disable to never compress literals.
|
|
1982
|
+
* Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
|
|
1983
|
+
* may still be emitted if huffman is not beneficial to use.)
|
|
1984
|
+
*
|
|
1985
|
+
* By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
|
|
1986
|
+
* literals compression based on the compression parameters - specifically,
|
|
1987
|
+
* negative compression levels do not use literal compression.
|
|
1649
1988
|
*/
|
|
1650
1989
|
#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
|
|
1651
1990
|
|
|
1652
|
-
/* Tries to fit compressed block size to be around targetCBlockSize.
|
|
1653
|
-
* No target when targetCBlockSize == 0.
|
|
1654
|
-
* There is no guarantee on compressed block size (default:0) */
|
|
1655
|
-
#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
|
|
1656
|
-
|
|
1657
1991
|
/* User's best guess of source size.
|
|
1658
1992
|
* Hint is not valid when srcSizeHint == 0.
|
|
1659
1993
|
* There is no guarantee that hint is close to actual source size,
|
|
@@ -1708,7 +2042,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
|
|
|
1708
2042
|
*
|
|
1709
2043
|
* Note that this means that the CDict tables can no longer be copied into the
|
|
1710
2044
|
* CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
|
|
1711
|
-
*
|
|
2045
|
+
* usable. The dictionary can only be attached or reloaded.
|
|
1712
2046
|
*
|
|
1713
2047
|
* In general, you should expect compression to be faster--sometimes very much
|
|
1714
2048
|
* so--and CDict creation to be slightly slower. Eventually, we will probably
|
|
@@ -1720,13 +2054,16 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
|
|
|
1720
2054
|
* Experimental parameter.
|
|
1721
2055
|
* Default is 0 == disabled. Set to 1 to enable.
|
|
1722
2056
|
*
|
|
1723
|
-
* Tells the compressor that
|
|
1724
|
-
*
|
|
1725
|
-
*
|
|
1726
|
-
*
|
|
1727
|
-
*
|
|
1728
|
-
*
|
|
1729
|
-
*
|
|
2057
|
+
* Tells the compressor that input data presented with ZSTD_inBuffer
|
|
2058
|
+
* will ALWAYS be the same between calls.
|
|
2059
|
+
* Technically, the @src pointer must never be changed,
|
|
2060
|
+
* and the @pos field can only be updated by zstd.
|
|
2061
|
+
* However, it's possible to increase the @size field,
|
|
2062
|
+
* allowing scenarios where more data can be appended after compressions starts.
|
|
2063
|
+
* These conditions are checked by the compressor,
|
|
2064
|
+
* and compression will fail if they are not respected.
|
|
2065
|
+
* Also, data in the ZSTD_inBuffer within the range [src, src + pos)
|
|
2066
|
+
* MUST not be modified during compression or it will result in data corruption.
|
|
1730
2067
|
*
|
|
1731
2068
|
* When this flag is enabled zstd won't allocate an input window buffer,
|
|
1732
2069
|
* because the user guarantees it can reference the ZSTD_inBuffer until
|
|
@@ -1734,18 +2071,15 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
|
|
|
1734
2071
|
* large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
|
|
1735
2072
|
* avoid the memcpy() from the input buffer to the input window buffer.
|
|
1736
2073
|
*
|
|
1737
|
-
* NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
|
|
1738
|
-
* That means this flag cannot be used with ZSTD_compressStream().
|
|
1739
|
-
*
|
|
1740
2074
|
* NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
|
|
1741
2075
|
* this flag is ALWAYS memory safe, and will never access out-of-bounds
|
|
1742
|
-
* memory. However, compression WILL fail if
|
|
2076
|
+
* memory. However, compression WILL fail if conditions are not respected.
|
|
1743
2077
|
*
|
|
1744
|
-
* WARNING: The data in the ZSTD_inBuffer in the range [
|
|
1745
|
-
* not be modified during compression or
|
|
1746
|
-
* is because zstd needs to reference data in the ZSTD_inBuffer to find
|
|
2078
|
+
* WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
|
|
2079
|
+
* not be modified during compression or it will result in data corruption.
|
|
2080
|
+
* This is because zstd needs to reference data in the ZSTD_inBuffer to find
|
|
1747
2081
|
* matches. Normally zstd maintains its own window buffer for this purpose,
|
|
1748
|
-
* but passing this flag tells zstd to
|
|
2082
|
+
* but passing this flag tells zstd to rely on user provided buffer instead.
|
|
1749
2083
|
*/
|
|
1750
2084
|
#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
|
|
1751
2085
|
|
|
@@ -1790,19 +2124,135 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
|
|
|
1790
2124
|
* Without validation, providing a sequence that does not conform to the zstd spec will cause
|
|
1791
2125
|
* undefined behavior, and may produce a corrupted block.
|
|
1792
2126
|
*
|
|
1793
|
-
* With validation enabled,
|
|
2127
|
+
* With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
|
|
1794
2128
|
* specifics regarding offset/matchlength requirements) then the function will bail out and
|
|
1795
2129
|
* return an error.
|
|
1796
2130
|
*
|
|
1797
2131
|
*/
|
|
1798
2132
|
#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
|
|
1799
2133
|
|
|
2134
|
+
/* ZSTD_c_useBlockSplitter
|
|
2135
|
+
* Controlled with ZSTD_paramSwitch_e enum.
|
|
2136
|
+
* Default is ZSTD_ps_auto.
|
|
2137
|
+
* Set to ZSTD_ps_disable to never use block splitter.
|
|
2138
|
+
* Set to ZSTD_ps_enable to always use block splitter.
|
|
2139
|
+
*
|
|
2140
|
+
* By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
|
|
2141
|
+
* block splitting based on the compression parameters.
|
|
2142
|
+
*/
|
|
2143
|
+
#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13
|
|
2144
|
+
|
|
2145
|
+
/* ZSTD_c_useRowMatchFinder
|
|
2146
|
+
* Controlled with ZSTD_paramSwitch_e enum.
|
|
2147
|
+
* Default is ZSTD_ps_auto.
|
|
2148
|
+
* Set to ZSTD_ps_disable to never use row-based matchfinder.
|
|
2149
|
+
* Set to ZSTD_ps_enable to force usage of row-based matchfinder.
|
|
2150
|
+
*
|
|
2151
|
+
* By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
|
|
2152
|
+
* the row-based matchfinder based on support for SIMD instructions and the window log.
|
|
2153
|
+
* Note that this only pertains to compression strategies: greedy, lazy, and lazy2
|
|
2154
|
+
*/
|
|
2155
|
+
#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
|
|
2156
|
+
|
|
2157
|
+
/* ZSTD_c_deterministicRefPrefix
|
|
2158
|
+
* Default is 0 == disabled. Set to 1 to enable.
|
|
2159
|
+
*
|
|
2160
|
+
* Zstd produces different results for prefix compression when the prefix is
|
|
2161
|
+
* directly adjacent to the data about to be compressed vs. when it isn't.
|
|
2162
|
+
* This is because zstd detects that the two buffers are contiguous and it can
|
|
2163
|
+
* use a more efficient match finding algorithm. However, this produces different
|
|
2164
|
+
* results than when the two buffers are non-contiguous. This flag forces zstd
|
|
2165
|
+
* to always load the prefix in non-contiguous mode, even if it happens to be
|
|
2166
|
+
* adjacent to the data, to guarantee determinism.
|
|
2167
|
+
*
|
|
2168
|
+
* If you really care about determinism when using a dictionary or prefix,
|
|
2169
|
+
* like when doing delta compression, you should select this option. It comes
|
|
2170
|
+
* at a speed penalty of about ~2.5% if the dictionary and data happened to be
|
|
2171
|
+
* contiguous, and is free if they weren't contiguous. We don't expect that
|
|
2172
|
+
* intentionally making the dictionary and data contiguous will be worth the
|
|
2173
|
+
* cost to memcpy() the data.
|
|
2174
|
+
*/
|
|
2175
|
+
#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
|
|
2176
|
+
|
|
2177
|
+
/* ZSTD_c_prefetchCDictTables
|
|
2178
|
+
* Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
|
|
2179
|
+
*
|
|
2180
|
+
* In some situations, zstd uses CDict tables in-place rather than copying them
|
|
2181
|
+
* into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
|
|
2182
|
+
* In such situations, compression speed is seriously impacted when CDict tables are
|
|
2183
|
+
* "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables
|
|
2184
|
+
* when they are used in-place.
|
|
2185
|
+
*
|
|
2186
|
+
* For sufficiently small inputs, the cost of the prefetch will outweigh the benefit.
|
|
2187
|
+
* For sufficiently large inputs, zstd will by default memcpy() CDict tables
|
|
2188
|
+
* into the working context, so there is no need to prefetch. This parameter is
|
|
2189
|
+
* targeted at a middle range of input sizes, where a prefetch is cheap enough to be
|
|
2190
|
+
* useful but memcpy() is too expensive. The exact range of input sizes where this
|
|
2191
|
+
* makes sense is best determined by careful experimentation.
|
|
2192
|
+
*
|
|
2193
|
+
* Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable,
|
|
2194
|
+
* but in the future zstd may conditionally enable this feature via an auto-detection
|
|
2195
|
+
* heuristic for cold CDicts.
|
|
2196
|
+
* Use ZSTD_ps_disable to opt out of prefetching under any circumstances.
|
|
2197
|
+
*/
|
|
2198
|
+
#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
|
|
2199
|
+
|
|
2200
|
+
/* ZSTD_c_enableSeqProducerFallback
|
|
2201
|
+
* Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
|
|
2202
|
+
*
|
|
2203
|
+
* Controls whether zstd will fall back to an internal sequence producer if an
|
|
2204
|
+
* external sequence producer is registered and returns an error code. This fallback
|
|
2205
|
+
* is block-by-block: the internal sequence producer will only be called for blocks
|
|
2206
|
+
* where the external sequence producer returns an error code. Fallback parsing will
|
|
2207
|
+
* follow any other cParam settings, such as compression level, the same as in a
|
|
2208
|
+
* normal (fully-internal) compression operation.
|
|
2209
|
+
*
|
|
2210
|
+
* The user is strongly encouraged to read the full Block-Level Sequence Producer API
|
|
2211
|
+
* documentation (below) before setting this parameter. */
|
|
2212
|
+
#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17
|
|
2213
|
+
|
|
2214
|
+
/* ZSTD_c_maxBlockSize
|
|
2215
|
+
* Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
|
|
2216
|
+
* The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
|
|
2217
|
+
*
|
|
2218
|
+
* This parameter can be used to set an upper bound on the blocksize
|
|
2219
|
+
* that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
|
|
2220
|
+
* bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
|
|
2221
|
+
* compressBound() inaccurate). Only currently meant to be used for testing.
|
|
2222
|
+
*
|
|
2223
|
+
*/
|
|
2224
|
+
#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
|
|
2225
|
+
|
|
2226
|
+
/* ZSTD_c_searchForExternalRepcodes
|
|
2227
|
+
* This parameter affects how zstd parses external sequences, such as sequences
|
|
2228
|
+
* provided through the compressSequences() API or from an external block-level
|
|
2229
|
+
* sequence producer.
|
|
2230
|
+
*
|
|
2231
|
+
* If set to ZSTD_ps_enable, the library will check for repeated offsets in
|
|
2232
|
+
* external sequences, even if those repcodes are not explicitly indicated in
|
|
2233
|
+
* the "rep" field. Note that this is the only way to exploit repcode matches
|
|
2234
|
+
* while using compressSequences() or an external sequence producer, since zstd
|
|
2235
|
+
* currently ignores the "rep" field of external sequences.
|
|
2236
|
+
*
|
|
2237
|
+
* If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
|
|
2238
|
+
* external sequences, regardless of whether the "rep" field has been set. This
|
|
2239
|
+
* reduces sequence compression overhead by about 25% while sacrificing some
|
|
2240
|
+
* compression ratio.
|
|
2241
|
+
*
|
|
2242
|
+
* The default value is ZSTD_ps_auto, for which the library will enable/disable
|
|
2243
|
+
* based on compression level.
|
|
2244
|
+
*
|
|
2245
|
+
* Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
|
|
2246
|
+
* set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
|
|
2247
|
+
*/
|
|
2248
|
+
#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
|
|
2249
|
+
|
|
1800
2250
|
/*! ZSTD_CCtx_getParameter() :
|
|
1801
2251
|
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
|
|
1802
2252
|
* and store it into int* value.
|
|
1803
2253
|
* @return : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1804
2254
|
*/
|
|
1805
|
-
|
|
2255
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
|
|
1806
2256
|
|
|
1807
2257
|
|
|
1808
2258
|
/*! ZSTD_CCtx_params :
|
|
@@ -1817,32 +2267,32 @@ ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
|
|
|
1817
2267
|
* These parameters will be applied to
|
|
1818
2268
|
* all subsequent frames.
|
|
1819
2269
|
* - ZSTD_compressStream2() : Do compression using the CCtx.
|
|
1820
|
-
* - ZSTD_freeCCtxParams() : Free the memory.
|
|
2270
|
+
* - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer.
|
|
1821
2271
|
*
|
|
1822
2272
|
* This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
|
|
1823
2273
|
* for static allocation of CCtx for single-threaded compression.
|
|
1824
2274
|
*/
|
|
1825
|
-
|
|
1826
|
-
|
|
2275
|
+
ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
|
|
2276
|
+
ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */
|
|
1827
2277
|
|
|
1828
2278
|
/*! ZSTD_CCtxParams_reset() :
|
|
1829
2279
|
* Reset params to default values.
|
|
1830
2280
|
*/
|
|
1831
|
-
|
|
2281
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
|
|
1832
2282
|
|
|
1833
2283
|
/*! ZSTD_CCtxParams_init() :
|
|
1834
2284
|
* Initializes the compression parameters of cctxParams according to
|
|
1835
2285
|
* compression level. All other parameters are reset to their default values.
|
|
1836
2286
|
*/
|
|
1837
|
-
|
|
2287
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
|
|
1838
2288
|
|
|
1839
2289
|
/*! ZSTD_CCtxParams_init_advanced() :
|
|
1840
2290
|
* Initializes the compression and frame parameters of cctxParams according to
|
|
1841
2291
|
* params. All other parameters are reset to their default values.
|
|
1842
2292
|
*/
|
|
1843
|
-
|
|
2293
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
|
|
1844
2294
|
|
|
1845
|
-
/*! ZSTD_CCtxParams_setParameter() :
|
|
2295
|
+
/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+
|
|
1846
2296
|
* Similar to ZSTD_CCtx_setParameter.
|
|
1847
2297
|
* Set one compression parameter, selected by enum ZSTD_cParameter.
|
|
1848
2298
|
* Parameters must be applied to a ZSTD_CCtx using
|
|
@@ -1850,14 +2300,14 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, Z
|
|
|
1850
2300
|
* @result : a code representing success or failure (which can be tested with
|
|
1851
2301
|
* ZSTD_isError()).
|
|
1852
2302
|
*/
|
|
1853
|
-
|
|
2303
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
|
|
1854
2304
|
|
|
1855
2305
|
/*! ZSTD_CCtxParams_getParameter() :
|
|
1856
2306
|
* Similar to ZSTD_CCtx_getParameter.
|
|
1857
2307
|
* Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
|
|
1858
2308
|
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1859
2309
|
*/
|
|
1860
|
-
|
|
2310
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
|
|
1861
2311
|
|
|
1862
2312
|
/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
|
|
1863
2313
|
* Apply a set of ZSTD_CCtx_params to the compression context.
|
|
@@ -1866,7 +2316,7 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_c
|
|
|
1866
2316
|
* if nbWorkers>=1, new parameters will be picked up at next job,
|
|
1867
2317
|
* with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
|
|
1868
2318
|
*/
|
|
1869
|
-
|
|
2319
|
+
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
|
|
1870
2320
|
ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
|
|
1871
2321
|
|
|
1872
2322
|
/*! ZSTD_compressStream2_simpleArgs() :
|
|
@@ -1875,7 +2325,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
|
|
|
1875
2325
|
* This variant might be helpful for binders from dynamic languages
|
|
1876
2326
|
* which have troubles handling structures containing memory pointers.
|
|
1877
2327
|
*/
|
|
1878
|
-
|
|
2328
|
+
ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
|
|
1879
2329
|
ZSTD_CCtx* cctx,
|
|
1880
2330
|
void* dst, size_t dstCapacity, size_t* dstPos,
|
|
1881
2331
|
const void* src, size_t srcSize, size_t* srcPos,
|
|
@@ -1891,33 +2341,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs (
|
|
|
1891
2341
|
* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
|
|
1892
2342
|
* Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
|
|
1893
2343
|
* Note 3 : Skippable Frame Identifiers are considered valid. */
|
|
1894
|
-
|
|
2344
|
+
ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
|
|
1895
2345
|
|
|
1896
2346
|
/*! ZSTD_createDDict_byReference() :
|
|
1897
2347
|
* Create a digested dictionary, ready to start decompression operation without startup delay.
|
|
1898
2348
|
* Dictionary content is referenced, and therefore stays in dictBuffer.
|
|
1899
2349
|
* It is important that dictBuffer outlives DDict,
|
|
1900
2350
|
* it must remain read accessible throughout the lifetime of DDict */
|
|
1901
|
-
|
|
2351
|
+
ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
|
|
1902
2352
|
|
|
1903
2353
|
/*! ZSTD_DCtx_loadDictionary_byReference() :
|
|
1904
2354
|
* Same as ZSTD_DCtx_loadDictionary(),
|
|
1905
2355
|
* but references `dict` content instead of copying it into `dctx`.
|
|
1906
2356
|
* This saves memory if `dict` remains around.,
|
|
1907
2357
|
* However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
|
|
1908
|
-
|
|
2358
|
+
ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
|
1909
2359
|
|
|
1910
2360
|
/*! ZSTD_DCtx_loadDictionary_advanced() :
|
|
1911
2361
|
* Same as ZSTD_DCtx_loadDictionary(),
|
|
1912
2362
|
* but gives direct control over
|
|
1913
2363
|
* how to load the dictionary (by copy ? by reference ?)
|
|
1914
2364
|
* and how to interpret it (automatic ? force raw mode ? full mode only ?). */
|
|
1915
|
-
|
|
2365
|
+
ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
|
|
1916
2366
|
|
|
1917
2367
|
/*! ZSTD_DCtx_refPrefix_advanced() :
|
|
1918
2368
|
* Same as ZSTD_DCtx_refPrefix(), but gives finer control over
|
|
1919
2369
|
* how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
|
|
1920
|
-
|
|
2370
|
+
ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
|
|
1921
2371
|
|
|
1922
2372
|
/*! ZSTD_DCtx_setMaxWindowSize() :
|
|
1923
2373
|
* Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
|
|
@@ -1926,14 +2376,14 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre
|
|
|
1926
2376
|
* By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
|
|
1927
2377
|
* @return : 0, or an error code (which can be tested using ZSTD_isError()).
|
|
1928
2378
|
*/
|
|
1929
|
-
|
|
2379
|
+
ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
|
|
1930
2380
|
|
|
1931
2381
|
/*! ZSTD_DCtx_getParameter() :
|
|
1932
2382
|
* Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
|
|
1933
2383
|
* and store it into int* value.
|
|
1934
2384
|
* @return : 0, or an error code (which can be tested with ZSTD_isError()).
|
|
1935
2385
|
*/
|
|
1936
|
-
|
|
2386
|
+
ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
|
|
1937
2387
|
|
|
1938
2388
|
/* ZSTD_d_format
|
|
1939
2389
|
* experimental parameter,
|
|
@@ -1953,7 +2403,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param
|
|
|
1953
2403
|
* in the range [dst, dst + pos) MUST not be modified during decompression
|
|
1954
2404
|
* or you will get data corruption.
|
|
1955
2405
|
*
|
|
1956
|
-
* When this
|
|
2406
|
+
* When this flag is enabled zstd won't allocate an output buffer, because
|
|
1957
2407
|
* it can write directly to the ZSTD_outBuffer, but it will still allocate
|
|
1958
2408
|
* an input buffer large enough to fit any compressed block. This will also
|
|
1959
2409
|
* avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
|
|
@@ -1983,12 +2433,66 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param
|
|
|
1983
2433
|
*/
|
|
1984
2434
|
#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
|
|
1985
2435
|
|
|
2436
|
+
/* ZSTD_d_refMultipleDDicts
|
|
2437
|
+
* Experimental parameter.
|
|
2438
|
+
* Default is 0 == disabled. Set to 1 to enable
|
|
2439
|
+
*
|
|
2440
|
+
* If enabled and dctx is allocated on the heap, then additional memory will be allocated
|
|
2441
|
+
* to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
|
|
2442
|
+
* using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
|
|
2443
|
+
* store all references. At decompression time, the appropriate dictID is selected
|
|
2444
|
+
* from the set of DDicts based on the dictID in the frame.
|
|
2445
|
+
*
|
|
2446
|
+
* Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
|
|
2447
|
+
*
|
|
2448
|
+
* Param has values of byte ZSTD_refMultipleDDicts_e
|
|
2449
|
+
*
|
|
2450
|
+
* WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
|
|
2451
|
+
* allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
|
|
2452
|
+
* Memory is allocated as per ZSTD_DCtx::customMem.
|
|
2453
|
+
*
|
|
2454
|
+
* Although this function allocates memory for the table, the user is still responsible for
|
|
2455
|
+
* memory management of the underlying ZSTD_DDict* themselves.
|
|
2456
|
+
*/
|
|
2457
|
+
#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
|
|
2458
|
+
|
|
2459
|
+
/* ZSTD_d_disableHuffmanAssembly
|
|
2460
|
+
* Set to 1 to disable the Huffman assembly implementation.
|
|
2461
|
+
* The default value is 0, which allows zstd to use the Huffman assembly
|
|
2462
|
+
* implementation if available.
|
|
2463
|
+
*
|
|
2464
|
+
* This parameter can be used to disable Huffman assembly at runtime.
|
|
2465
|
+
* If you want to disable it at compile time you can define the macro
|
|
2466
|
+
* ZSTD_DISABLE_ASM.
|
|
2467
|
+
*/
|
|
2468
|
+
#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
|
|
2469
|
+
|
|
2470
|
+
/* ZSTD_d_maxBlockSize
|
|
2471
|
+
* Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
|
|
2472
|
+
* The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
|
|
2473
|
+
*
|
|
2474
|
+
* Forces the decompressor to reject blocks whose content size is
|
|
2475
|
+
* larger than the configured maxBlockSize. When maxBlockSize is
|
|
2476
|
+
* larger than the windowSize, the windowSize is used instead.
|
|
2477
|
+
* This saves memory on the decoder when you know all blocks are small.
|
|
2478
|
+
*
|
|
2479
|
+
* This option is typically used in conjunction with ZSTD_c_maxBlockSize.
|
|
2480
|
+
*
|
|
2481
|
+
* WARNING: This causes the decoder to reject otherwise valid frames
|
|
2482
|
+
* that have block sizes larger than the configured maxBlockSize.
|
|
2483
|
+
*/
|
|
2484
|
+
#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6
|
|
2485
|
+
|
|
2486
|
+
|
|
1986
2487
|
/*! ZSTD_DCtx_setFormat() :
|
|
2488
|
+
* This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
|
|
1987
2489
|
* Instruct the decoder context about what kind of data to decode next.
|
|
1988
2490
|
* This instruction is mandatory to decode data without a fully-formed header,
|
|
1989
2491
|
* such ZSTD_f_zstd1_magicless for example.
|
|
1990
2492
|
* @return : 0, or an error code (which can be tested using ZSTD_isError()). */
|
|
1991
|
-
|
|
2493
|
+
ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
|
|
2494
|
+
ZSTDLIB_STATIC_API
|
|
2495
|
+
size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
|
|
1992
2496
|
|
|
1993
2497
|
/*! ZSTD_decompressStream_simpleArgs() :
|
|
1994
2498
|
* Same as ZSTD_decompressStream(),
|
|
@@ -1996,7 +2500,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
|
|
|
1996
2500
|
* This can be helpful for binders from dynamic languages
|
|
1997
2501
|
* which have troubles handling structures containing memory pointers.
|
|
1998
2502
|
*/
|
|
1999
|
-
|
|
2503
|
+
ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
|
|
2000
2504
|
ZSTD_DCtx* dctx,
|
|
2001
2505
|
void* dst, size_t dstCapacity, size_t* dstPos,
|
|
2002
2506
|
const void* src, size_t srcSize, size_t* srcPos);
|
|
@@ -2012,7 +2516,7 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
|
|
|
2012
2516
|
/*===== Advanced Streaming compression functions =====*/
|
|
2013
2517
|
|
|
2014
2518
|
/*! ZSTD_initCStream_srcSize() :
|
|
2015
|
-
* This function is
|
|
2519
|
+
* This function is DEPRECATED, and equivalent to:
|
|
2016
2520
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
2017
2521
|
* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
|
|
2018
2522
|
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
|
@@ -2021,15 +2525,16 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
|
|
|
2021
2525
|
* pledgedSrcSize must be correct. If it is not known at init time, use
|
|
2022
2526
|
* ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
|
|
2023
2527
|
* "0" also disables frame content size field. It may be enabled in the future.
|
|
2024
|
-
*
|
|
2528
|
+
* This prototype will generate compilation warnings.
|
|
2025
2529
|
*/
|
|
2026
|
-
|
|
2027
|
-
|
|
2530
|
+
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
|
|
2531
|
+
ZSTDLIB_STATIC_API
|
|
2532
|
+
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
|
|
2028
2533
|
int compressionLevel,
|
|
2029
2534
|
unsigned long long pledgedSrcSize);
|
|
2030
2535
|
|
|
2031
2536
|
/*! ZSTD_initCStream_usingDict() :
|
|
2032
|
-
* This function is
|
|
2537
|
+
* This function is DEPRECATED, and is equivalent to:
|
|
2033
2538
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
2034
2539
|
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
|
|
2035
2540
|
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
|
|
@@ -2038,81 +2543,85 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
|
|
|
2038
2543
|
* dict == NULL or dictSize < 8, in which case no dict is used.
|
|
2039
2544
|
* Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
|
|
2040
2545
|
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
|
|
2041
|
-
*
|
|
2546
|
+
* This prototype will generate compilation warnings.
|
|
2042
2547
|
*/
|
|
2043
|
-
|
|
2044
|
-
|
|
2548
|
+
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
|
|
2549
|
+
ZSTDLIB_STATIC_API
|
|
2550
|
+
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
|
|
2045
2551
|
const void* dict, size_t dictSize,
|
|
2046
2552
|
int compressionLevel);
|
|
2047
2553
|
|
|
2048
2554
|
/*! ZSTD_initCStream_advanced() :
|
|
2049
|
-
* This function is
|
|
2555
|
+
* This function is DEPRECATED, and is equivalent to:
|
|
2050
2556
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
2051
|
-
*
|
|
2052
|
-
* for ((param, value) : params) {
|
|
2053
|
-
* ZSTD_CCtx_setParameter(zcs, param, value);
|
|
2054
|
-
* }
|
|
2557
|
+
* ZSTD_CCtx_setParams(zcs, params);
|
|
2055
2558
|
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
|
2056
2559
|
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
|
|
2057
2560
|
*
|
|
2058
2561
|
* dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
|
|
2059
2562
|
* pledgedSrcSize must be correct.
|
|
2060
2563
|
* If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
|
|
2061
|
-
*
|
|
2564
|
+
* This prototype will generate compilation warnings.
|
|
2062
2565
|
*/
|
|
2063
|
-
|
|
2064
|
-
|
|
2566
|
+
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
|
|
2567
|
+
ZSTDLIB_STATIC_API
|
|
2568
|
+
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
|
2065
2569
|
const void* dict, size_t dictSize,
|
|
2066
2570
|
ZSTD_parameters params,
|
|
2067
2571
|
unsigned long long pledgedSrcSize);
|
|
2068
2572
|
|
|
2069
2573
|
/*! ZSTD_initCStream_usingCDict() :
|
|
2070
|
-
* This function is
|
|
2574
|
+
* This function is DEPRECATED, and equivalent to:
|
|
2071
2575
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
2072
2576
|
* ZSTD_CCtx_refCDict(zcs, cdict);
|
|
2073
2577
|
*
|
|
2074
2578
|
* note : cdict will just be referenced, and must outlive compression session
|
|
2075
|
-
*
|
|
2579
|
+
* This prototype will generate compilation warnings.
|
|
2076
2580
|
*/
|
|
2077
|
-
|
|
2581
|
+
ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
|
|
2582
|
+
ZSTDLIB_STATIC_API
|
|
2583
|
+
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
|
|
2078
2584
|
|
|
2079
2585
|
/*! ZSTD_initCStream_usingCDict_advanced() :
|
|
2080
|
-
* This function is DEPRECATED, and is
|
|
2586
|
+
* This function is DEPRECATED, and is equivalent to:
|
|
2081
2587
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
2082
|
-
*
|
|
2083
|
-
* for ((fParam, value) : fParams) {
|
|
2084
|
-
* ZSTD_CCtx_setParameter(zcs, fParam, value);
|
|
2085
|
-
* }
|
|
2588
|
+
* ZSTD_CCtx_setFParams(zcs, fParams);
|
|
2086
2589
|
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
|
2087
2590
|
* ZSTD_CCtx_refCDict(zcs, cdict);
|
|
2088
2591
|
*
|
|
2089
2592
|
* same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
|
|
2090
2593
|
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
|
|
2091
2594
|
* value ZSTD_CONTENTSIZE_UNKNOWN.
|
|
2092
|
-
*
|
|
2595
|
+
* This prototype will generate compilation warnings.
|
|
2093
2596
|
*/
|
|
2094
|
-
|
|
2095
|
-
|
|
2597
|
+
ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
|
|
2598
|
+
ZSTDLIB_STATIC_API
|
|
2599
|
+
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
|
|
2096
2600
|
const ZSTD_CDict* cdict,
|
|
2097
2601
|
ZSTD_frameParameters fParams,
|
|
2098
2602
|
unsigned long long pledgedSrcSize);
|
|
2099
2603
|
|
|
2100
2604
|
/*! ZSTD_resetCStream() :
|
|
2101
|
-
* This function is
|
|
2605
|
+
* This function is DEPRECATED, and is equivalent to:
|
|
2102
2606
|
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
|
|
2103
2607
|
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
|
|
2608
|
+
* Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
|
|
2609
|
+
* ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
|
|
2610
|
+
* explicitly specified.
|
|
2104
2611
|
*
|
|
2105
2612
|
* start a new frame, using same parameters from previous frame.
|
|
2106
|
-
* This is typically useful to skip dictionary loading stage, since it will
|
|
2613
|
+
* This is typically useful to skip dictionary loading stage, since it will reuse it in-place.
|
|
2107
2614
|
* Note that zcs must be init at least once before using ZSTD_resetCStream().
|
|
2108
2615
|
* If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
|
|
2109
2616
|
* If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
|
|
2110
2617
|
* For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
|
|
2111
2618
|
* but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
|
|
2112
2619
|
* @return : 0, or an error code (which can be tested using ZSTD_isError())
|
|
2113
|
-
*
|
|
2620
|
+
* This prototype will generate compilation warnings.
|
|
2114
2621
|
*/
|
|
2115
|
-
|
|
2622
|
+
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
|
|
2623
|
+
ZSTDLIB_STATIC_API
|
|
2624
|
+
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
|
|
2116
2625
|
|
|
2117
2626
|
|
|
2118
2627
|
typedef struct {
|
|
@@ -2130,7 +2639,7 @@ typedef struct {
|
|
|
2130
2639
|
* Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
|
|
2131
2640
|
* Aggregates progression inside active worker threads.
|
|
2132
2641
|
*/
|
|
2133
|
-
|
|
2642
|
+
ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
|
|
2134
2643
|
|
|
2135
2644
|
/*! ZSTD_toFlushNow() :
|
|
2136
2645
|
* Tell how many bytes are ready to be flushed immediately.
|
|
@@ -2145,7 +2654,7 @@ ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx
|
|
|
2145
2654
|
* therefore flush speed is limited by production speed of oldest job
|
|
2146
2655
|
* irrespective of the speed of concurrent (and newer) jobs.
|
|
2147
2656
|
*/
|
|
2148
|
-
|
|
2657
|
+
ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
|
|
2149
2658
|
|
|
2150
2659
|
|
|
2151
2660
|
/*===== Advanced Streaming decompression functions =====*/
|
|
@@ -2157,9 +2666,9 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
|
|
|
2157
2666
|
* ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
|
|
2158
2667
|
*
|
|
2159
2668
|
* note: no dictionary will be used if dict == NULL or dictSize < 8
|
|
2160
|
-
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
2161
2669
|
*/
|
|
2162
|
-
|
|
2670
|
+
ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
|
|
2671
|
+
ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
|
|
2163
2672
|
|
|
2164
2673
|
/*!
|
|
2165
2674
|
* This function is deprecated, and is equivalent to:
|
|
@@ -2168,27 +2677,211 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic
|
|
|
2168
2677
|
* ZSTD_DCtx_refDDict(zds, ddict);
|
|
2169
2678
|
*
|
|
2170
2679
|
* note : ddict is referenced, it must outlive decompression session
|
|
2171
|
-
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
|
|
2172
2680
|
*/
|
|
2173
|
-
|
|
2681
|
+
ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
|
|
2682
|
+
ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
|
|
2174
2683
|
|
|
2175
2684
|
/*!
|
|
2176
2685
|
* This function is deprecated, and is equivalent to:
|
|
2177
2686
|
*
|
|
2178
2687
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
|
2179
2688
|
*
|
|
2180
|
-
*
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2689
|
+
* reuse decompression parameters from previous init; saves dictionary loading
|
|
2690
|
+
*/
|
|
2691
|
+
ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
|
|
2692
|
+
ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
|
|
2693
|
+
|
|
2694
|
+
|
|
2695
|
+
/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
|
|
2696
|
+
*
|
|
2697
|
+
* *** OVERVIEW ***
|
|
2698
|
+
* The Block-Level Sequence Producer API allows users to provide their own custom
|
|
2699
|
+
* sequence producer which libzstd invokes to process each block. The produced list
|
|
2700
|
+
* of sequences (literals and matches) is then post-processed by libzstd to produce
|
|
2701
|
+
* valid compressed blocks.
|
|
2702
|
+
*
|
|
2703
|
+
* This block-level offload API is a more granular complement of the existing
|
|
2704
|
+
* frame-level offload API compressSequences() (introduced in v1.5.1). It offers
|
|
2705
|
+
* an easier migration story for applications already integrated with libzstd: the
|
|
2706
|
+
* user application continues to invoke the same compression functions
|
|
2707
|
+
* ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits
|
|
2708
|
+
* from the specific advantages of the external sequence producer. For example,
|
|
2709
|
+
* the sequence producer could be tuned to take advantage of known characteristics
|
|
2710
|
+
* of the input, to offer better speed / ratio, or could leverage hardware
|
|
2711
|
+
* acceleration not available within libzstd itself.
|
|
2712
|
+
*
|
|
2713
|
+
* See contrib/externalSequenceProducer for an example program employing the
|
|
2714
|
+
* Block-Level Sequence Producer API.
|
|
2715
|
+
*
|
|
2716
|
+
* *** USAGE ***
|
|
2717
|
+
* The user is responsible for implementing a function of type
|
|
2718
|
+
* ZSTD_sequenceProducer_F. For each block, zstd will pass the following
|
|
2719
|
+
* arguments to the user-provided function:
|
|
2720
|
+
*
|
|
2721
|
+
* - sequenceProducerState: a pointer to a user-managed state for the sequence
|
|
2722
|
+
* producer.
|
|
2723
|
+
*
|
|
2724
|
+
* - outSeqs, outSeqsCapacity: an output buffer for the sequence producer.
|
|
2725
|
+
* outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory
|
|
2726
|
+
* backing outSeqs is managed by the CCtx.
|
|
2727
|
+
*
|
|
2728
|
+
* - src, srcSize: an input buffer for the sequence producer to parse.
|
|
2729
|
+
* srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
|
|
2730
|
+
*
|
|
2731
|
+
* - dict, dictSize: a history buffer, which may be empty, which the sequence
|
|
2732
|
+
* producer may reference as it parses the src buffer. Currently, zstd will
|
|
2733
|
+
* always pass dictSize == 0 into external sequence producers, but this will
|
|
2734
|
+
* change in the future.
|
|
2735
|
+
*
|
|
2736
|
+
* - compressionLevel: a signed integer representing the zstd compression level
|
|
2737
|
+
* set by the user for the current operation. The sequence producer may choose
|
|
2738
|
+
* to use this information to change its compression strategy and speed/ratio
|
|
2739
|
+
* tradeoff. Note: the compression level does not reflect zstd parameters set
|
|
2740
|
+
* through the advanced API.
|
|
2741
|
+
*
|
|
2742
|
+
* - windowSize: a size_t representing the maximum allowed offset for external
|
|
2743
|
+
* sequences. Note that sequence offsets are sometimes allowed to exceed the
|
|
2744
|
+
* windowSize if a dictionary is present, see doc/zstd_compression_format.md
|
|
2745
|
+
* for details.
|
|
2746
|
+
*
|
|
2747
|
+
* The user-provided function shall return a size_t representing the number of
|
|
2748
|
+
* sequences written to outSeqs. This return value will be treated as an error
|
|
2749
|
+
* code if it is greater than outSeqsCapacity. The return value must be non-zero
|
|
2750
|
+
* if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided
|
|
2751
|
+
* for convenience, but any value greater than outSeqsCapacity will be treated as
|
|
2752
|
+
* an error code.
|
|
2753
|
+
*
|
|
2754
|
+
* If the user-provided function does not return an error code, the sequences
|
|
2755
|
+
* written to outSeqs must be a valid parse of the src buffer. Data corruption may
|
|
2756
|
+
* occur if the parse is not valid. A parse is defined to be valid if the
|
|
2757
|
+
* following conditions hold:
|
|
2758
|
+
* - The sum of matchLengths and literalLengths must equal srcSize.
|
|
2759
|
+
* - All sequences in the parse, except for the final sequence, must have
|
|
2760
|
+
* matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have
|
|
2761
|
+
* matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0.
|
|
2762
|
+
* - All offsets must respect the windowSize parameter as specified in
|
|
2763
|
+
* doc/zstd_compression_format.md.
|
|
2764
|
+
* - If the final sequence has matchLength == 0, it must also have offset == 0.
|
|
2765
|
+
*
|
|
2766
|
+
* zstd will only validate these conditions (and fail compression if they do not
|
|
2767
|
+
* hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
|
|
2768
|
+
* validation has a performance cost.
|
|
2769
|
+
*
|
|
2770
|
+
* If the user-provided function returns an error, zstd will either fall back
|
|
2771
|
+
* to an internal sequence producer or fail the compression operation. The user can
|
|
2772
|
+
* choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback
|
|
2773
|
+
* cParam. Fallback compression will follow any other cParam settings, such as
|
|
2774
|
+
* compression level, the same as in a normal compression operation.
|
|
2775
|
+
*
|
|
2776
|
+
* The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F
|
|
2777
|
+
* function by calling
|
|
2778
|
+
* ZSTD_registerSequenceProducer(cctx,
|
|
2779
|
+
* sequenceProducerState,
|
|
2780
|
+
* sequenceProducer)
|
|
2781
|
+
* This setting will persist until the next parameter reset of the CCtx.
|
|
2782
|
+
*
|
|
2783
|
+
* The sequenceProducerState must be initialized by the user before calling
|
|
2784
|
+
* ZSTD_registerSequenceProducer(). The user is responsible for destroying the
|
|
2785
|
+
* sequenceProducerState.
|
|
2786
|
+
*
|
|
2787
|
+
* *** LIMITATIONS ***
|
|
2788
|
+
* This API is compatible with all zstd compression APIs which respect advanced parameters.
|
|
2789
|
+
* However, there are three limitations:
|
|
2790
|
+
*
|
|
2791
|
+
* First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported.
|
|
2792
|
+
* COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level
|
|
2793
|
+
* external sequence producer.
|
|
2794
|
+
* - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some
|
|
2795
|
+
* cases (see its documentation for details). Users must explicitly set
|
|
2796
|
+
* ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external
|
|
2797
|
+
* sequence producer is registered.
|
|
2798
|
+
* - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
|
|
2799
|
+
* whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
|
|
2800
|
+
* check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence
|
|
2801
|
+
* Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog).
|
|
2802
|
+
*
|
|
2803
|
+
* Second, history buffers are not currently supported. Concretely, zstd will always pass
|
|
2804
|
+
* dictSize == 0 to the external sequence producer (for now). This has two implications:
|
|
2805
|
+
* - Dictionaries are not currently supported. Compression will *not* fail if the user
|
|
2806
|
+
* references a dictionary, but the dictionary won't have any effect.
|
|
2807
|
+
* - Stream history is not currently supported. All advanced compression APIs, including
|
|
2808
|
+
* streaming APIs, work with external sequence producers, but each block is treated as
|
|
2809
|
+
* an independent chunk without history from previous blocks.
|
|
2810
|
+
*
|
|
2811
|
+
* Third, multi-threading within a single compression is not currently supported. In other words,
|
|
2812
|
+
* COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered.
|
|
2813
|
+
* Multi-threading across compressions is fine: simply create one CCtx per thread.
|
|
2814
|
+
*
|
|
2815
|
+
* Long-term, we plan to overcome all three limitations. There is no technical blocker to
|
|
2816
|
+
* overcoming them. It is purely a question of engineering effort.
|
|
2817
|
+
*/
|
|
2818
|
+
|
|
2819
|
+
#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1))
|
|
2820
|
+
|
|
2821
|
+
typedef size_t (*ZSTD_sequenceProducer_F) (
|
|
2822
|
+
void* sequenceProducerState,
|
|
2823
|
+
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
|
|
2824
|
+
const void* src, size_t srcSize,
|
|
2825
|
+
const void* dict, size_t dictSize,
|
|
2826
|
+
int compressionLevel,
|
|
2827
|
+
size_t windowSize
|
|
2828
|
+
);
|
|
2829
|
+
|
|
2830
|
+
/*! ZSTD_registerSequenceProducer() :
|
|
2831
|
+
* Instruct zstd to use a block-level external sequence producer function.
|
|
2832
|
+
*
|
|
2833
|
+
* The sequenceProducerState must be initialized by the caller, and the caller is
|
|
2834
|
+
* responsible for managing its lifetime. This parameter is sticky across
|
|
2835
|
+
* compressions. It will remain set until the user explicitly resets compression
|
|
2836
|
+
* parameters.
|
|
2837
|
+
*
|
|
2838
|
+
* Sequence producer registration is considered to be an "advanced parameter",
|
|
2839
|
+
* part of the "advanced API". This means it will only have an effect on compression
|
|
2840
|
+
* APIs which respect advanced parameters, such as compress2() and compressStream2().
|
|
2841
|
+
* Older compression APIs such as compressCCtx(), which predate the introduction of
|
|
2842
|
+
* "advanced parameters", will ignore any external sequence producer setting.
|
|
2843
|
+
*
|
|
2844
|
+
* The sequence producer can be "cleared" by registering a NULL function pointer. This
|
|
2845
|
+
* removes all limitations described above in the "LIMITATIONS" section of the API docs.
|
|
2846
|
+
*
|
|
2847
|
+
* The user is strongly encouraged to read the full API documentation (above) before
|
|
2848
|
+
* calling this function. */
|
|
2849
|
+
ZSTDLIB_STATIC_API void
|
|
2850
|
+
ZSTD_registerSequenceProducer(
|
|
2851
|
+
ZSTD_CCtx* cctx,
|
|
2852
|
+
void* sequenceProducerState,
|
|
2853
|
+
ZSTD_sequenceProducer_F sequenceProducer
|
|
2854
|
+
);
|
|
2855
|
+
|
|
2856
|
+
/*! ZSTD_CCtxParams_registerSequenceProducer() :
|
|
2857
|
+
* Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params.
|
|
2858
|
+
* This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(),
|
|
2859
|
+
* which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx().
|
|
2860
|
+
*
|
|
2861
|
+
* If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx()
|
|
2862
|
+
* is required, then this function is for you. Otherwise, you probably don't need it.
|
|
2863
|
+
*
|
|
2864
|
+
* See tests/zstreamtest.c for example usage. */
|
|
2865
|
+
ZSTDLIB_STATIC_API void
|
|
2866
|
+
ZSTD_CCtxParams_registerSequenceProducer(
|
|
2867
|
+
ZSTD_CCtx_params* params,
|
|
2868
|
+
void* sequenceProducerState,
|
|
2869
|
+
ZSTD_sequenceProducer_F sequenceProducer
|
|
2870
|
+
);
|
|
2184
2871
|
|
|
2185
2872
|
|
|
2186
2873
|
/*********************************************************************
|
|
2187
|
-
* Buffer-less and synchronous inner streaming functions
|
|
2874
|
+
* Buffer-less and synchronous inner streaming functions (DEPRECATED)
|
|
2875
|
+
*
|
|
2876
|
+
* This API is deprecated, and will be removed in a future version.
|
|
2877
|
+
* It allows streaming (de)compression with user allocated buffers.
|
|
2878
|
+
* However, it is hard to use, and not as well tested as the rest of
|
|
2879
|
+
* our API.
|
|
2188
2880
|
*
|
|
2189
|
-
*
|
|
2190
|
-
*
|
|
2191
|
-
*
|
|
2881
|
+
* Please use the normal streaming API instead: ZSTD_compressStream2,
|
|
2882
|
+
* and ZSTD_decompressStream.
|
|
2883
|
+
* If there is functionality that you need, but it doesn't provide,
|
|
2884
|
+
* please open an issue on our GitHub.
|
|
2192
2885
|
********************************************************************* */
|
|
2193
2886
|
|
|
2194
2887
|
/**
|
|
@@ -2196,12 +2889,10 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
|
|
|
2196
2889
|
|
|
2197
2890
|
A ZSTD_CCtx object is required to track streaming operations.
|
|
2198
2891
|
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
|
|
2199
|
-
ZSTD_CCtx object can be
|
|
2892
|
+
ZSTD_CCtx object can be reused multiple times within successive compression operations.
|
|
2200
2893
|
|
|
2201
2894
|
Start by initializing a context.
|
|
2202
|
-
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression
|
|
2203
|
-
or ZSTD_compressBegin_advanced(), for finer parameter control.
|
|
2204
|
-
It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
|
|
2895
|
+
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
|
|
2205
2896
|
|
|
2206
2897
|
Then, consume your input using ZSTD_compressContinue().
|
|
2207
2898
|
There are some important considerations to keep in mind when using this advanced function :
|
|
@@ -2219,34 +2910,46 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
|
|
|
2219
2910
|
It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
|
|
2220
2911
|
Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
|
|
2221
2912
|
|
|
2222
|
-
`ZSTD_CCtx` object can be
|
|
2913
|
+
`ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again.
|
|
2223
2914
|
*/
|
|
2224
2915
|
|
|
2225
2916
|
/*===== Buffer-less streaming compression functions =====*/
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2917
|
+
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
|
|
2918
|
+
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
|
2919
|
+
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
|
|
2920
|
+
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
|
2921
|
+
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
|
|
2922
|
+
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
|
|
2923
|
+
|
|
2924
|
+
ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
|
|
2925
|
+
ZSTDLIB_STATIC_API
|
|
2926
|
+
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
|
|
2927
|
+
|
|
2928
|
+
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
|
|
2929
|
+
ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
2930
|
+
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
|
|
2931
|
+
ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
2932
|
+
|
|
2933
|
+
/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
|
|
2934
|
+
ZSTD_DEPRECATED("use advanced API to access custom parameters")
|
|
2935
|
+
ZSTDLIB_STATIC_API
|
|
2936
|
+
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
|
|
2937
|
+
ZSTD_DEPRECATED("use advanced API to access custom parameters")
|
|
2938
|
+
ZSTDLIB_STATIC_API
|
|
2939
|
+
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
|
|
2237
2940
|
/**
|
|
2238
2941
|
Buffer-less streaming decompression (synchronous mode)
|
|
2239
2942
|
|
|
2240
2943
|
A ZSTD_DCtx object is required to track streaming operations.
|
|
2241
2944
|
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
|
2242
|
-
A ZSTD_DCtx object can be
|
|
2945
|
+
A ZSTD_DCtx object can be reused multiple times.
|
|
2243
2946
|
|
|
2244
2947
|
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
|
|
2245
2948
|
Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
|
|
2246
2949
|
Data fragment must be large enough to ensure successful decoding.
|
|
2247
2950
|
`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
|
|
2248
|
-
|
|
2249
|
-
>0 : `srcSize` is too small, please provide at least
|
|
2951
|
+
result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
|
|
2952
|
+
>0 : `srcSize` is too small, please provide at least result bytes on next attempt.
|
|
2250
2953
|
errorCode, which can be tested using ZSTD_isError().
|
|
2251
2954
|
|
|
2252
2955
|
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
|
|
@@ -2265,7 +2968,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
|
|
2265
2968
|
|
|
2266
2969
|
The most memory efficient way is to use a round buffer of sufficient size.
|
|
2267
2970
|
Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
|
|
2268
|
-
which can
|
|
2971
|
+
which can return an error code if required value is too large for current system (in 32-bits mode).
|
|
2269
2972
|
In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
|
|
2270
2973
|
up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
|
|
2271
2974
|
which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
|
|
@@ -2285,7 +2988,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
|
|
2285
2988
|
ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
|
2286
2989
|
ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
|
|
2287
2990
|
|
|
2288
|
-
|
|
2991
|
+
result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
|
|
2289
2992
|
It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
|
|
2290
2993
|
It can also be an error code, which can be tested with ZSTD_isError().
|
|
2291
2994
|
|
|
@@ -2308,49 +3011,42 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
|
|
2308
3011
|
*/
|
|
2309
3012
|
|
|
2310
3013
|
/*===== Buffer-less streaming decompression functions =====*/
|
|
2311
|
-
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
|
|
2312
|
-
typedef struct {
|
|
2313
|
-
unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
|
|
2314
|
-
unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
|
|
2315
|
-
unsigned blockSizeMax;
|
|
2316
|
-
ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
|
|
2317
|
-
unsigned headerSize;
|
|
2318
|
-
unsigned dictID;
|
|
2319
|
-
unsigned checksumFlag;
|
|
2320
|
-
} ZSTD_frameHeader;
|
|
2321
3014
|
|
|
2322
|
-
|
|
2323
|
-
* decode Frame Header, or requires larger `srcSize`.
|
|
2324
|
-
* @return : 0, `zfhPtr` is correctly filled,
|
|
2325
|
-
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
|
2326
|
-
* or an error code, which can be tested using ZSTD_isError() */
|
|
2327
|
-
ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
|
2328
|
-
/*! ZSTD_getFrameHeader_advanced() :
|
|
2329
|
-
* same as ZSTD_getFrameHeader(),
|
|
2330
|
-
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
|
|
2331
|
-
ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
|
|
2332
|
-
ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
|
|
3015
|
+
ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
|
|
2333
3016
|
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
3017
|
+
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
|
3018
|
+
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
|
3019
|
+
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
|
2337
3020
|
|
|
2338
|
-
|
|
2339
|
-
|
|
3021
|
+
ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
|
3022
|
+
ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
2340
3023
|
|
|
2341
3024
|
/* misc */
|
|
2342
|
-
|
|
3025
|
+
ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
|
|
3026
|
+
ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
|
2343
3027
|
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
|
|
2344
|
-
|
|
3028
|
+
ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
|
2345
3029
|
|
|
2346
3030
|
|
|
2347
3031
|
|
|
2348
3032
|
|
|
2349
|
-
/*
|
|
2350
|
-
/** Block level API */
|
|
2351
|
-
/*
|
|
3033
|
+
/* ========================================= */
|
|
3034
|
+
/** Block level API (DEPRECATED) */
|
|
3035
|
+
/* ========================================= */
|
|
2352
3036
|
|
|
2353
3037
|
/*!
|
|
3038
|
+
|
|
3039
|
+
This API is deprecated in favor of the regular compression API.
|
|
3040
|
+
You can get the frame header down to 2 bytes by setting:
|
|
3041
|
+
- ZSTD_c_format = ZSTD_f_zstd1_magicless
|
|
3042
|
+
- ZSTD_c_contentSizeFlag = 0
|
|
3043
|
+
- ZSTD_c_checksumFlag = 0
|
|
3044
|
+
- ZSTD_c_dictIDFlag = 0
|
|
3045
|
+
|
|
3046
|
+
This API is not as well tested as our normal API, so we recommend not using it.
|
|
3047
|
+
We will be removing it in a future version. If the normal API doesn't provide
|
|
3048
|
+
the functionality you need, please open a GitHub issue.
|
|
3049
|
+
|
|
2354
3050
|
Block functions produce and decode raw zstd blocks, without frame metadata.
|
|
2355
3051
|
Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
|
2356
3052
|
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
|
|
@@ -2361,7 +3057,6 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
|
|
2361
3057
|
- It is necessary to init context before starting
|
|
2362
3058
|
+ compression : any ZSTD_compressBegin*() variant, including with dictionary
|
|
2363
3059
|
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
|
|
2364
|
-
+ copyCCtx() and copyDCtx() can be used too
|
|
2365
3060
|
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
|
|
2366
3061
|
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
|
|
2367
3062
|
+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.
|
|
@@ -2378,11 +3073,14 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
|
|
|
2378
3073
|
*/
|
|
2379
3074
|
|
|
2380
3075
|
/*===== Raw zstd block functions =====*/
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
3076
|
+
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
|
|
3077
|
+
ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
|
|
3078
|
+
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
|
|
3079
|
+
ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
3080
|
+
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
|
|
3081
|
+
ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
3082
|
+
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
|
|
3083
|
+
ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
|
|
2386
3084
|
|
|
2387
3085
|
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
|
|
2388
3086
|
|