extzstd 0.1.1 → 0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +18 -0
  3. data/README.md +15 -50
  4. data/contrib/zstd/CONTRIBUTING.md +1 -1
  5. data/contrib/zstd/COPYING +339 -0
  6. data/contrib/zstd/Makefile +82 -51
  7. data/contrib/zstd/NEWS +92 -5
  8. data/contrib/zstd/README.md +50 -41
  9. data/contrib/zstd/appveyor.yml +164 -102
  10. data/contrib/zstd/circle.yml +10 -22
  11. data/contrib/zstd/lib/BUCK +31 -10
  12. data/contrib/zstd/lib/Makefile +57 -31
  13. data/contrib/zstd/lib/README.md +68 -37
  14. data/contrib/zstd/lib/common/bitstream.h +130 -76
  15. data/contrib/zstd/lib/common/compiler.h +86 -0
  16. data/contrib/zstd/lib/common/error_private.c +15 -11
  17. data/contrib/zstd/lib/common/error_private.h +8 -8
  18. data/contrib/zstd/lib/common/fse.h +19 -9
  19. data/contrib/zstd/lib/common/fse_decompress.c +3 -22
  20. data/contrib/zstd/lib/common/huf.h +68 -26
  21. data/contrib/zstd/lib/common/mem.h +23 -35
  22. data/contrib/zstd/lib/common/pool.c +123 -63
  23. data/contrib/zstd/lib/common/pool.h +19 -10
  24. data/contrib/zstd/lib/common/threading.c +11 -16
  25. data/contrib/zstd/lib/common/threading.h +52 -33
  26. data/contrib/zstd/lib/common/xxhash.c +28 -22
  27. data/contrib/zstd/lib/common/zstd_common.c +40 -27
  28. data/contrib/zstd/lib/common/zstd_errors.h +43 -34
  29. data/contrib/zstd/lib/common/zstd_internal.h +131 -123
  30. data/contrib/zstd/lib/compress/fse_compress.c +17 -33
  31. data/contrib/zstd/lib/compress/huf_compress.c +15 -9
  32. data/contrib/zstd/lib/compress/zstd_compress.c +2096 -2363
  33. data/contrib/zstd/lib/compress/zstd_compress_internal.h +462 -0
  34. data/contrib/zstd/lib/compress/zstd_double_fast.c +309 -0
  35. data/contrib/zstd/lib/compress/zstd_double_fast.h +29 -0
  36. data/contrib/zstd/lib/compress/zstd_fast.c +243 -0
  37. data/contrib/zstd/lib/compress/zstd_fast.h +31 -0
  38. data/contrib/zstd/lib/compress/zstd_lazy.c +765 -0
  39. data/contrib/zstd/lib/compress/zstd_lazy.h +39 -0
  40. data/contrib/zstd/lib/compress/zstd_ldm.c +707 -0
  41. data/contrib/zstd/lib/compress/zstd_ldm.h +68 -0
  42. data/contrib/zstd/lib/compress/zstd_opt.c +785 -0
  43. data/contrib/zstd/lib/compress/zstd_opt.h +19 -908
  44. data/contrib/zstd/lib/compress/zstdmt_compress.c +737 -327
  45. data/contrib/zstd/lib/compress/zstdmt_compress.h +88 -26
  46. data/contrib/zstd/lib/decompress/huf_decompress.c +158 -50
  47. data/contrib/zstd/lib/decompress/zstd_decompress.c +884 -699
  48. data/contrib/zstd/lib/deprecated/zbuff.h +5 -4
  49. data/contrib/zstd/lib/deprecated/zbuff_common.c +5 -5
  50. data/contrib/zstd/lib/deprecated/zbuff_compress.c +6 -4
  51. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +5 -4
  52. data/contrib/zstd/lib/dictBuilder/cover.c +93 -77
  53. data/contrib/zstd/lib/dictBuilder/zdict.c +107 -92
  54. data/contrib/zstd/lib/dictBuilder/zdict.h +112 -102
  55. data/contrib/zstd/lib/legacy/zstd_legacy.h +9 -4
  56. data/contrib/zstd/lib/legacy/zstd_v01.c +7 -6
  57. data/contrib/zstd/lib/legacy/zstd_v01.h +5 -4
  58. data/contrib/zstd/lib/legacy/zstd_v02.c +27 -99
  59. data/contrib/zstd/lib/legacy/zstd_v02.h +5 -4
  60. data/contrib/zstd/lib/legacy/zstd_v03.c +26 -98
  61. data/contrib/zstd/lib/legacy/zstd_v03.h +5 -4
  62. data/contrib/zstd/lib/legacy/zstd_v04.c +22 -91
  63. data/contrib/zstd/lib/legacy/zstd_v04.h +5 -4
  64. data/contrib/zstd/lib/legacy/zstd_v05.c +23 -99
  65. data/contrib/zstd/lib/legacy/zstd_v05.h +5 -4
  66. data/contrib/zstd/lib/legacy/zstd_v06.c +22 -96
  67. data/contrib/zstd/lib/legacy/zstd_v06.h +5 -4
  68. data/contrib/zstd/lib/legacy/zstd_v07.c +19 -95
  69. data/contrib/zstd/lib/legacy/zstd_v07.h +5 -4
  70. data/contrib/zstd/lib/zstd.h +895 -271
  71. data/ext/extconf.rb +11 -2
  72. data/ext/extzstd.c +45 -128
  73. data/ext/extzstd.h +74 -31
  74. data/ext/extzstd_stream.c +401 -142
  75. data/ext/zstd_common.c +5 -0
  76. data/ext/zstd_compress.c +8 -0
  77. data/ext/zstd_decompress.c +1 -0
  78. data/ext/zstd_dictbuilder.c +2 -0
  79. data/lib/extzstd/version.rb +1 -1
  80. data/lib/extzstd.rb +48 -1
  81. data/test/test_basic.rb +9 -1
  82. metadata +17 -7
  83. data/HISTORY.ja +0 -10
  84. data/contrib/zstd/LICENSE-examples +0 -11
  85. data/contrib/zstd/PATENTS +0 -33
@@ -2,11 +2,11 @@
2
2
  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
-
10
10
  #if defined (__cplusplus)
11
11
  extern "C" {
12
12
  #endif
@@ -19,10 +19,12 @@ extern "C" {
19
19
 
20
20
 
21
21
  /* ===== ZSTDLIB_API : control library symbols visibility ===== */
22
- #if defined(__GNUC__) && (__GNUC__ >= 4)
23
- # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
24
- #else
25
- # define ZSTDLIB_VISIBILITY
22
+ #ifndef ZSTDLIB_VISIBILITY
23
+ # if defined(__GNUC__) && (__GNUC__ >= 4)
24
+ # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
25
+ # else
26
+ # define ZSTDLIB_VISIBILITY
27
+ # endif
26
28
  #endif
27
29
  #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
28
30
  # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
@@ -36,130 +38,149 @@ extern "C" {
36
38
  /*******************************************************************************************************
37
39
  Introduction
38
40
 
39
- zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
40
- at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
41
- decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
41
+ zstd, short for Zstandard, is a fast lossless compression algorithm,
42
+ targeting real-time compression scenarios at zlib-level and better compression ratios.
43
+ The zstd compression library provides in-memory compression and decompression functions.
44
+ The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
42
45
  Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
43
46
  Compression can be done in:
44
47
  - a single step (described as Simple API)
45
48
  - a single step, reusing a context (described as Explicit memory management)
46
49
  - unbounded multiple steps (described as Streaming compression)
47
- The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
50
+ The compression ratio achievable on small data can be highly improved using a dictionary in:
48
51
  - a single step (described as Simple dictionary API)
49
52
  - a single step, reusing a dictionary (described as Fast dictionary API)
50
53
 
51
54
  Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
52
- These APIs shall never be used with a dynamic library.
55
+ Advanced experimental APIs shall never be used with a dynamic library.
53
56
  They are not "stable", their definition may change in the future. Only static linking is allowed.
54
57
  *********************************************************************************************************/
55
58
 
56
59
  /*------ Version ------*/
57
60
  #define ZSTD_VERSION_MAJOR 1
58
- #define ZSTD_VERSION_MINOR 1
59
- #define ZSTD_VERSION_RELEASE 4
61
+ #define ZSTD_VERSION_MINOR 3
62
+ #define ZSTD_VERSION_RELEASE 3
63
+
64
+ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
65
+ ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */
60
66
 
61
67
  #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
62
68
  #define ZSTD_QUOTE(str) #str
63
69
  #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
64
70
  #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
65
-
66
- #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
67
- ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< library version number; to be used when checking dll version */
71
+ ZSTDLIB_API const char* ZSTD_versionString(void); /* v1.3.0 */
68
72
 
69
73
 
70
74
  /***************************************
71
75
  * Simple API
72
76
  ***************************************/
73
77
  /*! ZSTD_compress() :
74
- Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
75
- Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
76
- @return : compressed size written into `dst` (<= `dstCapacity),
77
- or an error code if it fails (which can be tested using ZSTD_isError()). */
78
+ * Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
79
+ * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
80
+ * @return : compressed size written into `dst` (<= `dstCapacity),
81
+ * or an error code if it fails (which can be tested using ZSTD_isError()). */
78
82
  ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
79
83
  const void* src, size_t srcSize,
80
84
  int compressionLevel);
81
85
 
82
86
  /*! ZSTD_decompress() :
83
- `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
84
- `dstCapacity` is an upper bound of originalSize.
85
- If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
86
- @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
87
- or an errorCode if it fails (which can be tested using ZSTD_isError()). */
87
+ * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
88
+ * `dstCapacity` is an upper bound of originalSize to regenerate.
89
+ * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
90
+ * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
91
+ * or an errorCode if it fails (which can be tested using ZSTD_isError()). */
88
92
  ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
89
93
  const void* src, size_t compressedSize);
90
94
 
95
+ /*! ZSTD_getFrameContentSize() : v1.3.0
96
+ * `src` should point to the start of a ZSTD encoded frame.
97
+ * `srcSize` must be at least as large as the frame header.
98
+ * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
99
+ * @return : - decompressed size of the frame in `src`, if known
100
+ * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
101
+ * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
102
+ * note 1 : a 0 return value means the frame is valid but "empty".
103
+ * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
104
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
105
+ * In which case, it's necessary to use streaming mode to decompress data.
106
+ * Optionally, application can rely on some implicit limit,
107
+ * as ZSTD_decompress() only needs an upper bound of decompressed size.
108
+ * (For example, data could be necessarily cut into blocks <= 16 KB).
109
+ * note 3 : decompressed size is always present when compression is done with ZSTD_compress()
110
+ * note 4 : decompressed size can be very large (64-bits value),
111
+ * potentially larger than what local system can handle as a single memory segment.
112
+ * In which case, it's necessary to use streaming mode to decompress data.
113
+ * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
114
+ * Always ensure return value fits within application's authorized limits.
115
+ * Each application can set its own limits.
116
+ * note 6 : This function replaces ZSTD_getDecompressedSize() */
117
+ #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
118
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
119
+ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
120
+
91
121
  /*! ZSTD_getDecompressedSize() :
92
- * NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
93
- * ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
94
- * frame, but distinguishes empty frames from frames with an unknown size, or errors.
95
- *
96
- * Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple
97
- * concatenated frames in one buffer, and so is more general.
98
- * As a result however, it requires more computation and entire frames to be passed to it,
99
- * as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
100
- *
101
- * 'src' is the start of a zstd compressed frame.
102
- * @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
103
- * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
104
- * When `return==0`, data to decompress could be any size.
105
- * In which case, it's necessary to use streaming mode to decompress data.
106
- * Optionally, application can still use ZSTD_decompress() while relying on implied limits.
107
- * (For example, data may be necessarily cut into blocks <= 16 KB).
108
- * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
109
- * note 3 : decompressed size can be very large (64-bits value),
110
- * potentially larger than what local system can handle as a single memory segment.
111
- * In which case, it's necessary to use streaming mode to decompress data.
112
- * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
113
- * Always ensure result fits within application's authorized limits.
114
- * Each application can set its own limits.
115
- * note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
122
+ * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
123
+ * Both functions work the same way,
124
+ * but ZSTD_getDecompressedSize() blends
125
+ * "empty", "unknown" and "error" results in the same return value (0),
126
+ * while ZSTD_getFrameContentSize() distinguishes them.
127
+ *
128
+ * 'src' is the start of a zstd compressed frame.
129
+ * @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. */
116
130
  ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
117
131
 
118
132
 
119
133
  /*====== Helper functions ======*/
120
- ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
134
+ #define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
121
135
  ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
122
136
  ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
123
137
  ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
138
+ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
124
139
 
125
140
 
126
141
  /***************************************
127
142
  * Explicit memory management
128
143
  ***************************************/
129
144
  /*= Compression context
130
- * When compressing many times,
131
- * it is recommended to allocate a context just once, and re-use it for each successive compression operation.
132
- * This will make workload friendlier for system's memory.
133
- * Use one context per thread for parallel execution in multi-threaded environments. */
145
+ * When compressing many times,
146
+ * it is recommended to allocate a context just once, and re-use it for each successive compression operation.
147
+ * This will make workload friendlier for system's memory.
148
+ * Use one context per thread for parallel execution in multi-threaded environments. */
134
149
  typedef struct ZSTD_CCtx_s ZSTD_CCtx;
135
150
  ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
136
151
  ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
137
152
 
138
153
  /*! ZSTD_compressCCtx() :
139
- Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
140
- ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
154
+ * Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
155
+ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx,
156
+ void* dst, size_t dstCapacity,
157
+ const void* src, size_t srcSize,
158
+ int compressionLevel);
141
159
 
142
160
  /*= Decompression context
143
- * When decompressing many times,
144
- * it is recommended to allocate a context just once, and re-use it for each successive compression operation.
145
- * This will make workload friendlier for system's memory.
146
- * Use one context per thread for parallel execution in multi-threaded environments. */
161
+ * When decompressing many times,
162
+ * it is recommended to allocate a context only once,
163
+ * and re-use it for each successive compression operation.
164
+ * This will make workload friendlier for system's memory.
165
+ * Use one context per thread for parallel execution. */
147
166
  typedef struct ZSTD_DCtx_s ZSTD_DCtx;
148
167
  ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
149
168
  ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
150
169
 
151
170
  /*! ZSTD_decompressDCtx() :
152
- * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */
153
- ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
171
+ * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
172
+ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
173
+ void* dst, size_t dstCapacity,
174
+ const void* src, size_t srcSize);
154
175
 
155
176
 
156
177
  /**************************
157
178
  * Simple dictionary API
158
179
  ***************************/
159
180
  /*! ZSTD_compress_usingDict() :
160
- * Compression using a predefined Dictionary (see dictBuilder/zdict.h).
161
- * Note : This function loads the dictionary, resulting in significant startup delay.
162
- * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
181
+ * Compression using a predefined Dictionary (see dictBuilder/zdict.h).
182
+ * Note : This function loads the dictionary, resulting in significant startup delay.
183
+ * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
163
184
  ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
164
185
  void* dst, size_t dstCapacity,
165
186
  const void* src, size_t srcSize,
@@ -167,36 +188,38 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
167
188
  int compressionLevel);
168
189
 
169
190
  /*! ZSTD_decompress_usingDict() :
170
- * Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
171
- * Dictionary must be identical to the one used during compression.
172
- * Note : This function loads the dictionary, resulting in significant startup delay.
173
- * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
191
+ * Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
192
+ * Dictionary must be identical to the one used during compression.
193
+ * Note : This function loads the dictionary, resulting in significant startup delay.
194
+ * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
174
195
  ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
175
196
  void* dst, size_t dstCapacity,
176
197
  const void* src, size_t srcSize,
177
198
  const void* dict,size_t dictSize);
178
199
 
179
200
 
180
- /****************************
181
- * Fast dictionary API
182
- ****************************/
201
+ /**********************************
202
+ * Bulk processing dictionary API
203
+ *********************************/
183
204
  typedef struct ZSTD_CDict_s ZSTD_CDict;
184
205
 
185
206
  /*! ZSTD_createCDict() :
186
- * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
187
- * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
188
- * ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
189
- * `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
190
- ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
207
+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
208
+ * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
209
+ * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
210
+ * `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict */
211
+ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
212
+ int compressionLevel);
191
213
 
192
214
  /*! ZSTD_freeCDict() :
193
- * Function frees memory allocated by ZSTD_createCDict(). */
215
+ * Function frees memory allocated by ZSTD_createCDict(). */
194
216
  ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
195
217
 
196
218
  /*! ZSTD_compress_usingCDict() :
197
- * Compression using a digested Dictionary.
198
- * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
199
- * Note that compression level is decided during dictionary creation. */
219
+ * Compression using a digested Dictionary.
220
+ * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
221
+ * Note that compression level is decided during dictionary creation.
222
+ * Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
200
223
  ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
201
224
  void* dst, size_t dstCapacity,
202
225
  const void* src, size_t srcSize,
@@ -206,17 +229,17 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
206
229
  typedef struct ZSTD_DDict_s ZSTD_DDict;
207
230
 
208
231
  /*! ZSTD_createDDict() :
209
- * Create a digested dictionary, ready to start decompression operation without startup delay.
210
- * dictBuffer can be released after DDict creation, as its content is copied inside DDict */
232
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
233
+ * dictBuffer can be released after DDict creation, as its content is copied inside DDict */
211
234
  ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
212
235
 
213
236
  /*! ZSTD_freeDDict() :
214
- * Function frees memory allocated with ZSTD_createDDict() */
237
+ * Function frees memory allocated with ZSTD_createDDict() */
215
238
  ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
216
239
 
217
240
  /*! ZSTD_decompress_usingDDict() :
218
- * Decompression using a digested Dictionary.
219
- * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
241
+ * Decompression using a digested Dictionary.
242
+ * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
220
243
  ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
221
244
  void* dst, size_t dstCapacity,
222
245
  const void* src, size_t srcSize,
@@ -273,14 +296,17 @@ typedef struct ZSTD_outBuffer_s {
273
296
  * ZSTD_endStream() instructs to finish a frame.
274
297
  * It will perform a flush and write frame epilogue.
275
298
  * The epilogue is required for decoders to consider a frame completed.
276
- * Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
299
+ * ZSTD_endStream() may not be able to flush full data if `output->size` is too small.
277
300
  * In which case, call again ZSTD_endStream() to complete the flush.
278
- * @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
301
+ * @return : 0 if frame fully completed and fully flushed,
302
+ or >0 if some data is still present within internal buffer
303
+ (value is minimum size estimation for remaining data to flush, but it could be more)
279
304
  * or an error code, which can be tested using ZSTD_isError().
280
305
  *
281
306
  * *******************************************************************/
282
307
 
283
- typedef struct ZSTD_CStream_s ZSTD_CStream;
308
+ typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
309
+ /* Continue to distinguish them for compatibility with versions <= v1.2.0 */
284
310
  /*===== ZSTD_CStream management functions =====*/
285
311
  ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
286
312
  ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
@@ -318,7 +344,8 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
318
344
  * The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
319
345
  * *******************************************************************************/
320
346
 
321
- typedef struct ZSTD_DStream_s ZSTD_DStream;
347
+ typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
348
+ /* Continue to distinguish them for compatibility with versions <= v1.2.0 */
322
349
  /*===== ZSTD_DStream management functions =====*/
323
350
  ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
324
351
  ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
@@ -333,50 +360,54 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
333
360
  #endif /* ZSTD_H_235446 */
334
361
 
335
362
 
336
- #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
337
- #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
338
363
 
339
364
  /****************************************************************************************
340
365
  * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
341
366
  * The definitions in this section are considered experimental.
342
- * They should never be used with a dynamic library, as they may change in the future.
343
- * They are provided for advanced usages.
367
+ * They should never be used with a dynamic library, as prototypes may change in the future.
368
+ * They are provided for advanced scenarios.
344
369
  * Use them only in association with static linking.
345
370
  * ***************************************************************************************/
346
371
 
372
+ #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
373
+ #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
374
+
347
375
  /* --- Constants ---*/
348
376
  #define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
349
377
  #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
378
+ #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
350
379
 
351
- #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
352
- #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
353
-
354
- #define ZSTD_WINDOWLOG_MAX_32 27
355
- #define ZSTD_WINDOWLOG_MAX_64 27
380
+ #define ZSTD_WINDOWLOG_MAX_32 30
381
+ #define ZSTD_WINDOWLOG_MAX_64 31
356
382
  #define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
357
- #define ZSTD_WINDOWLOG_MIN 10
358
- #define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
359
- #define ZSTD_HASHLOG_MIN 6
360
- #define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
361
- #define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
362
- #define ZSTD_HASHLOG3_MAX 17
363
- #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
364
- #define ZSTD_SEARCHLOG_MIN 1
365
- #define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
366
- #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
367
- #define ZSTD_TARGETLENGTH_MIN 4
368
- #define ZSTD_TARGETLENGTH_MAX 999
369
-
370
- #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
371
- #define ZSTD_FRAMEHEADERSIZE_MIN 6
372
- static const size_t ZSTD_frameHeaderSize_prefix = 5;
383
+ #define ZSTD_WINDOWLOG_MIN 10
384
+ #define ZSTD_HASHLOG_MAX MIN(ZSTD_WINDOWLOG_MAX, 30)
385
+ #define ZSTD_HASHLOG_MIN 6
386
+ #define ZSTD_CHAINLOG_MAX MIN(ZSTD_WINDOWLOG_MAX+1, 30)
387
+ #define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
388
+ #define ZSTD_HASHLOG3_MAX 17
389
+ #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
390
+ #define ZSTD_SEARCHLOG_MIN 1
391
+ #define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
392
+ #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
393
+ #define ZSTD_TARGETLENGTH_MIN 4 /* only useful for btopt */
394
+ #define ZSTD_TARGETLENGTH_MAX 999 /* only useful for btopt */
395
+ #define ZSTD_LDM_MINMATCH_MIN 4
396
+ #define ZSTD_LDM_MINMATCH_MAX 4096
397
+ #define ZSTD_LDM_BUCKETSIZELOG_MAX 8
398
+
399
+ #define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size to know frame header size */
400
+ #define ZSTD_FRAMEHEADERSIZE_MIN 6
401
+ #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
402
+ static const size_t ZSTD_frameHeaderSize_prefix = ZSTD_FRAMEHEADERSIZE_PREFIX;
373
403
  static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
374
404
  static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
375
405
  static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
376
406
 
377
407
 
378
408
  /*--- Advanced types ---*/
379
- typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */
409
+ typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
410
+ ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */
380
411
 
381
412
  typedef struct {
382
413
  unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -399,84 +430,145 @@ typedef struct {
399
430
  ZSTD_frameParameters fParams;
400
431
  } ZSTD_parameters;
401
432
 
402
- /*= Custom memory allocation functions */
433
+ typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
434
+
435
+ /*--- Custom memory allocation functions ---*/
403
436
  typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
404
437
  typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
405
438
  typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
439
+ /* use this constant to defer to stdlib's functions */
440
+ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };
441
+
406
442
 
407
443
  /***************************************
408
- * Compressed size functions
444
+ * Frame size functions
409
445
  ***************************************/
410
446
 
411
447
  /*! ZSTD_findFrameCompressedSize() :
412
448
  * `src` should point to the start of a ZSTD encoded frame or skippable frame
413
- * `srcSize` must be at least as large as the frame
414
- * @return : the compressed size of the frame pointed to by `src`, suitable to pass to
415
- * `ZSTD_decompress` or similar, or an error code if given invalid input. */
449
+ * `srcSize` must be >= first frame size
450
+ * @return : the compressed size of the first frame starting at `src`,
451
+ * suitable to pass to `ZSTD_decompress` or similar,
452
+ * or an error code if input is invalid */
416
453
  ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
417
454
 
455
+ /*! ZSTD_findDecompressedSize() :
456
+ * `src` should point the start of a series of ZSTD encoded and/or skippable frames
457
+ * `srcSize` must be the _exact_ size of this series
458
+ * (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`)
459
+ * @return : - decompressed size of all data in all successive frames
460
+ * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
461
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
462
+ *
463
+ * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
464
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
465
+ * In which case, it's necessary to use streaming mode to decompress data.
466
+ * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
467
+ * note 3 : decompressed size can be very large (64-bits value),
468
+ * potentially larger than what local system can handle as a single memory segment.
469
+ * In which case, it's necessary to use streaming mode to decompress data.
470
+ * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
471
+ * Always ensure result fits within application's authorized limits.
472
+ * Each application can set its own limits.
473
+ * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
474
+ * read each contained frame header. This is fast as most of the data is skipped,
475
+ * however it does mean that all frame data must be present and valid. */
476
+ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
477
+
478
+ /*! ZSTD_frameHeaderSize() :
479
+ * `src` should point to the start of a ZSTD frame
480
+ * `srcSize` must be >= ZSTD_frameHeaderSize_prefix.
481
+ * @return : size of the Frame Header */
482
+ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
483
+
484
+
418
485
  /***************************************
419
- * Decompressed size functions
486
+ * Context memory usage
420
487
  ***************************************/
421
- /*! ZSTD_getFrameContentSize() :
422
- * `src` should point to the start of a ZSTD encoded frame
423
- * `srcSize` must be at least as large as the frame header. A value greater than or equal
424
- * to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
425
- * @return : decompressed size of the frame pointed to be `src` if known, otherwise
426
- * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
427
- * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
428
- ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
429
488
 
430
- /*! ZSTD_findDecompressedSize() :
431
- * `src` should point the start of a series of ZSTD encoded and/or skippable frames
432
- * `srcSize` must be the _exact_ size of this series
433
- * (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
434
- * @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
435
- * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
436
- * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
437
- *
438
- * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
439
- * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
440
- * In which case, it's necessary to use streaming mode to decompress data.
441
- * Optionally, application can still use ZSTD_decompress() while relying on implied limits.
442
- * (For example, data may be necessarily cut into blocks <= 16 KB).
443
- * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
444
- * note 3 : decompressed size can be very large (64-bits value),
445
- * potentially larger than what local system can handle as a single memory segment.
446
- * In which case, it's necessary to use streaming mode to decompress data.
447
- * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
448
- * Always ensure result fits within application's authorized limits.
449
- * Each application can set its own limits.
450
- * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
451
- * read each contained frame header. This is efficient as most of the data is skipped,
452
- * however it does mean that all frame data must be present and valid. */
453
- ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
489
+ /*! ZSTD_sizeof_*() :
490
+ * These functions give the current memory usage of selected object.
491
+ * Object memory usage can evolve when re-used multiple times. */
492
+ ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
493
+ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
494
+ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
495
+ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
496
+ ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
497
+ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
498
+
499
+ /*! ZSTD_estimate*() :
500
+ * These functions make it possible to estimate memory usage
501
+ * of a future {D,C}Ctx, before its creation.
502
+ * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
503
+ * It will also consider src size to be arbitrarily "large", which is worst case.
504
+ * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
505
+ * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
506
+ * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1.
507
+ * Note : CCtx estimation is only correct for single-threaded compression */
508
+ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
509
+ ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
510
+ ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
511
+ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
512
+
513
+ /*! ZSTD_estimateCStreamSize() :
514
+ * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
515
+ * It will also consider src size to be arbitrarily "large", which is worst case.
516
+ * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
517
+ * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
518
+ * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1.
519
+ * Note : CStream estimation is only correct for single-threaded compression.
520
+ * ZSTD_DStream memory budget depends on window Size.
521
+ * This information can be passed manually, using ZSTD_estimateDStreamSize,
522
+ * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
523
+ * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
524
+ * an internal ?Dict will be created, which additional size is not estimated here.
525
+ * In this case, get total size by adding ZSTD_estimate?DictSize */
526
+ ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
527
+ ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
528
+ ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
529
+ ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
530
+ ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
531
+
532
+ typedef enum {
533
+ ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
534
+ ZSTD_dlm_byRef, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
535
+ } ZSTD_dictLoadMethod_e;
536
+
537
+ /*! ZSTD_estimate?DictSize() :
538
+ * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
539
+ * ZSTD_estimateCStreamSize_advanced_usingCParams() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
540
+ * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller
541
+ */
542
+ ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
543
+ ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
544
+ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
454
545
 
455
546
 
456
547
  /***************************************
457
548
  * Advanced compression functions
458
549
  ***************************************/
459
- /*! ZSTD_estimateCCtxSize() :
460
- * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
461
- * `frameContentSize` is an optional parameter, provide `0` if unknown */
462
- ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
463
-
464
550
  /*! ZSTD_createCCtx_advanced() :
465
551
  * Create a ZSTD compression context using external alloc and free functions */
466
552
  ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
467
553
 
468
- /*! ZSTD_sizeofCCtx() :
469
- * Gives the amount of memory used by a given ZSTD_CCtx */
470
- ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
554
+ /*! ZSTD_initStaticCCtx() : initialize a fixed-size zstd compression context
555
+ * workspace: The memory area to emplace the context into.
556
+ * Provided pointer must 8-bytes aligned.
557
+ * It must outlive context usage.
558
+ * workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
559
+ * to determine how large workspace must be to support scenario.
560
+ * @return : pointer to ZSTD_CCtx* (same address as workspace, but different type),
561
+ * or NULL if error (typically size too small)
562
+ * Note : zstd will never resize nor malloc() when using a static cctx.
563
+ * If it needs more memory than available, it will simply error out.
564
+ * Note 2 : there is no corresponding "free" function.
565
+ * Since workspace was allocated externally, it must be freed externally too.
566
+ * Limitation 1 : currently not compatible with internal CDict creation, such as
567
+ * ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
568
+ * Limitation 2 : currently not compatible with multi-threading
569
+ */
570
+ ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
471
571
 
472
- typedef enum {
473
- ZSTD_p_forceWindow, /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
474
- ZSTD_p_forceRawDict /* Force loading dictionary in "content-only" mode (no header analysis) */
475
- } ZSTD_CCtxParameter;
476
- /*! ZSTD_setCCtxParameter() :
477
- * Set advanced parameters, selected through enum ZSTD_CCtxParameter
478
- * @result : 0, or an error code (which can be tested with ZSTD_isError()) */
479
- ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
480
572
 
481
573
  /*! ZSTD_createCDict_byReference() :
482
574
  * Create a digested dictionary for compression
@@ -484,14 +576,37 @@ ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter par
484
576
  * It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
485
577
  ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
486
578
 
579
+ typedef enum { ZSTD_dm_auto=0, /* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
580
+ ZSTD_dm_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
581
+ ZSTD_dm_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */
582
+ } ZSTD_dictMode_e;
487
583
  /*! ZSTD_createCDict_advanced() :
488
584
  * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
489
- ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
490
- ZSTD_parameters params, ZSTD_customMem customMem);
491
-
492
- /*! ZSTD_sizeof_CDict() :
493
- * Gives the amount of memory used by a given ZSTD_sizeof_CDict */
494
- ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
585
+ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
586
+ ZSTD_dictLoadMethod_e dictLoadMethod,
587
+ ZSTD_dictMode_e dictMode,
588
+ ZSTD_compressionParameters cParams,
589
+ ZSTD_customMem customMem);
590
+
591
+ /*! ZSTD_initStaticCDict() :
592
+ * Generate a digested dictionary in provided memory area.
593
+ * workspace: The memory area to emplace the dictionary into.
594
+ * Provided pointer must 8-bytes aligned.
595
+ * It must outlive dictionary usage.
596
+ * workspaceSize: Use ZSTD_estimateCDictSize()
597
+ * to determine how large workspace must be.
598
+ * cParams : use ZSTD_getCParams() to transform a compression level
599
+ * into its relevants cParams.
600
+ * @return : pointer to ZSTD_CDict* (same address as workspace, but different type),
601
+ * or NULL if error (typically, size too small).
602
+ * Note : there is no corresponding "free" function.
603
+ * Since workspace was allocated externally, it must be freed externally.
604
+ */
605
+ ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict(
606
+ void* workspace, size_t workspaceSize,
607
+ const void* dict, size_t dictSize,
608
+ ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode,
609
+ ZSTD_compressionParameters cParams);
495
610
 
496
611
  /*! ZSTD_getCParams() :
497
612
  * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
@@ -500,7 +615,7 @@ ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, uns
500
615
 
501
616
  /*! ZSTD_getParams() :
502
617
  * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
503
- * All fields of `ZSTD_frameParameters` are set to default (0) */
618
+ * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
504
619
  ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
505
620
 
506
621
  /*! ZSTD_checkCParams() :
@@ -508,17 +623,24 @@ ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long l
508
623
  ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
509
624
 
510
625
  /*! ZSTD_adjustCParams() :
511
- * optimize params for a given `srcSize` and `dictSize`.
512
- * both values are optional, select `0` if unknown. */
626
+ * optimize params for a given `srcSize` and `dictSize`.
627
+ * both values are optional, select `0` if unknown. */
513
628
  ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
514
629
 
515
630
  /*! ZSTD_compress_advanced() :
516
- * Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
517
- ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
518
- void* dst, size_t dstCapacity,
519
- const void* src, size_t srcSize,
520
- const void* dict,size_t dictSize,
521
- ZSTD_parameters params);
631
+ * Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter */
632
+ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
633
+ void* dst, size_t dstCapacity,
634
+ const void* src, size_t srcSize,
635
+ const void* dict,size_t dictSize,
636
+ ZSTD_parameters params);
637
+
638
+ /*! ZSTD_compress_usingCDict_advanced() :
639
+ * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
640
+ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
641
+ void* dst, size_t dstCapacity,
642
+ const void* src, size_t srcSize,
643
+ const ZSTD_CDict* cdict, ZSTD_frameParameters fParams);
522
644
 
523
645
 
524
646
  /*--- Advanced decompression functions ---*/
@@ -530,32 +652,55 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
530
652
  * Note 3 : Skippable Frame Identifiers are considered valid. */
531
653
  ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
532
654
 
533
- /*! ZSTD_estimateDCtxSize() :
534
- * Gives the potential amount of memory allocated to create a ZSTD_DCtx */
535
- ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
536
-
537
655
  /*! ZSTD_createDCtx_advanced() :
538
656
  * Create a ZSTD decompression context using external alloc and free functions */
539
657
  ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
540
658
 
541
- /*! ZSTD_sizeof_DCtx() :
542
- * Gives the amount of memory used by a given ZSTD_DCtx */
543
- ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
659
+ /*! ZSTD_initStaticDCtx() : initialize a fixed-size zstd decompression context
660
+ * workspace: The memory area to emplace the context into.
661
+ * Provided pointer must 8-bytes aligned.
662
+ * It must outlive context usage.
663
+ * workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
664
+ * to determine how large workspace must be to support scenario.
665
+ * @return : pointer to ZSTD_DCtx* (same address as workspace, but different type),
666
+ * or NULL if error (typically size too small)
667
+ * Note : zstd will never resize nor malloc() when using a static dctx.
668
+ * If it needs more memory than available, it will simply error out.
669
+ * Note 2 : static dctx is incompatible with legacy support
670
+ * Note 3 : there is no corresponding "free" function.
671
+ * Since workspace was allocated externally, it must be freed externally.
672
+ * Limitation : currently not compatible with internal DDict creation,
673
+ * such as ZSTD_initDStream_usingDict().
674
+ */
675
+ ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
544
676
 
545
677
  /*! ZSTD_createDDict_byReference() :
546
678
  * Create a digested dictionary, ready to start decompression operation without startup delay.
547
- * Dictionary content is simply referenced, and therefore stays in dictBuffer.
548
- * It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
679
+ * Dictionary content is referenced, and therefore stays in dictBuffer.
680
+ * It is important that dictBuffer outlives DDict,
681
+ * it must remain read accessible throughout the lifetime of DDict */
549
682
  ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
550
683
 
551
684
  /*! ZSTD_createDDict_advanced() :
552
685
  * Create a ZSTD_DDict using external alloc and free, optionally by reference */
553
686
  ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
554
- unsigned byReference, ZSTD_customMem customMem);
555
-
556
- /*! ZSTD_sizeof_DDict() :
557
- * Gives the amount of memory used by a given ZSTD_DDict */
558
- ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
687
+ ZSTD_dictLoadMethod_e dictLoadMethod,
688
+ ZSTD_customMem customMem);
689
+
690
+ /*! ZSTD_initStaticDDict() :
691
+ * Generate a digested dictionary in provided memory area.
692
+ * workspace: The memory area to emplace the dictionary into.
693
+ * Provided pointer must 8-bytes aligned.
694
+ * It must outlive dictionary usage.
695
+ * workspaceSize: Use ZSTD_estimateDDictSize()
696
+ * to determine how large workspace must be.
697
+ * @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
698
+ * Note : there is no corresponding "free" function.
699
+ * Since workspace was allocated externally, it must be freed externally.
700
+ */
701
+ ZSTDLIB_API ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
702
+ const void* dict, size_t dictSize,
703
+ ZSTD_dictLoadMethod_e dictLoadMethod);
559
704
 
560
705
  /*! ZSTD_getDictID_fromDict() :
561
706
  * Provides the dictID stored within dictionary.
@@ -578,7 +723,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
578
723
  * Note : this use case also happens when using a non-conformant dictionary.
579
724
  * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
580
725
  * - This is not a Zstandard frame.
581
- * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
726
+ * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
582
727
  ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
583
728
 
584
729
 
@@ -588,31 +733,42 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
588
733
 
589
734
  /*===== Advanced Streaming compression functions =====*/
590
735
  ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
591
- ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
592
- ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
736
+ ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
737
+ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
738
+ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
593
739
  ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
594
- ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
740
+ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
595
741
  ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
596
- ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
597
- ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
742
+ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */
743
+
744
+ /*! ZSTD_resetCStream() :
745
+ * start a new compression job, using same parameters from previous job.
746
+ * This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
747
+ * Note that zcs must be init at least once before using ZSTD_resetCStream().
748
+ * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
749
+ * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
750
+ * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
751
+ * but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
752
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
753
+ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
598
754
 
599
755
 
600
756
  /*===== Advanced Streaming decompression functions =====*/
601
- typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
602
757
  ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
603
- ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
604
- ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
605
- ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
758
+ ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
759
+ typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
760
+ ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); /* obsolete : this API will be removed in a future version */
761
+ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
762
+ ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */
606
763
  ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
607
- ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
608
764
 
609
765
 
610
766
  /*********************************************************************
611
767
  * Buffer-less and synchronous inner streaming functions
612
768
  *
613
769
  * This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
614
- * But it's also a complex one, with many restrictions (documented below).
615
- * Prefer using normal streaming API for an easier experience
770
+ * But it's also a complex one, with several restrictions, documented below.
771
+ * Prefer normal streaming API for an easier experience.
616
772
  ********************************************************************* */
617
773
 
618
774
  /**
@@ -629,8 +785,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
629
785
 
630
786
  Then, consume your input using ZSTD_compressContinue().
631
787
  There are some important considerations to keep in mind when using this advanced function :
632
- - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
633
- - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks.
788
+ - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
789
+ - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
634
790
  - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
635
791
  Worst case evaluation is provided by ZSTD_compressBound().
636
792
  ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
@@ -641,22 +797,23 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
641
797
 
642
798
  Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
643
799
  It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
644
- Without last block mark, frames will be considered unfinished (corrupted) by decoders.
800
+ Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
645
801
 
646
- `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
802
+ `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
647
803
  */
648
804
 
649
805
  /*===== Buffer-less streaming compression functions =====*/
650
806
  ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
651
807
  ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
652
- ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
653
- ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
654
- ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
808
+ ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
809
+ ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
810
+ ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
811
+ ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
812
+
655
813
  ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
656
814
  ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
657
815
 
658
816
 
659
-
660
817
  /*-
661
818
  Buffer-less streaming decompression (synchronous mode)
662
819
 
@@ -664,38 +821,54 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
664
821
  Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
665
822
  A ZSTD_DCtx object can be re-used multiple times.
666
823
 
667
- First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
668
- It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
669
- such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
670
- and the dictionary ID used.
671
- (Note : content size is optional, it may not be present. 0 means : content size unknown).
672
- Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
673
- As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
674
- Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
675
- Frame parameters are extracted from the beginning of the compressed frame.
676
- Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
677
- @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
824
+ First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
825
+ Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
826
+ Data fragment must be large enough to ensure successful decoding.
827
+ `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
828
+ @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
678
829
  >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
679
830
  errorCode, which can be tested using ZSTD_isError().
680
831
 
681
- Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
682
- Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
832
+ It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
833
+ such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
834
+ Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
835
+ As a consequence, check that values remain within valid application range.
836
+ For example, do not allocate memory blindly, check that `windowSize` is within expectation.
837
+ Each application can set its own limits, depending on local restrictions.
838
+ For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
839
+
840
+ ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
841
+ ZSTD_decompressContinue() is very sensitive to contiguity,
842
+ if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
843
+ or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
844
+ There are multiple ways to guarantee this condition.
845
+
846
+ The most memory efficient way is to use a round buffer of sufficient size.
847
+ Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
848
+ which can @return an error code if required value is too large for current system (in 32-bits mode).
849
+ In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
850
+ up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
851
+ which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
852
+ At which point, decoding can resume from the beginning of the buffer.
853
+ Note that already decoded data stored in the buffer should be flushed before being overwritten.
854
+
855
+ There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
856
+
857
+ Finally, if you control the compression process, you can also ignore all buffer size rules,
858
+ as long as the encoder and decoder progress in "lock-step",
859
+ aka use exactly the same buffer sizes, break contiguity at the same place, etc.
860
+
861
+ Once buffers are setup, start decompression, with ZSTD_decompressBegin().
862
+ If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
683
863
 
684
864
  Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
685
865
  ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
686
866
  ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
687
867
 
688
- @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
689
- It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some metadata item.
868
+ @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
869
+ It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
690
870
  It can also be an error code, which can be tested with ZSTD_isError().
691
871
 
692
- ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
693
- They should preferably be located contiguously, prior to current block.
694
- Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters.
695
- ZSTD_decompressContinue() is very sensitive to contiguity,
696
- if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
697
- or that previous contiguous segment is large enough to properly handle maximum back-reference.
698
-
699
872
  A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
700
873
  Context can then be reset to start a new decompression.
701
874
 
@@ -705,38 +878,487 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
705
878
  == Special case : skippable frames ==
706
879
 
707
880
  Skippable frames allow integration of user-defined data into a flow of concatenated frames.
708
- Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
881
+ Skippable frames will be ignored (skipped) by decompressor.
882
+ The format of skippable frames is as follows :
709
883
  a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
710
884
  b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
711
885
  c) Frame Content - any content (User Data) of length equal to Frame Size
712
- For skippable frames ZSTD_decompressContinue() always returns 0.
713
- For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
714
- Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
715
- For purposes of decompression, it is valid in both cases to skip the frame using
716
- ZSTD_findFrameCompressedSize to find its size in bytes.
717
- It also returns Frame Size as fparamsPtr->frameContentSize.
886
+ For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
887
+ For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
718
888
  */
719
889
 
890
+ /*===== Buffer-less streaming decompression functions =====*/
891
+ typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
720
892
  typedef struct {
721
- unsigned long long frameContentSize;
722
- unsigned windowSize;
893
+ unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
894
+ unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
895
+ unsigned blockSizeMax;
896
+ ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
897
+ unsigned headerSize;
723
898
  unsigned dictID;
724
899
  unsigned checksumFlag;
725
- } ZSTD_frameParams;
900
+ } ZSTD_frameHeader;
901
+ ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
902
+ ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
726
903
 
727
- /*===== Buffer-less streaming decompression functions =====*/
728
- ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
729
904
  ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
730
905
  ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
731
- ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
906
+ ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
907
+
732
908
  ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
733
909
  ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
910
+
911
+ /* misc */
912
+ ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
734
913
  typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
735
914
  ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
736
915
 
737
- /**
738
- Block functions
739
916
 
917
+
918
+ /* ============================================ */
919
+ /** New advanced API (experimental) */
920
+ /* ============================================ */
921
+
922
+ /* notes on API design :
923
+ * In this proposal, parameters are pushed one by one into an existing context,
924
+ * and then applied on all subsequent compression jobs.
925
+ * When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT.
926
+ *
927
+ * This API is intended to replace all others experimental API.
928
+ * It can basically do all other use cases, and even new ones.
929
+ * In constrast with _advanced() variants, it stands a reasonable chance to become "stable",
930
+ * after a good testing period.
931
+ */
932
+
933
+ /* note on naming convention :
934
+ * Initially, the API favored names like ZSTD_setCCtxParameter() .
935
+ * In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
936
+ * The main driver is that it identifies more clearly the target object type.
937
+ * It feels clearer when considering multiple targets :
938
+ * ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
939
+ * ZSTD_CCtxParams_setParameter() (rather than ZSTD_setCCtxParamsParameter() )
940
+ * etc...
941
+ */
942
+
943
+ /* note on enum design :
944
+ * All enum will be pinned to explicit values before reaching "stable API" status */
945
+
946
+ typedef enum {
947
+ /* Question : should we have a format ZSTD_f_auto ?
948
+ * For the time being, it would mean exactly the same as ZSTD_f_zstd1.
949
+ * But, in the future, should several formats be supported,
950
+ * on the compression side, it would mean "default format".
951
+ * On the decompression side, it would mean "multi format",
952
+ * and ZSTD_f_zstd1 could be reserved to mean "accept *only* zstd frames".
953
+ * Since meaning is a little different, another option could be to define different enums for compression and decompression.
954
+ * This question could be kept for later, when there are actually multiple formats to support,
955
+ * but there is also the question of pinning enum values, and pinning value `0` is especially important */
956
+ ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
957
+ ZSTD_f_zstd1_magicless, /* Variant of zstd frame format, without initial 4-bytes magic number.
958
+ * Useful to save 4 bytes per generated frame.
959
+ * Decoder cannot recognise automatically this format, requiring instructions. */
960
+ } ZSTD_format_e;
961
+
962
+ typedef enum {
963
+ /* compression format */
964
+ ZSTD_p_format = 10, /* See ZSTD_format_e enum definition.
965
+ * Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
966
+
967
+ /* compression parameters */
968
+ ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
969
+ * Default level is ZSTD_CLEVEL_DEFAULT==3.
970
+ * Special: value 0 means "do not change cLevel". */
971
+ ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
972
+ * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
973
+ * Special: value 0 means "do not change windowLog".
974
+ * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
975
+ * requires setting the maximum window size at least as large during decompression. */
976
+ ZSTD_p_hashLog, /* Size of the probe table, as a power of 2.
977
+ * Resulting table size is (1 << (hashLog+2)).
978
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
979
+ * Larger tables improve compression ratio of strategies <= dFast,
980
+ * and improve speed of strategies > dFast.
981
+ * Special: value 0 means "do not change hashLog". */
982
+ ZSTD_p_chainLog, /* Size of the full-search table, as a power of 2.
983
+ * Resulting table size is (1 << (chainLog+2)).
984
+ * Larger tables result in better and slower compression.
985
+ * This parameter is useless when using "fast" strategy.
986
+ * Special: value 0 means "do not change chainLog". */
987
+ ZSTD_p_searchLog, /* Number of search attempts, as a power of 2.
988
+ * More attempts result in better and slower compression.
989
+ * This parameter is useless when using "fast" and "dFast" strategies.
990
+ * Special: value 0 means "do not change searchLog". */
991
+ ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller).
992
+ * Larger values make faster compression and decompression, but decrease ratio.
993
+ * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
994
+ * Note that currently, for all strategies < btopt, effective minimum is 4.
995
+ * Note that currently, for all strategies > fast, effective maximum is 6.
996
+ * Special: value 0 means "do not change minMatchLength". */
997
+ ZSTD_p_targetLength, /* Only useful for strategies >= btopt.
998
+ * Length of Match considered "good enough" to stop search.
999
+ * Larger values make compression stronger and slower.
1000
+ * Special: value 0 means "do not change targetLength". */
1001
+ ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
1002
+ * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
1003
+ * The higher the value of selected strategy, the more complex it is,
1004
+ * resulting in stronger and slower compression.
1005
+ * Special: value 0 means "do not change strategy". */
1006
+
1007
+ /* frame parameters */
1008
+ ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
1009
+ * Content size must be known at the beginning of compression,
1010
+ * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
1011
+ ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
1012
+ ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */
1013
+
1014
+ /* multi-threading parameters */
1015
+ ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
1016
+ * More threads improve speed, but also increase memory usage.
1017
+ * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
1018
+ * Special: value 0 means "do not change nbThreads" */
1019
+ ZSTD_p_jobSize, /* Size of a compression job. This value is only enforced in streaming (non-blocking) mode.
1020
+ * Each compression job is completed in parallel, so indirectly controls the nb of active threads.
1021
+ * 0 means default, which is dynamically determined based on compression parameters.
1022
+ * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
1023
+ * The minimum size is automatically and transparently enforced */
1024
+ ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
1025
+ * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
1026
+
1027
+ /* advanced parameters - may not remain available after API update */
1028
+ ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
1029
+ * even when referencing into Dictionary content (default:0) */
1030
+ ZSTD_p_enableLongDistanceMatching=1200, /* Enable long distance matching.
1031
+ * This parameter is designed to improve the compression
1032
+ * ratio for large inputs with long distance matches.
1033
+ * This increases the memory usage as well as window size.
1034
+ * Note: setting this parameter sets all the LDM parameters
1035
+ * as well as ZSTD_p_windowLog. It should be set after
1036
+ * ZSTD_p_compressionLevel and before ZSTD_p_windowLog and
1037
+ * other LDM parameters. Setting the compression level
1038
+ * after this parameter overrides the window log, though LDM
1039
+ * will remain enabled until explicitly disabled. */
1040
+ ZSTD_p_ldmHashLog, /* Size of the table for long distance matching, as a power of 2.
1041
+ * Larger values increase memory usage and compression ratio, but decrease
1042
+ * compression speed.
1043
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
1044
+ * (default: windowlog - 7). */
1045
+ ZSTD_p_ldmMinMatch, /* Minimum size of searched matches for long distance matcher.
1046
+ * Larger/too small values usually decrease compression ratio.
1047
+ * Must be clamped between ZSTD_LDM_MINMATCH_MIN
1048
+ * and ZSTD_LDM_MINMATCH_MAX (default: 64). */
1049
+ ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the LDM hash table for collision resolution.
1050
+ * Larger values usually improve collision resolution but may decrease
1051
+ * compression speed.
1052
+ * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX (default: 3). */
1053
+ ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table.
1054
+ * The default is MAX(0, (windowLog - ldmHashLog)) to
1055
+ * optimize hash table usage.
1056
+ * Larger values improve compression speed. Deviating far from the
1057
+ * default value will likely result in a decrease in compression ratio.
1058
+ * Must be clamped between 0 and ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN. */
1059
+
1060
+ } ZSTD_cParameter;
1061
+
1062
+
1063
+ /*! ZSTD_CCtx_setParameter() :
1064
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
1065
+ * Note : when `value` is an enum, cast it to unsigned for proper type checking.
1066
+ * @result : informational value (typically, the one being set, possibly corrected),
1067
+ * or an error code (which can be tested with ZSTD_isError()). */
1068
+ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
1069
+
1070
+ /*! ZSTD_CCtx_setPledgedSrcSize() :
1071
+ * Total input data size to be compressed as a single frame.
1072
+ * This value will be controlled at the end, and result in error if not respected.
1073
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1074
+ * Note 1 : 0 means zero, empty.
1075
+ * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
1076
+ * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
1077
+ * Note 2 : If all data is provided and consumed in a single round,
1078
+ * this value is overriden by srcSize instead. */
1079
+ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
1080
+
1081
+ /*! ZSTD_CCtx_loadDictionary() :
1082
+ * Create an internal CDict from dict buffer.
1083
+ * Decompression will have to use same buffer.
1084
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1085
+ * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
1086
+ * meaning "return to no-dictionary mode".
1087
+ * Note 1 : `dict` content will be copied internally. Use
1088
+ * ZSTD_CCtx_loadDictionary_byReference() to reference dictionary
1089
+ * content instead. The dictionary buffer must then outlive its
1090
+ * users.
1091
+ * Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
1092
+ * For this reason, compression parameters cannot be changed anymore after loading a dictionary.
1093
+ * It's also a CPU-heavy operation, with non-negligible impact on latency.
1094
+ * Note 3 : Dictionary will be used for all future compression jobs.
1095
+ * To return to "no-dictionary" situation, load a NULL dictionary
1096
+ * Note 5 : Use ZSTD_CCtx_loadDictionary_advanced() to select how dictionary
1097
+ * content will be interpreted.
1098
+ */
1099
+ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
1100
+ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
1101
+ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);
1102
+
1103
+
1104
+ /*! ZSTD_CCtx_refCDict() :
1105
+ * Reference a prepared dictionary, to be used for all next compression jobs.
1106
+ * Note that compression parameters are enforced from within CDict,
1107
+ * and supercede any compression parameter previously set within CCtx.
1108
+ * The dictionary will remain valid for future compression jobs using same CCtx.
1109
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1110
+ * Special : adding a NULL CDict means "return to no-dictionary mode".
1111
+ * Note 1 : Currently, only one dictionary can be managed.
1112
+ * Adding a new dictionary effectively "discards" any previous one.
1113
+ * Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
1114
+ */
1115
+ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
1116
+
1117
+ /*! ZSTD_CCtx_refPrefix() :
1118
+ * Reference a prefix (single-usage dictionary) for next compression job.
1119
+ * Decompression need same prefix to properly regenerate data.
1120
+ * Prefix is **only used once**. Tables are discarded at end of compression job.
1121
+ * Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
1122
+ * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
1123
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1124
+ * Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
1125
+ * Note 1 : Prefix buffer is referenced. It must outlive compression job.
1126
+ * Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
1127
+ * It's a CPU-heavy operation, with non-negligible impact on latency.
1128
+ * Note 3 : By default, the prefix is treated as raw content
1129
+ * (ZSTD_dm_rawContent). Use ZSTD_CCtx_refPrefix_advanced() to alter
1130
+ * dictMode. */
1131
+ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
1132
+ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode);
1133
+
1134
+
1135
+
1136
+ typedef enum {
1137
+ ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
1138
+ ZSTD_e_flush, /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
1139
+ ZSTD_e_end /* flush any remaining data and close current frame. Any additional data starts a new frame. */
1140
+ } ZSTD_EndDirective;
1141
+
1142
+ /*! ZSTD_compress_generic() :
1143
+ * Behave about the same as ZSTD_compressStream. To note :
1144
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
1145
+ * - Compression parameters cannot be changed once compression is started.
1146
+ * - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
1147
+ * - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
1148
+ * - In single-thread mode (default), function is blocking : it completed its job before returning to caller.
1149
+ * - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
1150
+ * and then immediately returns, just indicating that there is some data remaining to be flushed.
1151
+ * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
1152
+ * - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
1153
+ * - @return provides the minimum amount of data remaining to be flushed from internal buffers
1154
+ * or an error code, which can be tested using ZSTD_isError().
1155
+ * if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
1156
+ * This is useful to determine if a ZSTD_e_flush or ZSTD_e_end directive is completed.
1157
+ * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
1158
+ * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
1159
+ * Before starting a new compression job, or changing compression parameters,
1160
+ * it is required to fully flush internal buffers.
1161
+ */
1162
+ ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
1163
+ ZSTD_outBuffer* output,
1164
+ ZSTD_inBuffer* input,
1165
+ ZSTD_EndDirective endOp);
1166
+
1167
+ /*! ZSTD_CCtx_reset() :
1168
+ * Return a CCtx to clean state.
1169
+ * Useful after an error, or to interrupt an ongoing compression job and start a new one.
1170
+ * Any internal data not yet flushed is cancelled.
1171
+ * Dictionary (if any) is dropped.
1172
+ * All parameters are back to default values.
1173
+ * It's possible to modify compression parameters after a reset.
1174
+ */
1175
+ ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); /* Not ready yet ! */
1176
+
1177
+
1178
+ /*! ZSTD_compress_generic_simpleArgs() :
1179
+ * Same as ZSTD_compress_generic(),
1180
+ * but using only integral types as arguments.
1181
+ * Argument list is larger than ZSTD_{in,out}Buffer,
1182
+ * but can be helpful for binders from dynamic languages
1183
+ * which have troubles handling structures containing memory pointers.
1184
+ */
1185
+ ZSTDLIB_API size_t ZSTD_compress_generic_simpleArgs (
1186
+ ZSTD_CCtx* cctx,
1187
+ void* dst, size_t dstCapacity, size_t* dstPos,
1188
+ const void* src, size_t srcSize, size_t* srcPos,
1189
+ ZSTD_EndDirective endOp);
1190
+
1191
+
1192
+ /*! ZSTD_CCtx_params :
1193
+ * Quick howto :
1194
+ * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
1195
+ * - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
1196
+ * an existing ZSTD_CCtx_params structure.
1197
+ * This is similar to
1198
+ * ZSTD_CCtx_setParameter().
1199
+ * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
1200
+ * an existing CCtx.
1201
+ * These parameters will be applied to
1202
+ * all subsequent compression jobs.
1203
+ * - ZSTD_compress_generic() : Do compression using the CCtx.
1204
+ * - ZSTD_freeCCtxParams() : Free the memory.
1205
+ *
1206
+ * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
1207
+ * for static allocation for single-threaded compression.
1208
+ */
1209
+ ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
1210
+
1211
+ /*! ZSTD_resetCCtxParams() :
1212
+ * Reset params to default, with the default compression level.
1213
+ */
1214
+ ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params);
1215
+
1216
+ /*! ZSTD_initCCtxParams() :
1217
+ * Initializes the compression parameters of cctxParams according to
1218
+ * compression level. All other parameters are reset to their default values.
1219
+ */
1220
+ ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, int compressionLevel);
1221
+
1222
+ /*! ZSTD_initCCtxParams_advanced() :
1223
+ * Initializes the compression and frame parameters of cctxParams according to
1224
+ * params. All other parameters are reset to their default values.
1225
+ */
1226
+ ZSTDLIB_API size_t ZSTD_initCCtxParams_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
1227
+
1228
+ ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
1229
+
1230
+ /*! ZSTD_CCtxParam_setParameter() :
1231
+ * Similar to ZSTD_CCtx_setParameter.
1232
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
1233
+ * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
1234
+ * Note : when `value` is an enum, cast it to unsigned for proper type checking.
1235
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1236
+ */
1237
+ ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
1238
+
1239
+ /*! ZSTD_CCtx_setParametersUsingCCtxParams() :
1240
+ * Apply a set of ZSTD_CCtx_params to the compression context.
1241
+ * This must be done before the dictionary is loaded.
1242
+ * The pledgedSrcSize is treated as unknown.
1243
+ * Multithreading parameters are applied only if nbThreads > 1.
1244
+ */
1245
+ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
1246
+ ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
1247
+
1248
+
1249
+ /*=== Advanced parameters for decompression API ===*/
1250
+
1251
+ /* The following parameters must be set after creating a ZSTD_DCtx* (or ZSTD_DStream*) object,
1252
+ * but before starting decompression of a frame.
1253
+ */
1254
+
1255
+ /*! ZSTD_DCtx_loadDictionary() :
1256
+ * Create an internal DDict from dict buffer,
1257
+ * to be used to decompress next frames.
1258
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1259
+ * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
1260
+ * meaning "return to no-dictionary mode".
1261
+ * Note 1 : `dict` content will be copied internally.
1262
+ * Use ZSTD_DCtx_loadDictionary_byReference()
1263
+ * to reference dictionary content instead.
1264
+ * In which case, the dictionary buffer must outlive its users.
1265
+ * Note 2 : Loading a dictionary involves building tables,
1266
+ * which has a non-negligible impact on CPU usage and latency.
1267
+ * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
1268
+ * how dictionary content will be interpreted and loaded.
1269
+ */
1270
+ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */
1271
+ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */
1272
+ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); /* not implemented */
1273
+
1274
+
1275
+ /*! ZSTD_DCtx_refDDict() :
1276
+ * Reference a prepared dictionary, to be used to decompress next frames.
1277
+ * The dictionary remains active for decompression of future frames using same DCtx.
1278
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1279
+ * Note 1 : Currently, only one dictionary can be managed.
1280
+ * Referencing a new dictionary effectively "discards" any previous one.
1281
+ * Special : adding a NULL DDict means "return to no-dictionary mode".
1282
+ * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
1283
+ */
1284
+ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); /* not implemented */
1285
+
1286
+
1287
+ /*! ZSTD_DCtx_refPrefix() :
1288
+ * Reference a prefix (single-usage dictionary) for next compression job.
1289
+ * Prefix is **only used once**. It must be explicitly referenced before each frame.
1290
+ * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead.
1291
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1292
+ * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
1293
+ * Note 2 : Prefix buffer is referenced. It must outlive compression job.
1294
+ * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
1295
+ * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
1296
+ * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
1297
+ */
1298
+ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); /* not implemented */
1299
+ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode); /* not implemented */
1300
+
1301
+
1302
+ /*! ZSTD_DCtx_setMaxWindowSize() :
1303
+ * Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
1304
+ * This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario).
1305
+ * This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode.
1306
+ * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX)
1307
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
1308
+ */
1309
+ ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
1310
+
1311
+
1312
+ /*! ZSTD_DCtx_setFormat() :
1313
+ * Instruct the decoder context about what kind of data to decode next.
1314
+ * This instruction is mandatory to decode data without a fully-formed header,
1315
+ * such ZSTD_f_zstd1_magicless for example.
1316
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
1317
+ */
1318
+ ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
1319
+
1320
+
1321
+ /*! ZSTD_decompress_generic() :
1322
+ * Behave the same as ZSTD_decompressStream.
1323
+ * Decompression parameters cannot be changed once decompression is started.
1324
+ * @return : an error code, which can be tested using ZSTD_isError()
1325
+ * if >0, a hint, nb of expected input bytes for next invocation.
1326
+ * `0` means : a frame has just been fully decoded and flushed.
1327
+ */
1328
+ ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx,
1329
+ ZSTD_outBuffer* output,
1330
+ ZSTD_inBuffer* input);
1331
+
1332
+
1333
+ /*! ZSTD_decompress_generic_simpleArgs() :
1334
+ * Same as ZSTD_decompress_generic(),
1335
+ * but using only integral types as arguments.
1336
+ * Argument list is larger than ZSTD_{in,out}Buffer,
1337
+ * but can be helpful for binders from dynamic languages
1338
+ * which have troubles handling structures containing memory pointers.
1339
+ */
1340
+ ZSTDLIB_API size_t ZSTD_decompress_generic_simpleArgs (
1341
+ ZSTD_DCtx* dctx,
1342
+ void* dst, size_t dstCapacity, size_t* dstPos,
1343
+ const void* src, size_t srcSize, size_t* srcPos);
1344
+
1345
+
1346
+ /*! ZSTD_DCtx_reset() :
1347
+ * Return a DCtx to clean state.
1348
+ * If a decompression was ongoing, any internal data not yet flushed is cancelled.
1349
+ * All parameters are back to default values, including sticky ones.
1350
+ * Dictionary (if any) is dropped.
1351
+ * Parameters can be modified again after a reset.
1352
+ */
1353
+ ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
1354
+
1355
+
1356
+
1357
+ /* ============================ */
1358
+ /** Block level API */
1359
+ /* ============================ */
1360
+
1361
+ /*!
740
1362
  Block functions produce and decode raw zstd blocks, without frame metadata.
741
1363
  Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
742
1364
  User will have to take in charge required information to regenerate data, such as compressed and content sizes.
@@ -745,27 +1367,29 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
745
1367
  - Compressing and decompressing require a context structure
746
1368
  + Use ZSTD_createCCtx() and ZSTD_createDCtx()
747
1369
  - It is necessary to init context before starting
748
- + compression : ZSTD_compressBegin()
749
- + decompression : ZSTD_decompressBegin()
750
- + variants _usingDict() are also allowed
751
- + copyCCtx() and copyDCtx() work too
752
- - Block size is limited, it must be <= ZSTD_getBlockSizeMax()
753
- + If you need to compress more, cut data into multiple blocks
754
- + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large.
1370
+ + compression : any ZSTD_compressBegin*() variant, including with dictionary
1371
+ + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
1372
+ + copyCCtx() and copyDCtx() can be used too
1373
+ - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
1374
+ + If input is larger than a block size, it's necessary to split input data into multiple blocks
1375
+ + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
1376
+ Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
755
1377
  - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
756
1378
  In which case, nothing is produced into `dst`.
757
1379
  + User must test for such outcome and deal directly with uncompressed data
758
1380
  + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
759
- + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history.
760
- Use ZSTD_insertBlock() in such a case.
1381
+ + In case of multiple successive blocks, should some of them be uncompressed,
1382
+ decoder must be informed of their existence in order to follow proper history.
1383
+ Use ZSTD_insertBlock() for such a case.
761
1384
  */
762
1385
 
763
- #define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
1386
+ #define ZSTD_BLOCKSIZELOG_MAX 17
1387
+ #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */
764
1388
  /*===== Raw zstd block functions =====*/
765
- ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
1389
+ ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
766
1390
  ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
767
1391
  ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
768
- ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
1392
+ ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression */
769
1393
 
770
1394
 
771
1395
  #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */