zstdlib 0.10.0 → 0.11.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +8 -0
  3. data/ext/zstdlib_c/extconf.rb +2 -2
  4. data/ext/zstdlib_c/ruby/zlib-3.2/zstdlib.c +5090 -0
  5. data/ext/zstdlib_c/zstd-1.5.5/lib/common/allocations.h +55 -0
  6. data/ext/zstdlib_c/zstd-1.5.5/lib/common/bits.h +200 -0
  7. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/bitstream.h +19 -60
  8. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/compiler.h +26 -3
  9. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/cpu.h +1 -1
  10. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/debug.c +1 -1
  11. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/debug.h +1 -1
  12. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/entropy_common.c +12 -40
  13. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/error_private.c +9 -2
  14. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/error_private.h +1 -1
  15. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/fse.h +5 -83
  16. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/fse_decompress.c +7 -99
  17. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/huf.h +65 -156
  18. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/mem.h +39 -46
  19. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/pool.c +26 -10
  20. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/pool.h +7 -1
  21. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/portability_macros.h +22 -3
  22. data/ext/zstdlib_c/zstd-1.5.5/lib/common/threading.c +176 -0
  23. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/threading.h +5 -10
  24. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/xxhash.c +2 -2
  25. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/xxhash.h +8 -8
  26. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/zstd_common.c +1 -36
  27. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/zstd_deps.h +1 -1
  28. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/zstd_internal.h +17 -118
  29. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/common/zstd_trace.h +3 -3
  30. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/clevels.h +1 -1
  31. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/fse_compress.c +7 -124
  32. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/hist.c +1 -1
  33. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/hist.h +1 -1
  34. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/huf_compress.c +234 -169
  35. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress.c +1243 -538
  36. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_internal.h +225 -151
  37. data/ext/zstdlib_c/zstd-1.5.5/lib/compress/zstd_compress_literals.c +235 -0
  38. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_literals.h +16 -8
  39. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_sequences.c +3 -3
  40. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_sequences.h +1 -1
  41. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_superblock.c +25 -21
  42. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_compress_superblock.h +1 -1
  43. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_cwksp.h +128 -62
  44. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_double_fast.c +95 -33
  45. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_double_fast.h +3 -2
  46. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_fast.c +433 -148
  47. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_fast.h +3 -2
  48. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_lazy.c +398 -345
  49. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_lazy.h +4 -2
  50. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_ldm.c +5 -5
  51. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_ldm.h +1 -1
  52. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_ldm_geartab.h +1 -1
  53. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_opt.c +106 -80
  54. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstd_opt.h +1 -1
  55. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstdmt_compress.c +17 -9
  56. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/compress/zstdmt_compress.h +1 -1
  57. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/huf_decompress.c +434 -441
  58. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/huf_decompress_amd64.S +30 -39
  59. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_ddict.c +4 -4
  60. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_ddict.h +1 -1
  61. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_decompress.c +205 -80
  62. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_decompress_block.c +201 -81
  63. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_decompress_block.h +6 -1
  64. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/decompress/zstd_decompress_internal.h +4 -2
  65. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/zdict.h +53 -31
  66. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/zstd.h +580 -135
  67. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/lib/zstd_errors.h +27 -8
  68. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzclose.c +1 -1
  69. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzcompatibility.h +8 -8
  70. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzguts.h +10 -10
  71. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzlib.c +3 -3
  72. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzread.c +10 -10
  73. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/gzwrite.c +5 -5
  74. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/zstd_zlibwrapper.c +46 -44
  75. data/ext/zstdlib_c/{zstd-1.5.2 → zstd-1.5.5}/zlibWrapper/zstd_zlibwrapper.h +4 -1
  76. metadata +77 -74
  77. data/ext/zstdlib_c/zstd-1.5.2/lib/common/threading.c +0 -122
  78. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_literals.c +0 -159
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,21 +14,31 @@ extern "C" {
14
14
  #ifndef ZSTD_H_235446
15
15
  #define ZSTD_H_235446
16
16
 
17
- /* ====== Dependency ======*/
17
+ /* ====== Dependencies ======*/
18
18
  #include <limits.h> /* INT_MAX */
19
19
  #include <stddef.h> /* size_t */
20
20
 
21
21
 
22
22
  /* ===== ZSTDLIB_API : control library symbols visibility ===== */
23
23
  #ifndef ZSTDLIB_VISIBLE
24
- # if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
24
+ /* Backwards compatibility with old macro name */
25
+ # ifdef ZSTDLIB_VISIBILITY
26
+ # define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
27
+ # elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
25
28
  # define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
26
- # define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
27
29
  # else
28
30
  # define ZSTDLIB_VISIBLE
31
+ # endif
32
+ #endif
33
+
34
+ #ifndef ZSTDLIB_HIDDEN
35
+ # if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
36
+ # define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
37
+ # else
29
38
  # define ZSTDLIB_HIDDEN
30
39
  # endif
31
40
  #endif
41
+
32
42
  #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
33
43
  # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
34
44
  #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
@@ -37,6 +47,28 @@ extern "C" {
37
47
  # define ZSTDLIB_API ZSTDLIB_VISIBLE
38
48
  #endif
39
49
 
50
+ /* Deprecation warnings :
51
+ * Should these warnings be a problem, it is generally possible to disable them,
52
+ * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
53
+ * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
54
+ */
55
+ #ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
56
+ # define ZSTD_DEPRECATED(message) /* disable deprecation warnings */
57
+ #else
58
+ # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
59
+ # define ZSTD_DEPRECATED(message) [[deprecated(message)]]
60
+ # elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
61
+ # define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
62
+ # elif defined(__GNUC__) && (__GNUC__ >= 3)
63
+ # define ZSTD_DEPRECATED(message) __attribute__((deprecated))
64
+ # elif defined(_MSC_VER)
65
+ # define ZSTD_DEPRECATED(message) __declspec(deprecated(message))
66
+ # else
67
+ # pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
68
+ # define ZSTD_DEPRECATED(message)
69
+ # endif
70
+ #endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
71
+
40
72
 
41
73
  /*******************************************************************************
42
74
  Introduction
@@ -74,7 +106,7 @@ extern "C" {
74
106
  /*------ Version ------*/
75
107
  #define ZSTD_VERSION_MAJOR 1
76
108
  #define ZSTD_VERSION_MINOR 5
77
- #define ZSTD_VERSION_RELEASE 2
109
+ #define ZSTD_VERSION_RELEASE 5
78
110
  #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
79
111
 
80
112
  /*! ZSTD_versionNumber() :
@@ -116,7 +148,8 @@ ZSTDLIB_API const char* ZSTD_versionString(void);
116
148
  ***************************************/
117
149
  /*! ZSTD_compress() :
118
150
  * Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
119
- * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
151
+ * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
152
+ * enough space to successfully compress the data.
120
153
  * @return : compressed size written into `dst` (<= `dstCapacity),
121
154
  * or an error code if it fails (which can be tested using ZSTD_isError()). */
122
155
  ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
@@ -165,7 +198,9 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
165
198
  * "empty", "unknown" and "error" results to the same return value (0),
166
199
  * while ZSTD_getFrameContentSize() gives them separate return values.
167
200
  * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
168
- ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
201
+ ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
202
+ ZSTDLIB_API
203
+ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
169
204
 
170
205
  /*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
171
206
  * `src` should point to the start of a ZSTD frame or skippable frame.
@@ -177,8 +212,30 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
177
212
 
178
213
 
179
214
  /*====== Helper functions ======*/
180
- #define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
181
- ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
215
+ /* ZSTD_compressBound() :
216
+ * maximum compressed size in worst case single-pass scenario.
217
+ * When invoking `ZSTD_compress()` or any other one-pass compression function,
218
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
219
+ * as it eliminates one potential failure scenario,
220
+ * aka not enough room in dst buffer to write the compressed frame.
221
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
222
+ * In which case, ZSTD_compressBound() will return an error code
223
+ * which can be tested using ZSTD_isError().
224
+ *
225
+ * ZSTD_COMPRESSBOUND() :
226
+ * same as ZSTD_compressBound(), but as a macro.
227
+ * It can be used to produce constants, which can be useful for static allocation,
228
+ * for example to size a static array on stack.
229
+ * Will produce constant value 0 if srcSize too large.
230
+ */
231
+ #define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
232
+ #define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
233
+ ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
234
+ /* ZSTD_isError() :
235
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
236
+ * using ZSTD_isError().
237
+ * @return 1 if error, 0 otherwise
238
+ */
182
239
  ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
183
240
  ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
184
241
  ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
@@ -421,6 +478,9 @@ typedef enum {
421
478
  * ZSTD_c_validateSequences
422
479
  * ZSTD_c_useBlockSplitter
423
480
  * ZSTD_c_useRowMatchFinder
481
+ * ZSTD_c_prefetchCDictTables
482
+ * ZSTD_c_enableSeqProducerFallback
483
+ * ZSTD_c_maxBlockSize
424
484
  * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
425
485
  * note : never ever use experimentalParam? names directly;
426
486
  * also, the enums values themselves are unstable and can still change.
@@ -439,7 +499,11 @@ typedef enum {
439
499
  ZSTD_c_experimentalParam12=1009,
440
500
  ZSTD_c_experimentalParam13=1010,
441
501
  ZSTD_c_experimentalParam14=1011,
442
- ZSTD_c_experimentalParam15=1012
502
+ ZSTD_c_experimentalParam15=1012,
503
+ ZSTD_c_experimentalParam16=1013,
504
+ ZSTD_c_experimentalParam17=1014,
505
+ ZSTD_c_experimentalParam18=1015,
506
+ ZSTD_c_experimentalParam19=1016
443
507
  } ZSTD_cParameter;
444
508
 
445
509
  typedef struct {
@@ -502,7 +566,7 @@ typedef enum {
502
566
  * They will be used to compress next frame.
503
567
  * Resetting session never fails.
504
568
  * - The parameters : changes all parameters back to "default".
505
- * This removes any reference to any dictionary too.
569
+ * This also removes any reference to any dictionary or external sequence producer.
506
570
  * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
507
571
  * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
508
572
  * - Both : similar to resetting the session, followed by resetting parameters.
@@ -515,7 +579,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
515
579
  * Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
516
580
  * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
517
581
  * - The function is always blocking, returns when compression is completed.
518
- * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
582
+ * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
583
+ * enough space to successfully compress the data, though it is possible it fails for other reasons.
519
584
  * @return : compressed size written into `dst` (<= `dstCapacity),
520
585
  * or an error code if it fails (which can be tested using ZSTD_isError()).
521
586
  */
@@ -552,13 +617,15 @@ typedef enum {
552
617
  * ZSTD_d_stableOutBuffer
553
618
  * ZSTD_d_forceIgnoreChecksum
554
619
  * ZSTD_d_refMultipleDDicts
620
+ * ZSTD_d_disableHuffmanAssembly
555
621
  * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
556
622
  * note : never ever use experimentalParam? names directly
557
623
  */
558
624
  ZSTD_d_experimentalParam1=1000,
559
625
  ZSTD_d_experimentalParam2=1001,
560
626
  ZSTD_d_experimentalParam3=1002,
561
- ZSTD_d_experimentalParam4=1003
627
+ ZSTD_d_experimentalParam4=1003,
628
+ ZSTD_d_experimentalParam5=1004
562
629
 
563
630
  } ZSTD_dParameter;
564
631
 
@@ -737,8 +804,6 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
737
804
  * This following is a legacy streaming API, available since v1.0+ .
738
805
  * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
739
806
  * It is redundant, but remains fully supported.
740
- * Streaming in combination with advanced parameters and dictionary compression
741
- * can only be used through the new API.
742
807
  ******************************************************************************/
743
808
 
744
809
  /*!
@@ -747,6 +812,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
747
812
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
748
813
  * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
749
814
  * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
815
+ *
816
+ * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
817
+ * to compress with a dictionary.
750
818
  */
751
819
  ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
752
820
  /*!
@@ -797,13 +865,31 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer
797
865
 
798
866
  /*===== Streaming decompression functions =====*/
799
867
 
800
- /* This function is redundant with the advanced API and equivalent to:
868
+ /*! ZSTD_initDStream() :
869
+ * Initialize/reset DStream state for new decompression operation.
870
+ * Call before new decompression operation using same DStream.
801
871
  *
872
+ * Note : This function is redundant with the advanced API and equivalent to:
802
873
  * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
803
874
  * ZSTD_DCtx_refDDict(zds, NULL);
804
875
  */
805
876
  ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
806
877
 
878
+ /*! ZSTD_decompressStream() :
879
+ * Streaming decompression function.
880
+ * Call repetitively to consume full input updating it as necessary.
881
+ * Function will update both input and output `pos` fields exposing current state via these fields:
882
+ * - `input.pos < input.size`, some input remaining and caller should provide remaining input
883
+ * on the next call.
884
+ * - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
885
+ * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
886
+ * call ZSTD_decompressStream() again to flush remaining data to output.
887
+ * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
888
+ *
889
+ * @return : 0 when a frame is completely decoded and fully flushed,
890
+ * or an error code, which can be tested using ZSTD_isError(),
891
+ * or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
892
+ */
807
893
  ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
808
894
 
809
895
  ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
@@ -922,7 +1008,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
922
1008
  * If @return == 0, the dictID could not be decoded.
923
1009
  * This could for one of the following reasons :
924
1010
  * - The frame does not require a dictionary to be decoded (most common case).
925
- * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
1011
+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information.
926
1012
  * Note : this use case also happens when using a non-conformant dictionary.
927
1013
  * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
928
1014
  * - This is not a Zstandard frame.
@@ -934,9 +1020,11 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
934
1020
  * Advanced dictionary and prefix API (Requires v1.4.0+)
935
1021
  *
936
1022
  * This API allows dictionaries to be used with ZSTD_compress2(),
937
- * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
938
- * only reset with the context is reset with ZSTD_reset_parameters or
939
- * ZSTD_reset_session_and_parameters. Prefixes are single-use.
1023
+ * ZSTD_compressStream2(), and ZSTD_decompressDCtx().
1024
+ * Dictionaries are sticky, they remain valid when same context is re-used,
1025
+ * they only reset when the context is reset
1026
+ * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters.
1027
+ * In contrast, Prefixes are single-use.
940
1028
  ******************************************************************************/
941
1029
 
942
1030
 
@@ -946,8 +1034,9 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
946
1034
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
947
1035
  * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
948
1036
  * meaning "return to no-dictionary mode".
949
- * Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
950
- * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
1037
+ * Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
1038
+ * until parameters are reset, a new dictionary is loaded, or the dictionary
1039
+ * is explicitly invalidated by loading a NULL dictionary.
951
1040
  * Note 2 : Loading a dictionary involves building tables.
952
1041
  * It's also a CPU consuming operation, with non-negligible impact on latency.
953
1042
  * Tables are dependent on compression parameters, and for this reason,
@@ -956,11 +1045,15 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
956
1045
  * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
957
1046
  * In such a case, dictionary buffer must outlive its users.
958
1047
  * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
959
- * to precisely select how dictionary content must be interpreted. */
1048
+ * to precisely select how dictionary content must be interpreted.
1049
+ * Note 5 : This method does not benefit from LDM (long distance mode).
1050
+ * If you want to employ LDM on some large dictionary content,
1051
+ * prefer employing ZSTD_CCtx_refPrefix() described below.
1052
+ */
960
1053
  ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
961
1054
 
962
1055
  /*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
963
- * Reference a prepared dictionary, to be used for all next compressed frames.
1056
+ * Reference a prepared dictionary, to be used for all future compressed frames.
964
1057
  * Note that compression parameters are enforced from within CDict,
965
1058
  * and supersede any compression parameter previously set within CCtx.
966
1059
  * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
@@ -979,6 +1072,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
979
1072
  * Decompression will need same prefix to properly regenerate data.
980
1073
  * Compressing with a prefix is similar in outcome as performing a diff and compressing it,
981
1074
  * but performs much faster, especially during decompression (compression speed is tunable with compression level).
1075
+ * This method is compatible with LDM (long distance mode).
982
1076
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
983
1077
  * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
984
1078
  * Note 1 : Prefix buffer is referenced. It **must** outlive compression.
@@ -995,9 +1089,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
995
1089
  const void* prefix, size_t prefixSize);
996
1090
 
997
1091
  /*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
998
- * Create an internal DDict from dict buffer,
999
- * to be used to decompress next frames.
1000
- * The dictionary remains valid for all future frames, until explicitly invalidated.
1092
+ * Create an internal DDict from dict buffer, to be used to decompress all future frames.
1093
+ * The dictionary remains valid for all future frames, until explicitly invalidated, or
1094
+ * a new dictionary is loaded.
1001
1095
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1002
1096
  * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
1003
1097
  * meaning "return to no-dictionary mode".
@@ -1021,9 +1115,10 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
1021
1115
  * The memory for the table is allocated on the first call to refDDict, and can be
1022
1116
  * freed with ZSTD_freeDCtx().
1023
1117
  *
1118
+ * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
1119
+ * will be managed, and referencing a dictionary effectively "discards" any previous one.
1120
+ *
1024
1121
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1025
- * Note 1 : Currently, only one dictionary can be managed.
1026
- * Referencing a new dictionary effectively "discards" any previous one.
1027
1122
  * Special: referencing a NULL DDict means "return to no-dictionary mode".
1028
1123
  * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
1029
1124
  */
@@ -1086,28 +1181,6 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
1086
1181
  # endif
1087
1182
  #endif
1088
1183
 
1089
- /* Deprecation warnings :
1090
- * Should these warnings be a problem, it is generally possible to disable them,
1091
- * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
1092
- * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
1093
- */
1094
- #ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
1095
- # define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API /* disable deprecation warnings */
1096
- #else
1097
- # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
1098
- # define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_STATIC_API
1099
- # elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
1100
- # define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated(message)))
1101
- # elif defined(__GNUC__) && (__GNUC__ >= 3)
1102
- # define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated))
1103
- # elif defined(_MSC_VER)
1104
- # define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __declspec(deprecated(message))
1105
- # else
1106
- # pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
1107
- # define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API
1108
- # endif
1109
- #endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
1110
-
1111
1184
  /****************************************************************************************
1112
1185
  * experimental API (static linking only)
1113
1186
  ****************************************************************************************
@@ -1142,6 +1215,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
1142
1215
  #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
1143
1216
  #define ZSTD_STRATEGY_MIN ZSTD_fast
1144
1217
  #define ZSTD_STRATEGY_MAX ZSTD_btultra2
1218
+ #define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
1145
1219
 
1146
1220
 
1147
1221
  #define ZSTD_OVERLAPLOG_MIN 0
@@ -1322,7 +1396,7 @@ typedef enum {
1322
1396
  } ZSTD_paramSwitch_e;
1323
1397
 
1324
1398
  /***************************************
1325
- * Frame size functions
1399
+ * Frame header and size functions
1326
1400
  ***************************************/
1327
1401
 
1328
1402
  /*! ZSTD_findDecompressedSize() :
@@ -1369,33 +1443,113 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size
1369
1443
  * or an error code (if srcSize is too small) */
1370
1444
  ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
1371
1445
 
1446
+ typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
1447
+ typedef struct {
1448
+ unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
1449
+ unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
1450
+ unsigned blockSizeMax;
1451
+ ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
1452
+ unsigned headerSize;
1453
+ unsigned dictID;
1454
+ unsigned checksumFlag;
1455
+ unsigned _reserved1;
1456
+ unsigned _reserved2;
1457
+ } ZSTD_frameHeader;
1458
+
1459
+ /*! ZSTD_getFrameHeader() :
1460
+ * decode Frame Header, or requires larger `srcSize`.
1461
+ * @return : 0, `zfhPtr` is correctly filled,
1462
+ * >0, `srcSize` is too small, value is wanted `srcSize` amount,
1463
+ * or an error code, which can be tested using ZSTD_isError() */
1464
+ ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
1465
+ /*! ZSTD_getFrameHeader_advanced() :
1466
+ * same as ZSTD_getFrameHeader(),
1467
+ * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
1468
+ ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
1469
+
1470
+ /*! ZSTD_decompressionMargin() :
1471
+ * Zstd supports in-place decompression, where the input and output buffers overlap.
1472
+ * In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
1473
+ * and the input buffer must be at the end of the output buffer.
1474
+ *
1475
+ * _______________________ Output Buffer ________________________
1476
+ * | |
1477
+ * | ____ Input Buffer ____|
1478
+ * | | |
1479
+ * v v v
1480
+ * |---------------------------------------|-----------|----------|
1481
+ * ^ ^ ^
1482
+ * |___________________ Output_Size ___________________|_ Margin _|
1483
+ *
1484
+ * NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
1485
+ * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
1486
+ * ZSTD_decompressDCtx().
1487
+ * NOTE: This function supports multi-frame input.
1488
+ *
1489
+ * @param src The compressed frame(s)
1490
+ * @param srcSize The size of the compressed frame(s)
1491
+ * @returns The decompression margin or an error that can be checked with ZSTD_isError().
1492
+ */
1493
+ ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
1494
+
1495
+ /*! ZSTD_DECOMPRESS_MARGIN() :
1496
+ * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
1497
+ * the compressed frame, compute it from the original size and the blockSizeLog.
1498
+ * See ZSTD_decompressionMargin() for details.
1499
+ *
1500
+ * WARNING: This macro does not support multi-frame input, the input must be a single
1501
+ * zstd frame. If you need that support use the function, or implement it yourself.
1502
+ *
1503
+ * @param originalSize The original uncompressed size of the data.
1504
+ * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
1505
+ * Unless you explicitly set the windowLog smaller than
1506
+ * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
1507
+ */
1508
+ #define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \
1509
+ ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \
1510
+ 4 /* checksum */ + \
1511
+ ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
1512
+ (blockSize) /* One block of margin */ \
1513
+ ))
1514
+
1372
1515
  typedef enum {
1373
1516
  ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
1374
1517
  ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
1375
1518
  } ZSTD_sequenceFormat_e;
1376
1519
 
1520
+ /*! ZSTD_sequenceBound() :
1521
+ * `srcSize` : size of the input buffer
1522
+ * @return : upper-bound for the number of sequences that can be generated
1523
+ * from a buffer of srcSize bytes
1524
+ *
1525
+ * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
1526
+ */
1527
+ ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
1528
+
1377
1529
  /*! ZSTD_generateSequences() :
1378
- * Generate sequences using ZSTD_compress2, given a source buffer.
1530
+ * Generate sequences using ZSTD_compress2(), given a source buffer.
1379
1531
  *
1380
1532
  * Each block will end with a dummy sequence
1381
1533
  * with offset == 0, matchLength == 0, and litLength == length of last literals.
1382
1534
  * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
1383
1535
  * simply acts as a block delimiter.
1384
1536
  *
1385
- * zc can be used to insert custom compression params.
1386
- * This function invokes ZSTD_compress2
1537
+ * @zc can be used to insert custom compression params.
1538
+ * This function invokes ZSTD_compress2().
1387
1539
  *
1388
1540
  * The output of this function can be fed into ZSTD_compressSequences() with CCtx
1389
1541
  * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
1390
1542
  * @return : number of sequences generated
1391
1543
  */
1392
1544
 
1393
- ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
1394
- size_t outSeqsSize, const void* src, size_t srcSize);
1545
+ ZSTDLIB_STATIC_API size_t
1546
+ ZSTD_generateSequences( ZSTD_CCtx* zc,
1547
+ ZSTD_Sequence* outSeqs, size_t outSeqsSize,
1548
+ const void* src, size_t srcSize);
1395
1549
 
1396
1550
  /*! ZSTD_mergeBlockDelimiters() :
1397
1551
  * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
1398
- * by merging them into into the literals of the next sequence.
1552
+ * by merging them into the literals of the next sequence.
1399
1553
  *
1400
1554
  * As such, the final generated result has no explicit representation of block boundaries,
1401
1555
  * and the final last literals segment is not represented in the sequences.
@@ -1407,7 +1561,9 @@ ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* o
1407
1561
  ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
1408
1562
 
1409
1563
  /*! ZSTD_compressSequences() :
1410
- * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
1564
+ * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
1565
+ * @src contains the entire input (not just the literals).
1566
+ * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
1411
1567
  * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
1412
1568
  * The entire source is compressed into a single frame.
1413
1569
  *
@@ -1432,17 +1588,18 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si
1432
1588
  * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
1433
1589
  * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
1434
1590
  * and cannot emit an RLE block that disagrees with the repcode history
1435
- * @return : final compressed size or a ZSTD error.
1591
+ * @return : final compressed size, or a ZSTD error code.
1436
1592
  */
1437
- ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
1438
- const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
1439
- const void* src, size_t srcSize);
1593
+ ZSTDLIB_STATIC_API size_t
1594
+ ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
1595
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
1596
+ const void* src, size_t srcSize);
1440
1597
 
1441
1598
 
1442
1599
  /*! ZSTD_writeSkippableFrame() :
1443
1600
  * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
1444
1601
  *
1445
- * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
1602
+ * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
1446
1603
  * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
1447
1604
  * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
1448
1605
  * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
@@ -1500,8 +1657,11 @@ ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
1500
1657
  * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
1501
1658
  * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
1502
1659
  *
1503
- * Note 2 : only single-threaded compression is supported.
1660
+ * Note : only single-threaded compression is supported.
1504
1661
  * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
1662
+ *
1663
+ * Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
1664
+ * Size estimates assume that no external sequence producer is registered.
1505
1665
  */
1506
1666
  ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
1507
1667
  ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -1520,7 +1680,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
1520
1680
  * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
1521
1681
  * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
1522
1682
  * an internal ?Dict will be created, which additional size is not estimated here.
1523
- * In this case, get total size by adding ZSTD_estimate?DictSize */
1683
+ * In this case, get total size by adding ZSTD_estimate?DictSize
1684
+ * Note 2 : only single-threaded compression is supported.
1685
+ * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
1686
+ * Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
1687
+ * Size estimates assume that no external sequence producer is registered.
1688
+ */
1524
1689
  ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
1525
1690
  ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
1526
1691
  ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
@@ -1670,22 +1835,45 @@ ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
1670
1835
  * This function never fails (wide contract) */
1671
1836
  ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
1672
1837
 
1838
+ /*! ZSTD_CCtx_setCParams() :
1839
+ * Set all parameters provided within @p cparams into the working @p cctx.
1840
+ * Note : if modifying parameters during compression (MT mode only),
1841
+ * note that changes to the .windowLog parameter will be ignored.
1842
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
1843
+ * On failure, no parameters are updated.
1844
+ */
1845
+ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
1846
+
1847
+ /*! ZSTD_CCtx_setFParams() :
1848
+ * Set all parameters provided within @p fparams into the working @p cctx.
1849
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
1850
+ */
1851
+ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams);
1852
+
1853
+ /*! ZSTD_CCtx_setParams() :
1854
+ * Set all parameters provided within @p params into the working @p cctx.
1855
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
1856
+ */
1857
+ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params);
1858
+
1673
1859
  /*! ZSTD_compress_advanced() :
1674
1860
  * Note : this function is now DEPRECATED.
1675
1861
  * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
1676
1862
  * This prototype will generate compilation warnings. */
1677
1863
  ZSTD_DEPRECATED("use ZSTD_compress2")
1864
+ ZSTDLIB_STATIC_API
1678
1865
  size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
1679
- void* dst, size_t dstCapacity,
1680
- const void* src, size_t srcSize,
1681
- const void* dict,size_t dictSize,
1682
- ZSTD_parameters params);
1866
+ void* dst, size_t dstCapacity,
1867
+ const void* src, size_t srcSize,
1868
+ const void* dict,size_t dictSize,
1869
+ ZSTD_parameters params);
1683
1870
 
1684
1871
  /*! ZSTD_compress_usingCDict_advanced() :
1685
1872
  * Note : this function is now DEPRECATED.
1686
1873
  * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
1687
1874
  * This prototype will generate compilation warnings. */
1688
1875
  ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
1876
+ ZSTDLIB_STATIC_API
1689
1877
  size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
1690
1878
  void* dst, size_t dstCapacity,
1691
1879
  const void* src, size_t srcSize,
@@ -1829,13 +2017,16 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
1829
2017
  * Experimental parameter.
1830
2018
  * Default is 0 == disabled. Set to 1 to enable.
1831
2019
  *
1832
- * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same
1833
- * between calls, except for the modifications that zstd makes to pos (the
1834
- * caller must not modify pos). This is checked by the compressor, and
1835
- * compression will fail if it ever changes. This means the only flush
1836
- * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end
1837
- * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos)
1838
- * MUST not be modified during compression or you will get data corruption.
2020
+ * Tells the compressor that input data presented with ZSTD_inBuffer
2021
+ * will ALWAYS be the same between calls.
2022
+ * Technically, the @src pointer must never be changed,
2023
+ * and the @pos field can only be updated by zstd.
2024
+ * However, it's possible to increase the @size field,
2025
+ * allowing scenarios where more data can be appended after compressions starts.
2026
+ * These conditions are checked by the compressor,
2027
+ * and compression will fail if they are not respected.
2028
+ * Also, data in the ZSTD_inBuffer within the range [src, src + pos)
2029
+ * MUST not be modified during compression or it will result in data corruption.
1839
2030
  *
1840
2031
  * When this flag is enabled zstd won't allocate an input window buffer,
1841
2032
  * because the user guarantees it can reference the ZSTD_inBuffer until
@@ -1843,18 +2034,15 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
1843
2034
  * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
1844
2035
  * avoid the memcpy() from the input buffer to the input window buffer.
1845
2036
  *
1846
- * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
1847
- * That means this flag cannot be used with ZSTD_compressStream().
1848
- *
1849
2037
  * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
1850
2038
  * this flag is ALWAYS memory safe, and will never access out-of-bounds
1851
- * memory. However, compression WILL fail if you violate the preconditions.
2039
+ * memory. However, compression WILL fail if conditions are not respected.
1852
2040
  *
1853
- * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST
1854
- * not be modified during compression or you will get data corruption. This
1855
- * is because zstd needs to reference data in the ZSTD_inBuffer to find
2041
+ * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
2042
+ * not be modified during compression or it will result in data corruption.
2043
+ * This is because zstd needs to reference data in the ZSTD_inBuffer to find
1856
2044
  * matches. Normally zstd maintains its own window buffer for this purpose,
1857
- * but passing this flag tells zstd to use the user provided buffer.
2045
+ * but passing this flag tells zstd to rely on user provided buffer instead.
1858
2046
  */
1859
2047
  #define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
1860
2048
 
@@ -1899,7 +2087,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
1899
2087
  * Without validation, providing a sequence that does not conform to the zstd spec will cause
1900
2088
  * undefined behavior, and may produce a corrupted block.
1901
2089
  *
1902
- * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for
2090
+ * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
1903
2091
  * specifics regarding offset/matchlength requirements) then the function will bail out and
1904
2092
  * return an error.
1905
2093
  *
@@ -1949,6 +2137,79 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
1949
2137
  */
1950
2138
  #define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
1951
2139
 
2140
+ /* ZSTD_c_prefetchCDictTables
2141
+ * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
2142
+ *
2143
+ * In some situations, zstd uses CDict tables in-place rather than copying them
2144
+ * into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
2145
+ * In such situations, compression speed is seriously impacted when CDict tables are
2146
+ * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables
2147
+ * when they are used in-place.
2148
+ *
2149
+ * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit.
2150
+ * For sufficiently large inputs, zstd will by default memcpy() CDict tables
2151
+ * into the working context, so there is no need to prefetch. This parameter is
2152
+ * targeted at a middle range of input sizes, where a prefetch is cheap enough to be
2153
+ * useful but memcpy() is too expensive. The exact range of input sizes where this
2154
+ * makes sense is best determined by careful experimentation.
2155
+ *
2156
+ * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable,
2157
+ * but in the future zstd may conditionally enable this feature via an auto-detection
2158
+ * heuristic for cold CDicts.
2159
+ * Use ZSTD_ps_disable to opt out of prefetching under any circumstances.
2160
+ */
2161
+ #define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
2162
+
2163
+ /* ZSTD_c_enableSeqProducerFallback
2164
+ * Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
2165
+ *
2166
+ * Controls whether zstd will fall back to an internal sequence producer if an
2167
+ * external sequence producer is registered and returns an error code. This fallback
2168
+ * is block-by-block: the internal sequence producer will only be called for blocks
2169
+ * where the external sequence producer returns an error code. Fallback parsing will
2170
+ * follow any other cParam settings, such as compression level, the same as in a
2171
+ * normal (fully-internal) compression operation.
2172
+ *
2173
+ * The user is strongly encouraged to read the full Block-Level Sequence Producer API
2174
+ * documentation (below) before setting this parameter. */
2175
+ #define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17
2176
+
2177
+ /* ZSTD_c_maxBlockSize
2178
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
2179
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
2180
+ *
2181
+ * This parameter can be used to set an upper bound on the blocksize
2182
+ * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
2183
+ * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
2184
+ * compressBound() inaccurate). Only currently meant to be used for testing.
2185
+ *
2186
+ */
2187
+ #define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
2188
+
2189
+ /* ZSTD_c_searchForExternalRepcodes
2190
+ * This parameter affects how zstd parses external sequences, such as sequences
2191
+ * provided through the compressSequences() API or from an external block-level
2192
+ * sequence producer.
2193
+ *
2194
+ * If set to ZSTD_ps_enable, the library will check for repeated offsets in
2195
+ * external sequences, even if those repcodes are not explicitly indicated in
2196
+ * the "rep" field. Note that this is the only way to exploit repcode matches
2197
+ * while using compressSequences() or an external sequence producer, since zstd
2198
+ * currently ignores the "rep" field of external sequences.
2199
+ *
2200
+ * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
2201
+ * external sequences, regardless of whether the "rep" field has been set. This
2202
+ * reduces sequence compression overhead by about 25% while sacrificing some
2203
+ * compression ratio.
2204
+ *
2205
+ * The default value is ZSTD_ps_auto, for which the library will enable/disable
2206
+ * based on compression level.
2207
+ *
2208
+ * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
2209
+ * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
2210
+ */
2211
+ #define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
2212
+
1952
2213
  /*! ZSTD_CCtx_getParameter() :
1953
2214
  * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
1954
2215
  * and store it into int* value.
@@ -2105,7 +2366,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
2105
2366
  * in the range [dst, dst + pos) MUST not be modified during decompression
2106
2367
  * or you will get data corruption.
2107
2368
  *
2108
- * When this flags is enabled zstd won't allocate an output buffer, because
2369
+ * When this flag is enabled zstd won't allocate an output buffer, because
2109
2370
  * it can write directly to the ZSTD_outBuffer, but it will still allocate
2110
2371
  * an input buffer large enough to fit any compressed block. This will also
2111
2372
  * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
@@ -2158,6 +2419,17 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
2158
2419
  */
2159
2420
  #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
2160
2421
 
2422
+ /* ZSTD_d_disableHuffmanAssembly
2423
+ * Set to 1 to disable the Huffman assembly implementation.
2424
+ * The default value is 0, which allows zstd to use the Huffman assembly
2425
+ * implementation if available.
2426
+ *
2427
+ * This parameter can be used to disable Huffman assembly at runtime.
2428
+ * If you want to disable it at compile time you can define the macro
2429
+ * ZSTD_DISABLE_ASM.
2430
+ */
2431
+ #define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
2432
+
2161
2433
 
2162
2434
  /*! ZSTD_DCtx_setFormat() :
2163
2435
  * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
@@ -2166,6 +2438,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
2166
2438
  * such ZSTD_f_zstd1_magicless for example.
2167
2439
  * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
2168
2440
  ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
2441
+ ZSTDLIB_STATIC_API
2169
2442
  size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
2170
2443
 
2171
2444
  /*! ZSTD_decompressStream_simpleArgs() :
@@ -2202,6 +2475,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
2202
2475
  * This prototype will generate compilation warnings.
2203
2476
  */
2204
2477
  ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
2478
+ ZSTDLIB_STATIC_API
2205
2479
  size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
2206
2480
  int compressionLevel,
2207
2481
  unsigned long long pledgedSrcSize);
@@ -2219,17 +2493,15 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
2219
2493
  * This prototype will generate compilation warnings.
2220
2494
  */
2221
2495
  ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
2496
+ ZSTDLIB_STATIC_API
2222
2497
  size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
2223
2498
  const void* dict, size_t dictSize,
2224
2499
  int compressionLevel);
2225
2500
 
2226
2501
  /*! ZSTD_initCStream_advanced() :
2227
- * This function is DEPRECATED, and is approximately equivalent to:
2502
+ * This function is DEPRECATED, and is equivalent to:
2228
2503
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
2229
- * // Pseudocode: Set each zstd parameter and leave the rest as-is.
2230
- * for ((param, value) : params) {
2231
- * ZSTD_CCtx_setParameter(zcs, param, value);
2232
- * }
2504
+ * ZSTD_CCtx_setParams(zcs, params);
2233
2505
  * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
2234
2506
  * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
2235
2507
  *
@@ -2239,6 +2511,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
2239
2511
  * This prototype will generate compilation warnings.
2240
2512
  */
2241
2513
  ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
2514
+ ZSTDLIB_STATIC_API
2242
2515
  size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
2243
2516
  const void* dict, size_t dictSize,
2244
2517
  ZSTD_parameters params,
@@ -2253,15 +2526,13 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
2253
2526
  * This prototype will generate compilation warnings.
2254
2527
  */
2255
2528
  ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
2529
+ ZSTDLIB_STATIC_API
2256
2530
  size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
2257
2531
 
2258
2532
  /*! ZSTD_initCStream_usingCDict_advanced() :
2259
- * This function is DEPRECATED, and is approximately equivalent to:
2533
+ * This function is DEPRECATED, and is equivalent to:
2260
2534
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
2261
- * // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
2262
- * for ((fParam, value) : fParams) {
2263
- * ZSTD_CCtx_setParameter(zcs, fParam, value);
2264
- * }
2535
+ * ZSTD_CCtx_setFParams(zcs, fParams);
2265
2536
  * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
2266
2537
  * ZSTD_CCtx_refCDict(zcs, cdict);
2267
2538
  *
@@ -2271,6 +2542,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
2271
2542
  * This prototype will generate compilation warnings.
2272
2543
  */
2273
2544
  ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
2545
+ ZSTDLIB_STATIC_API
2274
2546
  size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
2275
2547
  const ZSTD_CDict* cdict,
2276
2548
  ZSTD_frameParameters fParams,
@@ -2295,6 +2567,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
2295
2567
  * This prototype will generate compilation warnings.
2296
2568
  */
2297
2569
  ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
2570
+ ZSTDLIB_STATIC_API
2298
2571
  size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
2299
2572
 
2300
2573
 
@@ -2340,8 +2613,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
2340
2613
  * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
2341
2614
  *
2342
2615
  * note: no dictionary will be used if dict == NULL or dictSize < 8
2343
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
2344
2616
  */
2617
+ ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
2345
2618
  ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
2346
2619
 
2347
2620
  /*!
@@ -2351,8 +2624,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const vo
2351
2624
  * ZSTD_DCtx_refDDict(zds, ddict);
2352
2625
  *
2353
2626
  * note : ddict is referenced, it must outlive decompression session
2354
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
2355
2627
  */
2628
+ ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
2356
2629
  ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
2357
2630
 
2358
2631
  /*!
@@ -2361,17 +2634,185 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const Z
2361
2634
  * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
2362
2635
  *
2363
2636
  * re-use decompression parameters from previous init; saves dictionary loading
2364
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
2365
2637
  */
2638
+ ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
2366
2639
  ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
2367
2640
 
2368
2641
 
2642
+ /* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
2643
+ *
2644
+ * *** OVERVIEW ***
2645
+ * The Block-Level Sequence Producer API allows users to provide their own custom
2646
+ * sequence producer which libzstd invokes to process each block. The produced list
2647
+ * of sequences (literals and matches) is then post-processed by libzstd to produce
2648
+ * valid compressed blocks.
2649
+ *
2650
+ * This block-level offload API is a more granular complement of the existing
2651
+ * frame-level offload API compressSequences() (introduced in v1.5.1). It offers
2652
+ * an easier migration story for applications already integrated with libzstd: the
2653
+ * user application continues to invoke the same compression functions
2654
+ * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits
2655
+ * from the specific advantages of the external sequence producer. For example,
2656
+ * the sequence producer could be tuned to take advantage of known characteristics
2657
+ * of the input, to offer better speed / ratio, or could leverage hardware
2658
+ * acceleration not available within libzstd itself.
2659
+ *
2660
+ * See contrib/externalSequenceProducer for an example program employing the
2661
+ * Block-Level Sequence Producer API.
2662
+ *
2663
+ * *** USAGE ***
2664
+ * The user is responsible for implementing a function of type
2665
+ * ZSTD_sequenceProducer_F. For each block, zstd will pass the following
2666
+ * arguments to the user-provided function:
2667
+ *
2668
+ * - sequenceProducerState: a pointer to a user-managed state for the sequence
2669
+ * producer.
2670
+ *
2671
+ * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer.
2672
+ * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory
2673
+ * backing outSeqs is managed by the CCtx.
2674
+ *
2675
+ * - src, srcSize: an input buffer for the sequence producer to parse.
2676
+ * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
2677
+ *
2678
+ * - dict, dictSize: a history buffer, which may be empty, which the sequence
2679
+ * producer may reference as it parses the src buffer. Currently, zstd will
2680
+ * always pass dictSize == 0 into external sequence producers, but this will
2681
+ * change in the future.
2682
+ *
2683
+ * - compressionLevel: a signed integer representing the zstd compression level
2684
+ * set by the user for the current operation. The sequence producer may choose
2685
+ * to use this information to change its compression strategy and speed/ratio
2686
+ * tradeoff. Note: the compression level does not reflect zstd parameters set
2687
+ * through the advanced API.
2688
+ *
2689
+ * - windowSize: a size_t representing the maximum allowed offset for external
2690
+ * sequences. Note that sequence offsets are sometimes allowed to exceed the
2691
+ * windowSize if a dictionary is present, see doc/zstd_compression_format.md
2692
+ * for details.
2693
+ *
2694
+ * The user-provided function shall return a size_t representing the number of
2695
+ * sequences written to outSeqs. This return value will be treated as an error
2696
+ * code if it is greater than outSeqsCapacity. The return value must be non-zero
2697
+ * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided
2698
+ * for convenience, but any value greater than outSeqsCapacity will be treated as
2699
+ * an error code.
2700
+ *
2701
+ * If the user-provided function does not return an error code, the sequences
2702
+ * written to outSeqs must be a valid parse of the src buffer. Data corruption may
2703
+ * occur if the parse is not valid. A parse is defined to be valid if the
2704
+ * following conditions hold:
2705
+ * - The sum of matchLengths and literalLengths must equal srcSize.
2706
+ * - All sequences in the parse, except for the final sequence, must have
2707
+ * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have
2708
+ * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0.
2709
+ * - All offsets must respect the windowSize parameter as specified in
2710
+ * doc/zstd_compression_format.md.
2711
+ * - If the final sequence has matchLength == 0, it must also have offset == 0.
2712
+ *
2713
+ * zstd will only validate these conditions (and fail compression if they do not
2714
+ * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
2715
+ * validation has a performance cost.
2716
+ *
2717
+ * If the user-provided function returns an error, zstd will either fall back
2718
+ * to an internal sequence producer or fail the compression operation. The user can
2719
+ * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback
2720
+ * cParam. Fallback compression will follow any other cParam settings, such as
2721
+ * compression level, the same as in a normal compression operation.
2722
+ *
2723
+ * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F
2724
+ * function by calling
2725
+ * ZSTD_registerSequenceProducer(cctx,
2726
+ * sequenceProducerState,
2727
+ * sequenceProducer)
2728
+ * This setting will persist until the next parameter reset of the CCtx.
2729
+ *
2730
+ * The sequenceProducerState must be initialized by the user before calling
2731
+ * ZSTD_registerSequenceProducer(). The user is responsible for destroying the
2732
+ * sequenceProducerState.
2733
+ *
2734
+ * *** LIMITATIONS ***
2735
+ * This API is compatible with all zstd compression APIs which respect advanced parameters.
2736
+ * However, there are three limitations:
2737
+ *
2738
+ * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported.
2739
+ * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level
2740
+ * external sequence producer.
2741
+ * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some
2742
+ * cases (see its documentation for details). Users must explicitly set
2743
+ * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external
2744
+ * sequence producer is registered.
2745
+ * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
2746
+ * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
2747
+ * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence
2748
+ * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog).
2749
+ *
2750
+ * Second, history buffers are not currently supported. Concretely, zstd will always pass
2751
+ * dictSize == 0 to the external sequence producer (for now). This has two implications:
2752
+ * - Dictionaries are not currently supported. Compression will *not* fail if the user
2753
+ * references a dictionary, but the dictionary won't have any effect.
2754
+ * - Stream history is not currently supported. All advanced compression APIs, including
2755
+ * streaming APIs, work with external sequence producers, but each block is treated as
2756
+ * an independent chunk without history from previous blocks.
2757
+ *
2758
+ * Third, multi-threading within a single compression is not currently supported. In other words,
2759
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered.
2760
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
2761
+ *
2762
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
2763
+ * overcoming them. It is purely a question of engineering effort.
2764
+ */
2765
+
2766
+ #define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1))
2767
+
2768
+ typedef size_t ZSTD_sequenceProducer_F (
2769
+ void* sequenceProducerState,
2770
+ ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
2771
+ const void* src, size_t srcSize,
2772
+ const void* dict, size_t dictSize,
2773
+ int compressionLevel,
2774
+ size_t windowSize
2775
+ );
2776
+
2777
+ /*! ZSTD_registerSequenceProducer() :
2778
+ * Instruct zstd to use a block-level external sequence producer function.
2779
+ *
2780
+ * The sequenceProducerState must be initialized by the caller, and the caller is
2781
+ * responsible for managing its lifetime. This parameter is sticky across
2782
+ * compressions. It will remain set until the user explicitly resets compression
2783
+ * parameters.
2784
+ *
2785
+ * Sequence producer registration is considered to be an "advanced parameter",
2786
+ * part of the "advanced API". This means it will only have an effect on compression
2787
+ * APIs which respect advanced parameters, such as compress2() and compressStream2().
2788
+ * Older compression APIs such as compressCCtx(), which predate the introduction of
2789
+ * "advanced parameters", will ignore any external sequence producer setting.
2790
+ *
2791
+ * The sequence producer can be "cleared" by registering a NULL function pointer. This
2792
+ * removes all limitations described above in the "LIMITATIONS" section of the API docs.
2793
+ *
2794
+ * The user is strongly encouraged to read the full API documentation (above) before
2795
+ * calling this function. */
2796
+ ZSTDLIB_STATIC_API void
2797
+ ZSTD_registerSequenceProducer(
2798
+ ZSTD_CCtx* cctx,
2799
+ void* sequenceProducerState,
2800
+ ZSTD_sequenceProducer_F* sequenceProducer
2801
+ );
2802
+
2803
+
2369
2804
  /*********************************************************************
2370
- * Buffer-less and synchronous inner streaming functions
2805
+ * Buffer-less and synchronous inner streaming functions (DEPRECATED)
2806
+ *
2807
+ * This API is deprecated, and will be removed in a future version.
2808
+ * It allows streaming (de)compression with user allocated buffers.
2809
+ * However, it is hard to use, and not as well tested as the rest of
2810
+ * our API.
2371
2811
  *
2372
- * This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
2373
- * But it's also a complex one, with several restrictions, documented below.
2374
- * Prefer normal streaming API for an easier experience.
2812
+ * Please use the normal streaming API instead: ZSTD_compressStream2,
2813
+ * and ZSTD_decompressStream.
2814
+ * If there is functionality that you need, but it doesn't provide,
2815
+ * please open an issue on our GitHub.
2375
2816
  ********************************************************************* */
2376
2817
 
2377
2818
  /**
@@ -2383,7 +2824,6 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
2383
2824
 
2384
2825
  Start by initializing a context.
2385
2826
  Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
2386
- It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
2387
2827
 
2388
2828
  Then, consume your input using ZSTD_compressContinue().
2389
2829
  There are some important considerations to keep in mind when using this advanced function :
@@ -2405,18 +2845,28 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
2405
2845
  */
2406
2846
 
2407
2847
  /*===== Buffer-less streaming compression functions =====*/
2848
+ ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
2408
2849
  ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
2850
+ ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
2409
2851
  ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
2852
+ ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
2410
2853
  ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
2411
- ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
2412
2854
 
2855
+ ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
2856
+ ZSTDLIB_STATIC_API
2857
+ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
2858
+
2859
+ ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
2413
2860
  ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
2861
+ ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
2414
2862
  ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
2415
2863
 
2416
2864
  /* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
2417
2865
  ZSTD_DEPRECATED("use advanced API to access custom parameters")
2866
+ ZSTDLIB_STATIC_API
2418
2867
  size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
2419
2868
  ZSTD_DEPRECATED("use advanced API to access custom parameters")
2869
+ ZSTDLIB_STATIC_API
2420
2870
  size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
2421
2871
  /**
2422
2872
  Buffer-less streaming decompression (synchronous mode)
@@ -2429,8 +2879,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
2429
2879
  Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
2430
2880
  Data fragment must be large enough to ensure successful decoding.
2431
2881
  `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
2432
- @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
2433
- >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
2882
+ result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
2883
+ >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
2434
2884
  errorCode, which can be tested using ZSTD_isError().
2435
2885
 
2436
2886
  It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
@@ -2449,7 +2899,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
2449
2899
 
2450
2900
  The most memory efficient way is to use a round buffer of sufficient size.
2451
2901
  Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
2452
- which can @return an error code if required value is too large for current system (in 32-bits mode).
2902
+ which can return an error code if required value is too large for current system (in 32-bits mode).
2453
2903
  In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
2454
2904
  up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
2455
2905
  which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
@@ -2469,7 +2919,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
2469
2919
  ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
2470
2920
  ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
2471
2921
 
2472
- @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
2922
+ result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
2473
2923
  It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
2474
2924
  It can also be an error code, which can be tested with ZSTD_isError().
2475
2925
 
@@ -2492,27 +2942,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
2492
2942
  */
2493
2943
 
2494
2944
  /*===== Buffer-less streaming decompression functions =====*/
2495
- typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
2496
- typedef struct {
2497
- unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
2498
- unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
2499
- unsigned blockSizeMax;
2500
- ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
2501
- unsigned headerSize;
2502
- unsigned dictID;
2503
- unsigned checksumFlag;
2504
- } ZSTD_frameHeader;
2505
2945
 
2506
- /*! ZSTD_getFrameHeader() :
2507
- * decode Frame Header, or requires larger `srcSize`.
2508
- * @return : 0, `zfhPtr` is correctly filled,
2509
- * >0, `srcSize` is too small, value is wanted `srcSize` amount,
2510
- * or an error code, which can be tested using ZSTD_isError() */
2511
- ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
2512
- /*! ZSTD_getFrameHeader_advanced() :
2513
- * same as ZSTD_getFrameHeader(),
2514
- * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
2515
- ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
2516
2946
  ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
2517
2947
 
2518
2948
  ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
@@ -2523,6 +2953,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
2523
2953
  ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
2524
2954
 
2525
2955
  /* misc */
2956
+ ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
2526
2957
  ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
2527
2958
  typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
2528
2959
  ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
@@ -2530,11 +2961,23 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
2530
2961
 
2531
2962
 
2532
2963
 
2533
- /* ============================ */
2534
- /** Block level API */
2535
- /* ============================ */
2964
+ /* ========================================= */
2965
+ /** Block level API (DEPRECATED) */
2966
+ /* ========================================= */
2536
2967
 
2537
2968
  /*!
2969
+
2970
+ This API is deprecated in favor of the regular compression API.
2971
+ You can get the frame header down to 2 bytes by setting:
2972
+ - ZSTD_c_format = ZSTD_f_zstd1_magicless
2973
+ - ZSTD_c_contentSizeFlag = 0
2974
+ - ZSTD_c_checksumFlag = 0
2975
+ - ZSTD_c_dictIDFlag = 0
2976
+
2977
+ This API is not as well tested as our normal API, so we recommend not using it.
2978
+ We will be removing it in a future version. If the normal API doesn't provide
2979
+ the functionality you need, please open a GitHub issue.
2980
+
2538
2981
  Block functions produce and decode raw zstd blocks, without frame metadata.
2539
2982
  Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
2540
2983
  But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
@@ -2545,7 +2988,6 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
2545
2988
  - It is necessary to init context before starting
2546
2989
  + compression : any ZSTD_compressBegin*() variant, including with dictionary
2547
2990
  + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
2548
- + copyCCtx() and copyDCtx() can be used too
2549
2991
  - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
2550
2992
  + If input is larger than a block size, it's necessary to split input data into multiple blocks
2551
2993
  + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
@@ -2562,12 +3004,15 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
2562
3004
  */
2563
3005
 
2564
3006
  /*===== Raw zstd block functions =====*/
3007
+ ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
2565
3008
  ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
3009
+ ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
2566
3010
  ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
3011
+ ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
2567
3012
  ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
3013
+ ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
2568
3014
  ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
2569
3015
 
2570
-
2571
3016
  #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
2572
3017
 
2573
3018
  #if defined (__cplusplus)