zstdlib 0.4.0-x64-mingw32 → 0.5.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (70) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +6 -1
  3. data/README.md +1 -1
  4. data/ext/zstdlib/extconf.rb +2 -2
  5. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/bitstream.h +3 -2
  6. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/compiler.h +14 -2
  7. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/cpu.h +0 -0
  8. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/debug.c +0 -0
  9. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/debug.h +0 -0
  10. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/entropy_common.c +0 -0
  11. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/error_private.c +0 -0
  12. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/error_private.h +0 -0
  13. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/fse.h +1 -1
  14. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/fse_decompress.c +2 -0
  15. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/huf.h +0 -0
  16. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/mem.h +73 -0
  17. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/pool.c +7 -3
  18. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/pool.h +0 -0
  19. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/threading.c +46 -1
  20. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/threading.h +32 -1
  21. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/xxhash.c +0 -0
  22. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/xxhash.h +0 -0
  23. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/zstd_common.c +0 -0
  24. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/zstd_errors.h +0 -0
  25. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/zstd_internal.h +32 -55
  26. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/fse_compress.c +0 -0
  27. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/hist.c +0 -0
  28. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/hist.h +0 -0
  29. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/huf_compress.c +0 -0
  30. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress.c +633 -436
  31. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress_internal.h +54 -12
  32. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress_literals.c +10 -5
  33. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress_literals.h +1 -1
  34. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress_sequences.c +3 -3
  35. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress_sequences.h +1 -1
  36. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_cwksp.h +535 -0
  37. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_double_fast.c +9 -9
  38. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_double_fast.h +0 -0
  39. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_fast.c +30 -39
  40. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_fast.h +0 -0
  41. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_lazy.c +5 -5
  42. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_lazy.h +0 -0
  43. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_ldm.c +4 -4
  44. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_ldm.h +0 -0
  45. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_opt.c +1 -1
  46. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_opt.h +0 -0
  47. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstdmt_compress.c +32 -26
  48. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstdmt_compress.h +0 -0
  49. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/huf_decompress.c +2 -0
  50. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_ddict.c +0 -0
  51. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_ddict.h +0 -0
  52. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_decompress.c +14 -16
  53. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_decompress_block.c +144 -146
  54. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_decompress_block.h +0 -0
  55. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_decompress_internal.h +0 -0
  56. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/zstd.h +161 -59
  57. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzclose.c +1 -1
  58. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzcompatibility.h +0 -0
  59. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzguts.h +0 -0
  60. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzlib.c +9 -9
  61. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzread.c +16 -8
  62. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzwrite.c +8 -8
  63. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/zstd_zlibwrapper.c +15 -11
  64. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/zstd_zlibwrapper.h +0 -0
  65. data/lib/2.2/zstdlib.so +0 -0
  66. data/lib/2.3/zstdlib.so +0 -0
  67. data/lib/2.4/zstdlib.so +0 -0
  68. data/lib/2.5/zstdlib.so +0 -0
  69. data/lib/2.6/zstdlib.so +0 -0
  70. metadata +62 -61
@@ -15,6 +15,7 @@ extern "C" {
15
15
  #define ZSTD_H_235446
16
16
 
17
17
  /* ====== Dependency ======*/
18
+ #include <limits.h> /* INT_MAX */
18
19
  #include <stddef.h> /* size_t */
19
20
 
20
21
 
@@ -71,7 +72,7 @@ extern "C" {
71
72
  /*------ Version ------*/
72
73
  #define ZSTD_VERSION_MAJOR 1
73
74
  #define ZSTD_VERSION_MINOR 4
74
- #define ZSTD_VERSION_RELEASE 3
75
+ #define ZSTD_VERSION_RELEASE 4
75
76
 
76
77
  #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
77
78
  ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */
@@ -196,9 +197,13 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
196
197
  ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
197
198
 
198
199
  /*! ZSTD_compressCCtx() :
199
- * Same as ZSTD_compress(), using an explicit ZSTD_CCtx
200
- * The function will compress at requested compression level,
201
- * ignoring any other parameter */
200
+ * Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
201
+ * Important : in order to behave similarly to `ZSTD_compress()`,
202
+ * this function compresses at requested compression level,
203
+ * __ignoring any other parameter__ .
204
+ * If any advanced parameter was set using the advanced API,
205
+ * they will all be reset. Only `compressionLevel` remains.
206
+ */
202
207
  ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
203
208
  void* dst, size_t dstCapacity,
204
209
  const void* src, size_t srcSize,
@@ -233,7 +238,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
233
238
  * using ZSTD_CCtx_set*() functions.
234
239
  * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
235
240
  * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
236
- * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()
241
+ * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
237
242
  *
238
243
  * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
239
244
  *
@@ -261,18 +266,26 @@ typedef enum {
261
266
 
262
267
  /* compression parameters
263
268
  * Note: When compressing with a ZSTD_CDict these parameters are superseded
264
- * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
265
- * for more info (superseded-by-cdict). */
266
- ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
269
+ * by the parameters used to construct the ZSTD_CDict.
270
+ * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
271
+ ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
272
+ * Note that exact compression parameters are dynamically determined,
273
+ * depending on both compression level and srcSize (when known).
267
274
  * Default level is ZSTD_CLEVEL_DEFAULT==3.
268
275
  * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
269
276
  * Note 1 : it's possible to pass a negative compression level.
270
- * Note 2 : setting a level sets all default values of other compression parameters */
277
+ * Note 2 : setting a level resets all other compression parameters to default */
278
+ /* Advanced compression parameters :
279
+ * It's possible to pin down compression parameters to some specific values.
280
+ * In which case, these values are no longer dynamically selected by the compressor */
271
281
  ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
282
+ * This will set a memory budget for streaming decompression,
283
+ * with larger values requiring more memory
284
+ * and typically compressing more.
272
285
  * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
273
286
  * Special: value 0 means "use default windowLog".
274
287
  * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
275
- * requires explicitly allowing such window size at decompression stage if using streaming. */
288
+ * requires explicitly allowing such size at streaming decompression stage. */
276
289
  ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
277
290
  * Resulting memory usage is (1 << (hashLog+2)).
278
291
  * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
@@ -283,13 +296,13 @@ typedef enum {
283
296
  * Resulting memory usage is (1 << (chainLog+2)).
284
297
  * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
285
298
  * Larger tables result in better and slower compression.
286
- * This parameter is useless when using "fast" strategy.
299
+ * This parameter is useless for "fast" strategy.
287
300
  * It's still useful when using "dfast" strategy,
288
301
  * in which case it defines a secondary probe table.
289
302
  * Special: value 0 means "use default chainLog". */
290
303
  ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
291
304
  * More attempts result in better and slower compression.
292
- * This parameter is useless when using "fast" and "dFast" strategies.
305
+ * This parameter is useless for "fast" and "dFast" strategies.
293
306
  * Special: value 0 means "use default searchLog". */
294
307
  ZSTD_c_minMatch=105, /* Minimum size of searched matches.
295
308
  * Note that Zstandard can still find matches of smaller size,
@@ -344,7 +357,7 @@ typedef enum {
344
357
  ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
345
358
  * Content size must be known at the beginning of compression.
346
359
  * This is automatically the case when using ZSTD_compress2(),
347
- * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
360
+ * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
348
361
  ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
349
362
  ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
350
363
 
@@ -363,7 +376,7 @@ typedef enum {
363
376
  * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
364
377
  * 0 means default, which is dynamically determined based on compression parameters.
365
378
  * Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
366
- * The minimum size is automatically and transparently enforced */
379
+ * The minimum size is automatically and transparently enforced. */
367
380
  ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
368
381
  * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
369
382
  * It helps preserve compression ratio, while each job is compressed in parallel.
@@ -386,6 +399,7 @@ typedef enum {
386
399
  * ZSTD_c_forceAttachDict
387
400
  * ZSTD_c_literalCompressionMode
388
401
  * ZSTD_c_targetCBlockSize
402
+ * ZSTD_c_srcSizeHint
389
403
  * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
390
404
  * note : never ever use experimentalParam? names directly;
391
405
  * also, the enums values themselves are unstable and can still change.
@@ -396,6 +410,7 @@ typedef enum {
396
410
  ZSTD_c_experimentalParam4=1001,
397
411
  ZSTD_c_experimentalParam5=1002,
398
412
  ZSTD_c_experimentalParam6=1003,
413
+ ZSTD_c_experimentalParam7=1004
399
414
  } ZSTD_cParameter;
400
415
 
401
416
  typedef struct {
@@ -793,12 +808,17 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
793
808
  typedef struct ZSTD_CDict_s ZSTD_CDict;
794
809
 
795
810
  /*! ZSTD_createCDict() :
796
- * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
797
- * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
811
+ * When compressing multiple messages or blocks using the same dictionary,
812
+ * it's recommended to digest the dictionary only once, since it's a costly operation.
813
+ * ZSTD_createCDict() will create a state from digesting a dictionary.
814
+ * The resulting state can be used for future compression operations with very limited startup cost.
798
815
  * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
799
- * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
800
- * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
801
- * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
816
+ * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
817
+ * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
818
+ * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
819
+ * in which case the only thing that it transports is the @compressionLevel.
820
+ * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
821
+ * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
802
822
  ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
803
823
  int compressionLevel);
804
824
 
@@ -925,7 +945,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
925
945
  * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
926
946
  * It's a CPU consuming operation, with non-negligible impact on latency.
927
947
  * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
928
- * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
948
+ * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
929
949
  * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
930
950
  ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
931
951
  const void* prefix, size_t prefixSize);
@@ -969,7 +989,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
969
989
  * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
970
990
  * Prefix buffer must remain unmodified up to the end of frame,
971
991
  * reached when ZSTD_decompressStream() returns 0.
972
- * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
992
+ * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
973
993
  * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
974
994
  * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
975
995
  * A full dictionary is more costly, as it requires building tables.
@@ -1014,8 +1034,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
1014
1034
  * Some of them might be removed in the future (especially when redundant with existing stable functions)
1015
1035
  * ***************************************************************************************/
1016
1036
 
1017
- #define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */
1018
- #define ZSTD_FRAMEHEADERSIZE_MIN 6
1037
+ #define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
1038
+ #define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
1019
1039
  #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
1020
1040
  #define ZSTD_SKIPPABLEHEADERSIZE 8
1021
1041
 
@@ -1063,6 +1083,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
1063
1083
  /* Advanced parameter bounds */
1064
1084
  #define ZSTD_TARGETCBLOCKSIZE_MIN 64
1065
1085
  #define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
1086
+ #define ZSTD_SRCSIZEHINT_MIN 0
1087
+ #define ZSTD_SRCSIZEHINT_MAX INT_MAX
1066
1088
 
1067
1089
  /* internal */
1068
1090
  #define ZSTD_HASHLOG3_MAX 17
@@ -1072,6 +1094,24 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
1072
1094
 
1073
1095
  typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
1074
1096
 
1097
+ typedef struct {
1098
+ unsigned int matchPos; /* Match pos in dst */
1099
+ /* If seqDef.offset > 3, then this is seqDef.offset - 3
1100
+ * If seqDef.offset < 3, then this is the corresponding repeat offset
1101
+ * But if seqDef.offset < 3 and litLength == 0, this is the
1102
+ * repeat offset before the corresponding repeat offset
1103
+ * And if seqDef.offset == 3 and litLength == 0, this is the
1104
+ * most recent repeat offset - 1
1105
+ */
1106
+ unsigned int offset;
1107
+ unsigned int litLength; /* Literal length */
1108
+ unsigned int matchLength; /* Match length */
1109
+ /* 0 when seq not rep and seqDef.offset otherwise
1110
+ * when litLength == 0 this will be <= 4, otherwise <= 3 like normal
1111
+ */
1112
+ unsigned int rep;
1113
+ } ZSTD_Sequence;
1114
+
1075
1115
  typedef struct {
1076
1116
  unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
1077
1117
  unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
@@ -1101,21 +1141,12 @@ typedef enum {
1101
1141
 
1102
1142
  typedef enum {
1103
1143
  ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
1104
- ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
1144
+ ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
1105
1145
  } ZSTD_dictLoadMethod_e;
1106
1146
 
1107
1147
  typedef enum {
1108
- /* Opened question : should we have a format ZSTD_f_auto ?
1109
- * Today, it would mean exactly the same as ZSTD_f_zstd1.
1110
- * But, in the future, should several formats become supported,
1111
- * on the compression side, it would mean "default format".
1112
- * On the decompression side, it would mean "automatic format detection",
1113
- * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
1114
- * Since meaning is a little different, another option could be to define different enums for compression and decompression.
1115
- * This question could be kept for later, when there are actually multiple formats to support,
1116
- * but there is also the question of pinning enum values, and pinning value `0` is especially important */
1117
1148
  ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
1118
- ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number.
1149
+ ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
1119
1150
  * Useful to save 4 bytes per generated frame.
1120
1151
  * Decoder cannot recognise automatically this format, requiring this instruction. */
1121
1152
  } ZSTD_format_e;
@@ -1126,7 +1157,7 @@ typedef enum {
1126
1157
  * to evolve and should be considered only in the context of extremely
1127
1158
  * advanced performance tuning.
1128
1159
  *
1129
- * Zstd currently supports the use of a CDict in two ways:
1160
+ * Zstd currently supports the use of a CDict in three ways:
1130
1161
  *
1131
1162
  * - The contents of the CDict can be copied into the working context. This
1132
1163
  * means that the compression can search both the dictionary and input
@@ -1142,6 +1173,12 @@ typedef enum {
1142
1173
  * working context's tables can be reused). For small inputs, this can be
1143
1174
  * faster than copying the CDict's tables.
1144
1175
  *
1176
+ * - The CDict's tables are not used at all, and instead we use the working
1177
+ * context alone to reload the dictionary and use params based on the source
1178
+ * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
1179
+ * This method is effective when the dictionary sizes are very small relative
1180
+ * to the input size, and the input size is fairly large to begin with.
1181
+ *
1145
1182
  * Zstd has a simple internal heuristic that selects which strategy to use
1146
1183
  * at the beginning of a compression. However, if experimentation shows that
1147
1184
  * Zstd is making poor choices, it is possible to override that choice with
@@ -1150,6 +1187,7 @@ typedef enum {
1150
1187
  ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
1151
1188
  ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
1152
1189
  ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
1190
+ ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
1153
1191
  } ZSTD_dictAttachPref_e;
1154
1192
 
1155
1193
  typedef enum {
@@ -1158,7 +1196,7 @@ typedef enum {
1158
1196
  * levels will be compressed. */
1159
1197
  ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
1160
1198
  * emitted if Huffman compression is not profitable. */
1161
- ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */
1199
+ ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
1162
1200
  } ZSTD_literalCompressionMode_e;
1163
1201
 
1164
1202
 
@@ -1210,20 +1248,38 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS
1210
1248
  * or an error code (if srcSize is too small) */
1211
1249
  ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
1212
1250
 
1251
+ /*! ZSTD_getSequences() :
1252
+ * Extract sequences from the sequence store
1253
+ * zc can be used to insert custom compression params.
1254
+ * This function invokes ZSTD_compress2
1255
+ * @return : number of sequences extracted
1256
+ */
1257
+ ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
1258
+ size_t outSeqsSize, const void* src, size_t srcSize);
1259
+
1213
1260
 
1214
1261
  /***************************************
1215
1262
  * Memory management
1216
1263
  ***************************************/
1217
1264
 
1218
1265
  /*! ZSTD_estimate*() :
1219
- * These functions make it possible to estimate memory usage
1220
- * of a future {D,C}Ctx, before its creation.
1221
- * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
1222
- * It will also consider src size to be arbitrarily "large", which is worst case.
1223
- * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
1224
- * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
1225
- * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
1226
- * Note : CCtx size estimation is only correct for single-threaded compression. */
1266
+ * These functions make it possible to estimate memory usage of a future
1267
+ * {D,C}Ctx, before its creation.
1268
+ *
1269
+ * ZSTD_estimateCCtxSize() will provide a budget large enough for any
1270
+ * compression level up to selected one. Unlike ZSTD_estimateCStreamSize*(),
1271
+ * this estimate does not include space for a window buffer, so this estimate
1272
+ * is guaranteed to be enough for single-shot compressions, but not streaming
1273
+ * compressions. It will however assume the input may be arbitrarily large,
1274
+ * which is the worst case. If srcSize is known to always be small,
1275
+ * ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
1276
+ * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with
1277
+ * ZSTD_getCParams() to create cParams from compressionLevel.
1278
+ * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with
1279
+ * ZSTD_CCtxParams_setParameter().
1280
+ *
1281
+ * Note: only single-threaded compression is supported. This function will
1282
+ * return an error code if ZSTD_c_nbWorkers is >= 1. */
1227
1283
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
1228
1284
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
1229
1285
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
@@ -1334,7 +1390,8 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS
1334
1390
  * Create a digested dictionary for compression
1335
1391
  * Dictionary content is just referenced, not duplicated.
1336
1392
  * As a consequence, `dictBuffer` **must** outlive CDict,
1337
- * and its content must remain unmodified throughout the lifetime of CDict. */
1393
+ * and its content must remain unmodified throughout the lifetime of CDict.
1394
+ * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
1338
1395
  ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
1339
1396
 
1340
1397
  /*! ZSTD_getCParams() :
@@ -1361,7 +1418,9 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
1361
1418
  ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
1362
1419
 
1363
1420
  /*! ZSTD_compress_advanced() :
1364
- * Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */
1421
+ * Note : this function is now DEPRECATED.
1422
+ * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
1423
+ * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */
1365
1424
  ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
1366
1425
  void* dst, size_t dstCapacity,
1367
1426
  const void* src, size_t srcSize,
@@ -1369,7 +1428,9 @@ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
1369
1428
  ZSTD_parameters params);
1370
1429
 
1371
1430
  /*! ZSTD_compress_usingCDict_advanced() :
1372
- * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
1431
+ * Note : this function is now REDUNDANT.
1432
+ * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
1433
+ * This prototype will be marked as deprecated and generate compilation warning in some future version */
1373
1434
  ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
1374
1435
  void* dst, size_t dstCapacity,
1375
1436
  const void* src, size_t srcSize,
@@ -1441,6 +1502,12 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
1441
1502
  * There is no guarantee on compressed block size (default:0) */
1442
1503
  #define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
1443
1504
 
1505
+ /* User's best guess of source size.
1506
+ * Hint is not valid when srcSizeHint == 0.
1507
+ * There is no guarantee that hint is close to actual source size,
1508
+ * but compression ratio may regress significantly if guess considerably underestimates */
1509
+ #define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
1510
+
1444
1511
  /*! ZSTD_CCtx_getParameter() :
1445
1512
  * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
1446
1513
  * and store it into int* value.
@@ -1613,8 +1680,13 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
1613
1680
  * pledgedSrcSize must be correct. If it is not known at init time, use
1614
1681
  * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
1615
1682
  * "0" also disables frame content size field. It may be enabled in the future.
1683
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1616
1684
  */
1617
- ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
1685
+ ZSTDLIB_API size_t
1686
+ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
1687
+ int compressionLevel,
1688
+ unsigned long long pledgedSrcSize);
1689
+
1618
1690
  /**! ZSTD_initCStream_usingDict() :
1619
1691
  * This function is deprecated, and is equivalent to:
1620
1692
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -1623,42 +1695,66 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
1623
1695
  *
1624
1696
  * Creates of an internal CDict (incompatible with static CCtx), except if
1625
1697
  * dict == NULL or dictSize < 8, in which case no dict is used.
1626
- * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
1698
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
1627
1699
  * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
1700
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1628
1701
  */
1629
- ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
1702
+ ZSTDLIB_API size_t
1703
+ ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
1704
+ const void* dict, size_t dictSize,
1705
+ int compressionLevel);
1706
+
1630
1707
  /**! ZSTD_initCStream_advanced() :
1631
1708
  * This function is deprecated, and is approximately equivalent to:
1632
1709
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1633
- * ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is
1710
+ * // Pseudocode: Set each zstd parameter and leave the rest as-is.
1711
+ * for ((param, value) : params) {
1712
+ * ZSTD_CCtx_setParameter(zcs, param, value);
1713
+ * }
1634
1714
  * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
1635
1715
  * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
1636
1716
  *
1637
- * pledgedSrcSize must be correct. If srcSize is not known at init time, use
1638
- * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
1717
+ * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
1718
+ * pledgedSrcSize must be correct.
1719
+ * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
1720
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1639
1721
  */
1640
- ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
1641
- ZSTD_parameters params, unsigned long long pledgedSrcSize);
1722
+ ZSTDLIB_API size_t
1723
+ ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
1724
+ const void* dict, size_t dictSize,
1725
+ ZSTD_parameters params,
1726
+ unsigned long long pledgedSrcSize);
1727
+
1642
1728
  /**! ZSTD_initCStream_usingCDict() :
1643
1729
  * This function is deprecated, and equivalent to:
1644
1730
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1645
1731
  * ZSTD_CCtx_refCDict(zcs, cdict);
1646
1732
  *
1647
1733
  * note : cdict will just be referenced, and must outlive compression session
1734
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1648
1735
  */
1649
1736
  ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
1737
+
1650
1738
  /**! ZSTD_initCStream_usingCDict_advanced() :
1651
- * This function is deprecated, and is approximately equivalent to:
1739
+ * This function is DEPRECATED, and is approximately equivalent to:
1652
1740
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1653
- * ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is
1741
+ * // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
1742
+ * for ((fParam, value) : fParams) {
1743
+ * ZSTD_CCtx_setParameter(zcs, fParam, value);
1744
+ * }
1654
1745
  * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
1655
1746
  * ZSTD_CCtx_refCDict(zcs, cdict);
1656
1747
  *
1657
1748
  * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
1658
1749
  * pledgedSrcSize must be correct. If srcSize is not known at init time, use
1659
1750
  * value ZSTD_CONTENTSIZE_UNKNOWN.
1751
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1660
1752
  */
1661
- ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
1753
+ ZSTDLIB_API size_t
1754
+ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
1755
+ const ZSTD_CDict* cdict,
1756
+ ZSTD_frameParameters fParams,
1757
+ unsigned long long pledgedSrcSize);
1662
1758
 
1663
1759
  /*! ZSTD_resetCStream() :
1664
1760
  * This function is deprecated, and is equivalent to:
@@ -1673,6 +1769,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
1673
1769
  * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
1674
1770
  * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
1675
1771
  * @return : 0, or an error code (which can be tested using ZSTD_isError())
1772
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1676
1773
  */
1677
1774
  ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
1678
1775
 
@@ -1718,8 +1815,10 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
1718
1815
  * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
1719
1816
  *
1720
1817
  * note: no dictionary will be used if dict == NULL or dictSize < 8
1818
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1721
1819
  */
1722
1820
  ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
1821
+
1723
1822
  /**
1724
1823
  * This function is deprecated, and is equivalent to:
1725
1824
  *
@@ -1727,14 +1826,17 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic
1727
1826
  * ZSTD_DCtx_refDDict(zds, ddict);
1728
1827
  *
1729
1828
  * note : ddict is referenced, it must outlive decompression session
1829
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1730
1830
  */
1731
1831
  ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
1832
+
1732
1833
  /**
1733
1834
  * This function is deprecated, and is equivalent to:
1734
1835
  *
1735
1836
  * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
1736
1837
  *
1737
1838
  * re-use decompression parameters from previous init; saves dictionary loading
1839
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1738
1840
  */
1739
1841
  ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
1740
1842
 
@@ -1908,7 +2010,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
1908
2010
 
1909
2011
  /*!
1910
2012
  Block functions produce and decode raw zstd blocks, without frame metadata.
1911
- Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
2013
+ Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
1912
2014
  But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
1913
2015
 
1914
2016
  A few rules to respect :