zstd-ruby 1.4.0.0 → 1.4.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +35 -0
  3. data/README.md +2 -2
  4. data/ext/zstdruby/libzstd/Makefile +274 -107
  5. data/ext/zstdruby/libzstd/README.md +75 -16
  6. data/ext/zstdruby/libzstd/common/bitstream.h +59 -51
  7. data/ext/zstdruby/libzstd/common/compiler.h +154 -5
  8. data/ext/zstdruby/libzstd/common/cpu.h +1 -3
  9. data/ext/zstdruby/libzstd/common/debug.c +11 -31
  10. data/ext/zstdruby/libzstd/common/debug.h +22 -49
  11. data/ext/zstdruby/libzstd/common/entropy_common.c +201 -75
  12. data/ext/zstdruby/libzstd/common/error_private.c +3 -1
  13. data/ext/zstdruby/libzstd/common/error_private.h +7 -3
  14. data/ext/zstdruby/libzstd/common/fse.h +50 -42
  15. data/ext/zstdruby/libzstd/common/fse_decompress.c +134 -50
  16. data/ext/zstdruby/libzstd/common/huf.h +41 -38
  17. data/ext/zstdruby/libzstd/common/mem.h +68 -22
  18. data/ext/zstdruby/libzstd/common/pool.c +30 -20
  19. data/ext/zstdruby/libzstd/common/pool.h +3 -3
  20. data/ext/zstdruby/libzstd/common/threading.c +51 -4
  21. data/ext/zstdruby/libzstd/common/threading.h +36 -4
  22. data/ext/zstdruby/libzstd/common/xxhash.c +39 -89
  23. data/ext/zstdruby/libzstd/common/xxhash.h +12 -32
  24. data/ext/zstdruby/libzstd/common/zstd_common.c +10 -10
  25. data/ext/zstdruby/libzstd/common/zstd_deps.h +111 -0
  26. data/ext/zstdruby/libzstd/common/zstd_errors.h +3 -1
  27. data/ext/zstdruby/libzstd/common/zstd_internal.h +231 -72
  28. data/ext/zstdruby/libzstd/common/zstd_trace.c +42 -0
  29. data/ext/zstdruby/libzstd/common/zstd_trace.h +152 -0
  30. data/ext/zstdruby/libzstd/compress/fse_compress.c +47 -63
  31. data/ext/zstdruby/libzstd/compress/hist.c +41 -63
  32. data/ext/zstdruby/libzstd/compress/hist.h +13 -33
  33. data/ext/zstdruby/libzstd/compress/huf_compress.c +288 -172
  34. data/ext/zstdruby/libzstd/compress/zstd_compress.c +2504 -1626
  35. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +446 -85
  36. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +158 -0
  37. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +29 -0
  38. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +433 -0
  39. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +54 -0
  40. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +849 -0
  41. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
  42. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +561 -0
  43. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +82 -60
  44. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
  45. data/ext/zstdruby/libzstd/compress/zstd_fast.c +106 -80
  46. data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
  47. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +411 -105
  48. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +21 -1
  49. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +296 -207
  50. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +14 -3
  51. data/ext/zstdruby/libzstd/compress/zstd_ldm_geartab.h +103 -0
  52. data/ext/zstdruby/libzstd/compress/zstd_opt.c +260 -148
  53. data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
  54. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +153 -440
  55. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +29 -110
  56. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +356 -238
  57. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +20 -16
  58. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +3 -3
  59. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +641 -238
  60. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +600 -371
  61. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +8 -5
  62. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +40 -9
  63. data/ext/zstdruby/libzstd/deprecated/zbuff.h +9 -8
  64. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
  65. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
  66. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
  67. data/ext/zstdruby/libzstd/dictBuilder/cover.c +197 -78
  68. data/ext/zstdruby/libzstd/dictBuilder/cover.h +52 -7
  69. data/ext/zstdruby/libzstd/dictBuilder/divsufsort.c +1 -1
  70. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +84 -66
  71. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +58 -36
  72. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +60 -31
  73. data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
  74. data/ext/zstdruby/libzstd/dll/example/README.md +16 -22
  75. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +8 -4
  76. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +115 -111
  77. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +1 -1
  78. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +28 -14
  79. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +1 -1
  80. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +28 -14
  81. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +1 -1
  82. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +36 -19
  83. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +1 -1
  84. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +122 -107
  85. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +2 -2
  86. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +29 -23
  87. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +1 -1
  88. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +34 -24
  89. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +1 -1
  90. data/ext/zstdruby/libzstd/libzstd.pc.in +2 -1
  91. data/ext/zstdruby/libzstd/zstd.h +655 -118
  92. data/lib/zstd-ruby/version.rb +1 -1
  93. data/zstd-ruby.gemspec +1 -1
  94. metadata +20 -10
  95. data/.travis.yml +0 -14
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -19,7 +19,7 @@ extern "C" {
19
19
  * Dependencies
20
20
  ***************************************/
21
21
  #include <stddef.h> /* size_t */
22
- #include "mem.h" /* U64, U32 */
22
+ #include "../common/mem.h" /* U64, U32 */
23
23
 
24
24
 
25
25
  /* *************************************
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +14,7 @@
14
14
  #include <stddef.h> /* size_t, ptrdiff_t */
15
15
  #include <string.h> /* memcpy */
16
16
  #include <stdlib.h> /* malloc, free, qsort */
17
- #include "error_private.h"
17
+ #include "../common/error_private.h"
18
18
 
19
19
 
20
20
 
@@ -82,7 +82,11 @@ extern "C" {
82
82
  * Basic Types
83
83
  *****************************************************************/
84
84
  #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
85
- # include <stdint.h>
85
+ # if defined(_AIX)
86
+ # include <inttypes.h>
87
+ # else
88
+ # include <stdint.h> /* intptr_t */
89
+ # endif
86
90
  typedef uint8_t BYTE;
87
91
  typedef uint16_t U16;
88
92
  typedef int16_t S16;
@@ -860,7 +864,7 @@ MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
860
864
  _BitScanReverse ( &r, val );
861
865
  return (unsigned) r;
862
866
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
863
- return 31 - __builtin_clz (val);
867
+ return __builtin_clz (val) ^ 31;
864
868
  # else /* Software version */
865
869
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
866
870
  U32 v = val;
@@ -1862,7 +1866,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
1862
1866
 
1863
1867
  if (!srcSize) return ERROR(srcSize_wrong);
1864
1868
  iSize = ip[0];
1865
- //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1869
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
1866
1870
 
1867
1871
  if (iSize >= 128) { /* special header */
1868
1872
  if (iSize >= (242)) { /* RLE */
@@ -2014,7 +2018,7 @@ size_t HUFv06_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
2014
2018
  HUFv06_DEltX2* const dt = (HUFv06_DEltX2*)dtPtr;
2015
2019
 
2016
2020
  HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
2017
- //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
2021
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
2018
2022
 
2019
2023
  iSize = HUFv06_readStats(huffWeight, HUFv06_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
2020
2024
  if (HUFv06_isError(iSize)) return iSize;
@@ -2340,7 +2344,7 @@ size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2340
2344
 
2341
2345
  HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
2342
2346
  if (memLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
2343
- //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2347
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
2344
2348
 
2345
2349
  iSize = HUFv06_readStats(weightList, HUFv06_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
2346
2350
  if (HUFv06_isError(iSize)) return iSize;
@@ -2664,13 +2668,13 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2664
2668
 
2665
2669
  { U32 algoNb = 0;
2666
2670
  if (Dtime[1] < Dtime[0]) algoNb = 1;
2667
- // if (Dtime[2] < Dtime[algoNb]) algoNb = 2; /* current speed of HUFv06_decompress4X6 is not good */
2671
+ /* if (Dtime[2] < Dtime[algoNb]) algoNb = 2; */ /* current speed of HUFv06_decompress4X6 is not good */
2668
2672
  return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
2669
2673
  }
2670
2674
 
2671
- //return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
2672
- //return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
2673
- //return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); /* multi-streams quad-symbols decoding */
2675
+ /* return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
2676
+ /* return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
2677
+ /* return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams quad-symbols decoding */
2674
2678
  }
2675
2679
  /*
2676
2680
  Common functions of Zstd compression library
@@ -3025,7 +3029,7 @@ typedef struct
3025
3029
  * Provides the size of compressed block from block header `src` */
3026
3030
  static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
3027
3031
  {
3028
- const BYTE* const in = (const BYTE* const)src;
3032
+ const BYTE* const in = (const BYTE*)src;
3029
3033
  U32 cSize;
3030
3034
 
3031
3035
  if (srcSize < ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
@@ -3219,7 +3223,7 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3219
3223
  FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
3220
3224
  const void* src, size_t srcSize)
3221
3225
  {
3222
- const BYTE* const istart = (const BYTE* const)src;
3226
+ const BYTE* const istart = (const BYTE*)src;
3223
3227
  const BYTE* const iend = istart + srcSize;
3224
3228
  const BYTE* ip = istart;
3225
3229
 
@@ -3242,14 +3246,12 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3242
3246
  }
3243
3247
 
3244
3248
  /* FSE table descriptors */
3249
+ if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
3245
3250
  { U32 const LLtype = *ip >> 6;
3246
3251
  U32 const Offtype = (*ip >> 4) & 3;
3247
3252
  U32 const MLtype = (*ip >> 2) & 3;
3248
3253
  ip++;
3249
3254
 
3250
- /* check */
3251
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
3252
-
3253
3255
  /* Build DTables */
3254
3256
  { size_t const bhSize = ZSTDv06_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
3255
3257
  if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
@@ -3443,7 +3445,7 @@ static size_t ZSTDv06_decompressSequences(
3443
3445
  {
3444
3446
  const BYTE* ip = (const BYTE*)seqStart;
3445
3447
  const BYTE* const iend = ip + seqSize;
3446
- BYTE* const ostart = (BYTE* const)dst;
3448
+ BYTE* const ostart = (BYTE*)dst;
3447
3449
  BYTE* const oend = ostart + maxDstSize;
3448
3450
  BYTE* op = ostart;
3449
3451
  const BYTE* litPtr = dctx->litPtr;
@@ -3503,8 +3505,10 @@ static size_t ZSTDv06_decompressSequences(
3503
3505
  { size_t const lastLLSize = litEnd - litPtr;
3504
3506
  if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
3505
3507
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
3506
- memcpy(op, litPtr, lastLLSize);
3507
- op += lastLLSize;
3508
+ if (lastLLSize > 0) {
3509
+ memcpy(op, litPtr, lastLLSize);
3510
+ op += lastLLSize;
3511
+ }
3508
3512
  }
3509
3513
 
3510
3514
  return op-ostart;
@@ -3557,7 +3561,7 @@ static size_t ZSTDv06_decompressFrame(ZSTDv06_DCtx* dctx,
3557
3561
  {
3558
3562
  const BYTE* ip = (const BYTE*)src;
3559
3563
  const BYTE* const iend = ip + srcSize;
3560
- BYTE* const ostart = (BYTE* const)dst;
3564
+ BYTE* const ostart = (BYTE*)dst;
3561
3565
  BYTE* op = ostart;
3562
3566
  BYTE* const oend = ostart + dstCapacity;
3563
3567
  size_t remainingSize = srcSize;
@@ -3672,7 +3676,7 @@ void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cS
3672
3676
  blockProperties_t blockProperties = { bt_compressed, 0 };
3673
3677
 
3674
3678
  /* Frame Header */
3675
- { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
3679
+ { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, srcSize);
3676
3680
  if (ZSTDv06_isError(frameHeaderSize)) {
3677
3681
  ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
3678
3682
  return;
@@ -4002,7 +4006,9 @@ size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* zbd)
4002
4006
  MEM_STATIC size_t ZBUFFv06_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
4003
4007
  {
4004
4008
  size_t length = MIN(dstCapacity, srcSize);
4005
- memcpy(dst, src, length);
4009
+ if (length > 0) {
4010
+ memcpy(dst, src, length);
4011
+ }
4006
4012
  return length;
4007
4013
  }
4008
4014
 
@@ -4111,7 +4117,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4111
4117
  if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
4112
4118
  zbd->outEnd = zbd->outStart + decodedSize;
4113
4119
  zbd->stage = ZBUFFds_flush;
4114
- // break; /* ZBUFFds_flush follows */
4120
+ /* break; */ /* ZBUFFds_flush follows */
4115
4121
  }
4116
4122
  }
4117
4123
  /* fall-through */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -17,14 +17,14 @@
17
17
  #ifndef XXH_STATIC_LINKING_ONLY
18
18
  # define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
19
19
  #endif
20
- #include "xxhash.h" /* XXH64_* */
20
+ #include "../common/xxhash.h" /* XXH64_* */
21
21
  #include "zstd_v07.h"
22
22
 
23
23
  #define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
24
24
  #define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
25
25
  #define ZSTDv07_STATIC_LINKING_ONLY
26
26
 
27
- #include "error_private.h"
27
+ #include "../common/error_private.h"
28
28
 
29
29
 
30
30
  #ifdef ZSTDv07_STATIC_LINKING_ONLY
@@ -242,7 +242,11 @@ extern "C" {
242
242
  * Basic Types
243
243
  *****************************************************************/
244
244
  #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
245
- # include <stdint.h>
245
+ # if defined(_AIX)
246
+ # include <inttypes.h>
247
+ # else
248
+ # include <stdint.h> /* intptr_t */
249
+ # endif
246
250
  typedef uint8_t BYTE;
247
251
  typedef uint16_t U16;
248
252
  typedef int16_t S16;
@@ -530,7 +534,7 @@ MEM_STATIC unsigned BITv07_highbit32 (U32 val)
530
534
  _BitScanReverse ( &r, val );
531
535
  return (unsigned) r;
532
536
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
533
- return 31 - __builtin_clz (val);
537
+ return __builtin_clz (val) ^ 31;
534
538
  # else /* Software version */
535
539
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
536
540
  U32 v = val;
@@ -1314,7 +1318,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
1314
1318
 
1315
1319
  if (!srcSize) return ERROR(srcSize_wrong);
1316
1320
  iSize = ip[0];
1317
- //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1321
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
1318
1322
 
1319
1323
  if (iSize >= 128) { /* special header */
1320
1324
  if (iSize >= (242)) { /* RLE */
@@ -1784,7 +1788,7 @@ size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSi
1784
1788
  HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr;
1785
1789
 
1786
1790
  HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable));
1787
- //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
1791
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
1788
1792
 
1789
1793
  iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
1790
1794
  if (HUFv07_isError(iSize)) return iSize;
@@ -2148,7 +2152,7 @@ size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSi
2148
2152
 
2149
2153
  HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */
2150
2154
  if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
2151
- //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2155
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
2152
2156
 
2153
2157
  iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
2154
2158
  if (HUFv07_isError(iSize)) return iSize;
@@ -2530,8 +2534,8 @@ size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2530
2534
  return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
2531
2535
  }
2532
2536
 
2533
- //return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
2534
- //return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
2537
+ /* return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
2538
+ /* return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
2535
2539
  }
2536
2540
 
2537
2541
  size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
@@ -3254,7 +3258,7 @@ typedef struct
3254
3258
  * Provides the size of compressed block from block header `src` */
3255
3259
  static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
3256
3260
  {
3257
- const BYTE* const in = (const BYTE* const)src;
3261
+ const BYTE* const in = (const BYTE*)src;
3258
3262
  U32 cSize;
3259
3263
 
3260
3264
  if (srcSize < ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
@@ -3272,7 +3276,9 @@ static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProper
3272
3276
  static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
3273
3277
  {
3274
3278
  if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
3275
- memcpy(dst, src, srcSize);
3279
+ if (srcSize > 0) {
3280
+ memcpy(dst, src, srcSize);
3281
+ }
3276
3282
  return srcSize;
3277
3283
  }
3278
3284
 
@@ -3447,7 +3453,7 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
3447
3453
  FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable,
3448
3454
  const void* src, size_t srcSize)
3449
3455
  {
3450
- const BYTE* const istart = (const BYTE* const)src;
3456
+ const BYTE* const istart = (const BYTE*)src;
3451
3457
  const BYTE* const iend = istart + srcSize;
3452
3458
  const BYTE* ip = istart;
3453
3459
 
@@ -3470,14 +3476,12 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
3470
3476
  }
3471
3477
 
3472
3478
  /* FSE table descriptors */
3479
+ if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
3473
3480
  { U32 const LLtype = *ip >> 6;
3474
3481
  U32 const OFtype = (*ip >> 4) & 3;
3475
3482
  U32 const MLtype = (*ip >> 2) & 3;
3476
3483
  ip++;
3477
3484
 
3478
- /* check */
3479
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
3480
-
3481
3485
  /* Build DTables */
3482
3486
  { size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
3483
3487
  if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected);
@@ -3668,7 +3672,7 @@ static size_t ZSTDv07_decompressSequences(
3668
3672
  {
3669
3673
  const BYTE* ip = (const BYTE*)seqStart;
3670
3674
  const BYTE* const iend = ip + seqSize;
3671
- BYTE* const ostart = (BYTE* const)dst;
3675
+ BYTE* const ostart = (BYTE*)dst;
3672
3676
  BYTE* const oend = ostart + maxDstSize;
3673
3677
  BYTE* op = ostart;
3674
3678
  const BYTE* litPtr = dctx->litPtr;
@@ -3714,10 +3718,12 @@ static size_t ZSTDv07_decompressSequences(
3714
3718
 
3715
3719
  /* last literal segment */
3716
3720
  { size_t const lastLLSize = litEnd - litPtr;
3717
- //if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
3721
+ /* if (litPtr > litEnd) return ERROR(corruption_detected); */ /* too many literals already used */
3718
3722
  if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
3719
- memcpy(op, litPtr, lastLLSize);
3720
- op += lastLLSize;
3723
+ if (lastLLSize > 0) {
3724
+ memcpy(op, litPtr, lastLLSize);
3725
+ op += lastLLSize;
3726
+ }
3721
3727
  }
3722
3728
 
3723
3729
  return op-ostart;
@@ -3778,7 +3784,9 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
3778
3784
  static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
3779
3785
  {
3780
3786
  if (length > dstCapacity) return ERROR(dstSize_tooSmall);
3781
- memset(dst, byte, length);
3787
+ if (length > 0) {
3788
+ memset(dst, byte, length);
3789
+ }
3782
3790
  return length;
3783
3791
  }
3784
3792
 
@@ -3791,7 +3799,7 @@ static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx,
3791
3799
  {
3792
3800
  const BYTE* ip = (const BYTE*)src;
3793
3801
  const BYTE* const iend = ip + srcSize;
3794
- BYTE* const ostart = (BYTE* const)dst;
3802
+ BYTE* const ostart = (BYTE*)dst;
3795
3803
  BYTE* const oend = ostart + dstCapacity;
3796
3804
  BYTE* op = ostart;
3797
3805
  size_t remainingSize = srcSize;
@@ -3918,7 +3926,7 @@ void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cS
3918
3926
  }
3919
3927
 
3920
3928
  /* Frame Header */
3921
- { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
3929
+ { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, srcSize);
3922
3930
  if (ZSTDv07_isError(frameHeaderSize)) {
3923
3931
  ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
3924
3932
  return;
@@ -4380,7 +4388,9 @@ size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd)
4380
4388
  MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
4381
4389
  {
4382
4390
  size_t const length = MIN(dstCapacity, srcSize);
4383
- memcpy(dst, src, length);
4391
+ if (length > 0) {
4392
+ memcpy(dst, src, length);
4393
+ }
4384
4394
  return length;
4385
4395
  }
4386
4396
 
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -3,8 +3,9 @@
3
3
  # BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
4
4
 
5
5
  prefix=@PREFIX@
6
- libdir=@LIBDIR@
6
+ exec_prefix=@EXEC_PREFIX@
7
7
  includedir=@INCLUDEDIR@
8
+ libdir=@LIBDIR@
8
9
 
9
10
  Name: zstd
10
11
  Description: fast lossless compression algorithm library
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -15,6 +15,7 @@ extern "C" {
15
15
  #define ZSTD_H_235446
16
16
 
17
17
  /* ====== Dependency ======*/
18
+ #include <limits.h> /* INT_MAX */
18
19
  #include <stddef.h> /* size_t */
19
20
 
20
21
 
@@ -71,27 +72,32 @@ extern "C" {
71
72
  /*------ Version ------*/
72
73
  #define ZSTD_VERSION_MAJOR 1
73
74
  #define ZSTD_VERSION_MINOR 4
74
- #define ZSTD_VERSION_RELEASE 0
75
-
75
+ #define ZSTD_VERSION_RELEASE 9
76
76
  #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
77
- ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */
77
+
78
+ /*! ZSTD_versionNumber() :
79
+ * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */
80
+ ZSTDLIB_API unsigned ZSTD_versionNumber(void);
78
81
 
79
82
  #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
80
83
  #define ZSTD_QUOTE(str) #str
81
84
  #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
82
85
  #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
83
- ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */
84
86
 
85
- /***************************************
86
- * Default constant
87
- ***************************************/
87
+ /*! ZSTD_versionString() :
88
+ * Return runtime library version, like "1.4.5". Requires v1.3.0+. */
89
+ ZSTDLIB_API const char* ZSTD_versionString(void);
90
+
91
+ /* *************************************
92
+ * Default constant
93
+ ***************************************/
88
94
  #ifndef ZSTD_CLEVEL_DEFAULT
89
95
  # define ZSTD_CLEVEL_DEFAULT 3
90
96
  #endif
91
97
 
92
- /***************************************
93
- * Constants
94
- ***************************************/
98
+ /* *************************************
99
+ * Constants
100
+ ***************************************/
95
101
 
96
102
  /* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
97
103
  #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
@@ -183,17 +189,26 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres
183
189
  ***************************************/
184
190
  /*= Compression context
185
191
  * When compressing many times,
186
- * it is recommended to allocate a context just once, and re-use it for each successive compression operation.
192
+ * it is recommended to allocate a context just once,
193
+ * and re-use it for each successive compression operation.
187
194
  * This will make workload friendlier for system's memory.
188
- * Use one context per thread for parallel execution in multi-threaded environments. */
195
+ * Note : re-using context is just a speed / resource optimization.
196
+ * It doesn't change the compression ratio, which remains identical.
197
+ * Note 2 : In multi-threaded environments,
198
+ * use one different context per thread for parallel execution.
199
+ */
189
200
  typedef struct ZSTD_CCtx_s ZSTD_CCtx;
190
201
  ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
191
202
  ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
192
203
 
193
204
  /*! ZSTD_compressCCtx() :
194
- * Same as ZSTD_compress(), using an explicit ZSTD_CCtx
195
- * The function will compress at requested compression level,
196
- * ignoring any other parameter */
205
+ * Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
206
+ * Important : in order to behave similarly to `ZSTD_compress()`,
207
+ * this function compresses at requested compression level,
208
+ * __ignoring any other parameter__ .
209
+ * If any advanced parameter was set using the advanced API,
210
+ * they will all be reset. Only `compressionLevel` remains.
211
+ */
197
212
  ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
198
213
  void* dst, size_t dstCapacity,
199
214
  const void* src, size_t srcSize,
@@ -228,7 +243,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
228
243
  * using ZSTD_CCtx_set*() functions.
229
244
  * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
230
245
  * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
231
- * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()
246
+ * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
232
247
  *
233
248
  * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
234
249
  *
@@ -256,18 +271,29 @@ typedef enum {
256
271
 
257
272
  /* compression parameters
258
273
  * Note: When compressing with a ZSTD_CDict these parameters are superseded
259
- * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
260
- * for more info (superseded-by-cdict). */
261
- ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
274
+ * by the parameters used to construct the ZSTD_CDict.
275
+ * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
276
+ ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
277
+ * Note that exact compression parameters are dynamically determined,
278
+ * depending on both compression level and srcSize (when known).
262
279
  * Default level is ZSTD_CLEVEL_DEFAULT==3.
263
280
  * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
264
281
  * Note 1 : it's possible to pass a negative compression level.
265
- * Note 2 : setting a level sets all default values of other compression parameters */
282
+ * Note 2 : setting a level does not automatically set all other compression parameters
283
+ * to default. Setting this will however eventually dynamically impact the compression
284
+ * parameters which have not been manually set. The manually set
285
+ * ones will 'stick'. */
286
+ /* Advanced compression parameters :
287
+ * It's possible to pin down compression parameters to some specific values.
288
+ * In which case, these values are no longer dynamically selected by the compressor */
266
289
  ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
290
+ * This will set a memory budget for streaming decompression,
291
+ * with larger values requiring more memory
292
+ * and typically compressing more.
267
293
  * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
268
294
  * Special: value 0 means "use default windowLog".
269
295
  * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
270
- * requires explicitly allowing such window size at decompression stage if using streaming. */
296
+ * requires explicitly allowing such size at streaming decompression stage. */
271
297
  ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
272
298
  * Resulting memory usage is (1 << (hashLog+2)).
273
299
  * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
@@ -278,13 +304,13 @@ typedef enum {
278
304
  * Resulting memory usage is (1 << (chainLog+2)).
279
305
  * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
280
306
  * Larger tables result in better and slower compression.
281
- * This parameter is useless when using "fast" strategy.
307
+ * This parameter is useless for "fast" strategy.
282
308
  * It's still useful when using "dfast" strategy,
283
309
  * in which case it defines a secondary probe table.
284
310
  * Special: value 0 means "use default chainLog". */
285
311
  ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
286
312
  * More attempts result in better and slower compression.
287
- * This parameter is useless when using "fast" and "dFast" strategies.
313
+ * This parameter is useless for "fast" and "dFast" strategies.
288
314
  * Special: value 0 means "use default searchLog". */
289
315
  ZSTD_c_minMatch=105, /* Minimum size of searched matches.
290
316
  * Note that Zstandard can still find matches of smaller size,
@@ -313,7 +339,9 @@ typedef enum {
313
339
  * for large inputs, by finding large matches at long distance.
314
340
  * It increases memory usage and window size.
315
341
  * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
316
- * except when expressly set to a different value. */
342
+ * except when expressly set to a different value.
343
+ * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
344
+ * compression strategy >= ZSTD_btopt (== compression level 16+) */
317
345
  ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
318
346
  * Larger values increase memory usage and compression ratio,
319
347
  * but decrease compression speed.
@@ -339,26 +367,30 @@ typedef enum {
339
367
  ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
340
368
  * Content size must be known at the beginning of compression.
341
369
  * This is automatically the case when using ZSTD_compress2(),
342
- * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
370
+ * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
343
371
  ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
344
372
  ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
345
373
 
346
374
  /* multi-threading parameters */
347
- /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
348
- * They return an error otherwise. */
375
+ /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
376
+ * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
377
+ * In a situation where it's unknown if the linked library supports multi-threading or not,
378
+ * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
379
+ */
349
380
  ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
350
- * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :
381
+ * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
351
382
  * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
352
- * while compression work is performed in parallel, within worker threads.
383
+ * while compression is performed in parallel, within worker thread(s).
353
384
  * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
354
385
  * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
355
386
  * More workers improve speed, but also increase memory usage.
356
- * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
387
+ * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
388
+ * compression is performed inside Caller's thread, and all invocations are blocking */
357
389
  ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
358
390
  * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
359
391
  * 0 means default, which is dynamically determined based on compression parameters.
360
392
  * Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
361
- * The minimum size is automatically and transparently enforced */
393
+ * The minimum size is automatically and transparently enforced. */
362
394
  ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
363
395
  * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
364
396
  * It helps preserve compression ratio, while each job is compressed in parallel.
@@ -380,6 +412,13 @@ typedef enum {
380
412
  * ZSTD_c_forceMaxWindow
381
413
  * ZSTD_c_forceAttachDict
382
414
  * ZSTD_c_literalCompressionMode
415
+ * ZSTD_c_targetCBlockSize
416
+ * ZSTD_c_srcSizeHint
417
+ * ZSTD_c_enableDedicatedDictSearch
418
+ * ZSTD_c_stableInBuffer
419
+ * ZSTD_c_stableOutBuffer
420
+ * ZSTD_c_blockDelimiters
421
+ * ZSTD_c_validateSequences
383
422
  * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
384
423
  * note : never ever use experimentalParam? names directly;
385
424
  * also, the enums values themselves are unstable and can still change.
@@ -389,6 +428,13 @@ typedef enum {
389
428
  ZSTD_c_experimentalParam3=1000,
390
429
  ZSTD_c_experimentalParam4=1001,
391
430
  ZSTD_c_experimentalParam5=1002,
431
+ ZSTD_c_experimentalParam6=1003,
432
+ ZSTD_c_experimentalParam7=1004,
433
+ ZSTD_c_experimentalParam8=1005,
434
+ ZSTD_c_experimentalParam9=1006,
435
+ ZSTD_c_experimentalParam10=1007,
436
+ ZSTD_c_experimentalParam11=1008,
437
+ ZSTD_c_experimentalParam12=1009
392
438
  } ZSTD_cParameter;
393
439
 
394
440
  typedef struct {
@@ -497,11 +543,17 @@ typedef enum {
497
543
  /* note : additional experimental parameters are also available
498
544
  * within the experimental section of the API.
499
545
  * At the time of this writing, they include :
500
- * ZSTD_c_format
546
+ * ZSTD_d_format
547
+ * ZSTD_d_stableOutBuffer
548
+ * ZSTD_d_forceIgnoreChecksum
549
+ * ZSTD_d_refMultipleDDicts
501
550
  * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
502
551
  * note : never ever use experimentalParam? names directly
503
552
  */
504
- ZSTD_d_experimentalParam1=1000
553
+ ZSTD_d_experimentalParam1=1000,
554
+ ZSTD_d_experimentalParam2=1001,
555
+ ZSTD_d_experimentalParam3=1002,
556
+ ZSTD_d_experimentalParam4=1003
505
557
 
506
558
  } ZSTD_dParameter;
507
559
 
@@ -637,8 +689,9 @@ typedef enum {
637
689
  * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
638
690
  * - output->pos must be <= dstCapacity, input->pos must be <= srcSize
639
691
  * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
692
+ * - endOp must be a valid directive
640
693
  * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
641
- * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
694
+ * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
642
695
  * and then immediately returns, just indicating that there is some data remaining to be flushed.
643
696
  * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
644
697
  * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
@@ -657,17 +710,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
657
710
  ZSTD_inBuffer* input,
658
711
  ZSTD_EndDirective endOp);
659
712
 
713
+
714
+ /* These buffer sizes are softly recommended.
715
+ * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
716
+ * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
717
+ * reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
718
+ *
719
+ * However, note that these recommendations are from the perspective of a C caller program.
720
+ * If the streaming interface is invoked from some other language,
721
+ * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
722
+ * a major performance rule is to reduce crossing such interface to an absolute minimum.
723
+ * It's not rare that performance ends being spent more into the interface, rather than compression itself.
724
+ * In which cases, prefer using large buffers, as large as practical,
725
+ * for both input and output, to reduce the nb of roundtrips.
726
+ */
660
727
  ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
661
- ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
728
+ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
662
729
 
663
- /*******************************************************************************
664
- * This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and
665
- * ZSTD_compressStream2(). It is redundant, but is still fully supported.
730
+
731
+ /* *****************************************************************************
732
+ * This following is a legacy streaming API.
733
+ * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
734
+ * It is redundant, but remains fully supported.
666
735
  * Advanced parameters and dictionary compression can only be used through the
667
736
  * new API.
668
737
  ******************************************************************************/
669
738
 
670
- /**
739
+ /*!
671
740
  * Equivalent to:
672
741
  *
673
742
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -675,16 +744,16 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
675
744
  * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
676
745
  */
677
746
  ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
678
- /**
747
+ /*!
679
748
  * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
680
749
  * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
681
750
  * the next read size (if non-zero and not an error). ZSTD_compressStream2()
682
- * returns the number of bytes left to flush (if non-zero and not an error).
751
+ * returns the minimum nb of bytes left to flush (if non-zero and not an error).
683
752
  */
684
753
  ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
685
- /** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
754
+ /*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
686
755
  ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
687
- /** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
756
+ /*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
688
757
  ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
689
758
 
690
759
 
@@ -725,7 +794,7 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
725
794
 
726
795
  /* This function is redundant with the advanced API and equivalent to:
727
796
  *
728
- * ZSTD_DCtx_reset(zds);
797
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
729
798
  * ZSTD_DCtx_refDDict(zds, NULL);
730
799
  */
731
800
  ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
@@ -770,12 +839,17 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
770
839
  typedef struct ZSTD_CDict_s ZSTD_CDict;
771
840
 
772
841
  /*! ZSTD_createCDict() :
773
- * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
774
- * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
842
+ * When compressing multiple messages or blocks using the same dictionary,
843
+ * it's recommended to digest the dictionary only once, since it's a costly operation.
844
+ * ZSTD_createCDict() will create a state from digesting a dictionary.
845
+ * The resulting state can be used for future compression operations with very limited startup cost.
775
846
  * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
776
- * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
777
- * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
778
- * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
847
+ * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
848
+ * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
849
+ * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
850
+ * in which case the only thing that it transports is the @compressionLevel.
851
+ * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
852
+ * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
779
853
  ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
780
854
  int compressionLevel);
781
855
 
@@ -876,7 +950,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
876
950
  * Reference a prepared dictionary, to be used for all next compressed frames.
877
951
  * Note that compression parameters are enforced from within CDict,
878
952
  * and supersede any compression parameter previously set within CCtx.
879
- * The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
953
+ * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
880
954
  * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
881
955
  * The dictionary will remain valid for future compressed frames using same CCtx.
882
956
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
@@ -902,7 +976,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
902
976
  * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
903
977
  * It's a CPU consuming operation, with non-negligible impact on latency.
904
978
  * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
905
- * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
979
+ * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
906
980
  * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
907
981
  ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
908
982
  const void* prefix, size_t prefixSize);
@@ -927,6 +1001,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
927
1001
  /*! ZSTD_DCtx_refDDict() :
928
1002
  * Reference a prepared dictionary, to be used to decompress next frames.
929
1003
  * The dictionary remains active for decompression of future frames using same DCtx.
1004
+ *
1005
+ * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
1006
+ * will store the DDict references in a table, and the DDict used for decompression
1007
+ * will be determined at decompression time, as per the dict ID in the frame.
1008
+ * The memory for the table is allocated on the first call to refDDict, and can be
1009
+ * freed with ZSTD_freeDCtx().
1010
+ *
930
1011
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
931
1012
  * Note 1 : Currently, only one dictionary can be managed.
932
1013
  * Referencing a new dictionary effectively "discards" any previous one.
@@ -946,7 +1027,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
946
1027
  * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
947
1028
  * Prefix buffer must remain unmodified up to the end of frame,
948
1029
  * reached when ZSTD_decompressStream() returns 0.
949
- * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
1030
+ * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
950
1031
  * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
951
1032
  * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
952
1033
  * A full dictionary is more costly, as it requires building tables.
@@ -969,7 +1050,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
969
1050
  #endif /* ZSTD_H_235446 */
970
1051
 
971
1052
 
972
- /****************************************************************************************
1053
+ /* **************************************************************************************
973
1054
  * ADVANCED AND EXPERIMENTAL FUNCTIONS
974
1055
  ****************************************************************************************
975
1056
  * The definitions in the following section are considered experimental.
@@ -991,8 +1072,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
991
1072
  * Some of them might be removed in the future (especially when redundant with existing stable functions)
992
1073
  * ***************************************************************************************/
993
1074
 
994
- #define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */
995
- #define ZSTD_FRAMEHEADERSIZE_MIN 6
1075
+ #define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
1076
+ #define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
996
1077
  #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
997
1078
  #define ZSTD_SKIPPABLEHEADERSIZE 8
998
1079
 
@@ -1037,6 +1118,12 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
1037
1118
  #define ZSTD_LDM_HASHRATELOG_MIN 0
1038
1119
  #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
1039
1120
 
1121
+ /* Advanced parameter bounds */
1122
+ #define ZSTD_TARGETCBLOCKSIZE_MIN 64
1123
+ #define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
1124
+ #define ZSTD_SRCSIZEHINT_MIN 0
1125
+ #define ZSTD_SRCSIZEHINT_MAX INT_MAX
1126
+
1040
1127
  /* internal */
1041
1128
  #define ZSTD_HASHLOG3_MAX 17
1042
1129
 
@@ -1045,6 +1132,43 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
1045
1132
 
1046
1133
  typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
1047
1134
 
1135
+ typedef struct {
1136
+ unsigned int offset; /* The offset of the match. (NOT the same as the offset code)
1137
+ * If offset == 0 and matchLength == 0, this sequence represents the last
1138
+ * literals in the block of litLength size.
1139
+ */
1140
+
1141
+ unsigned int litLength; /* Literal length of the sequence. */
1142
+ unsigned int matchLength; /* Match length of the sequence. */
1143
+
1144
+ /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
1145
+ * In this case, we will treat the sequence as a marker for a block boundary.
1146
+ */
1147
+
1148
+ unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'.
1149
+ * Ranges from [0, 3].
1150
+ *
1151
+ * Repeat offsets are essentially previous offsets from previous sequences sorted in
1152
+ * recency order. For more detail, see doc/zstd_compression_format.md
1153
+ *
1154
+ * If rep == 0, then 'offset' does not contain a repeat offset.
1155
+ * If rep > 0:
1156
+ * If litLength != 0:
1157
+ * rep == 1 --> offset == repeat_offset_1
1158
+ * rep == 2 --> offset == repeat_offset_2
1159
+ * rep == 3 --> offset == repeat_offset_3
1160
+ * If litLength == 0:
1161
+ * rep == 1 --> offset == repeat_offset_2
1162
+ * rep == 2 --> offset == repeat_offset_3
1163
+ * rep == 3 --> offset == repeat_offset_1 - 1
1164
+ *
1165
+ * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
1166
+ * 'rep', but repeat offsets do not necessarily need to be calculated from an external
1167
+ * sequence provider's perspective. For example, ZSTD_compressSequences() does not
1168
+ * use this 'rep' field at all (as of now).
1169
+ */
1170
+ } ZSTD_Sequence;
1171
+
1048
1172
  typedef struct {
1049
1173
  unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
1050
1174
  unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
@@ -1074,32 +1198,35 @@ typedef enum {
1074
1198
 
1075
1199
  typedef enum {
1076
1200
  ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
1077
- ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
1201
+ ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
1078
1202
  } ZSTD_dictLoadMethod_e;
1079
1203
 
1080
1204
  typedef enum {
1081
- /* Opened question : should we have a format ZSTD_f_auto ?
1082
- * Today, it would mean exactly the same as ZSTD_f_zstd1.
1083
- * But, in the future, should several formats become supported,
1084
- * on the compression side, it would mean "default format".
1085
- * On the decompression side, it would mean "automatic format detection",
1086
- * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
1087
- * Since meaning is a little different, another option could be to define different enums for compression and decompression.
1088
- * This question could be kept for later, when there are actually multiple formats to support,
1089
- * but there is also the question of pinning enum values, and pinning value `0` is especially important */
1090
1205
  ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
1091
- ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number.
1206
+ ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
1092
1207
  * Useful to save 4 bytes per generated frame.
1093
1208
  * Decoder cannot recognise automatically this format, requiring this instruction. */
1094
1209
  } ZSTD_format_e;
1095
1210
 
1211
+ typedef enum {
1212
+ /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
1213
+ ZSTD_d_validateChecksum = 0,
1214
+ ZSTD_d_ignoreChecksum = 1
1215
+ } ZSTD_forceIgnoreChecksum_e;
1216
+
1217
+ typedef enum {
1218
+ /* Note: this enum controls ZSTD_d_refMultipleDDicts */
1219
+ ZSTD_rmd_refSingleDDict = 0,
1220
+ ZSTD_rmd_refMultipleDDicts = 1
1221
+ } ZSTD_refMultipleDDicts_e;
1222
+
1096
1223
  typedef enum {
1097
1224
  /* Note: this enum and the behavior it controls are effectively internal
1098
1225
  * implementation details of the compressor. They are expected to continue
1099
1226
  * to evolve and should be considered only in the context of extremely
1100
1227
  * advanced performance tuning.
1101
1228
  *
1102
- * Zstd currently supports the use of a CDict in two ways:
1229
+ * Zstd currently supports the use of a CDict in three ways:
1103
1230
  *
1104
1231
  * - The contents of the CDict can be copied into the working context. This
1105
1232
  * means that the compression can search both the dictionary and input
@@ -1115,6 +1242,12 @@ typedef enum {
1115
1242
  * working context's tables can be reused). For small inputs, this can be
1116
1243
  * faster than copying the CDict's tables.
1117
1244
  *
1245
+ * - The CDict's tables are not used at all, and instead we use the working
1246
+ * context alone to reload the dictionary and use params based on the source
1247
+ * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
1248
+ * This method is effective when the dictionary sizes are very small relative
1249
+ * to the input size, and the input size is fairly large to begin with.
1250
+ *
1118
1251
  * Zstd has a simple internal heuristic that selects which strategy to use
1119
1252
  * at the beginning of a compression. However, if experimentation shows that
1120
1253
  * Zstd is making poor choices, it is possible to override that choice with
@@ -1123,6 +1256,7 @@ typedef enum {
1123
1256
  ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
1124
1257
  ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
1125
1258
  ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
1259
+ ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
1126
1260
  } ZSTD_dictAttachPref_e;
1127
1261
 
1128
1262
  typedef enum {
@@ -1131,7 +1265,7 @@ typedef enum {
1131
1265
  * levels will be compressed. */
1132
1266
  ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
1133
1267
  * emitted if Huffman compression is not profitable. */
1134
- ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */
1268
+ ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
1135
1269
  } ZSTD_literalCompressionMode_e;
1136
1270
 
1137
1271
 
@@ -1162,12 +1296,12 @@ typedef enum {
1162
1296
  * however it does mean that all frame data must be present and valid. */
1163
1297
  ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
1164
1298
 
1165
- /** ZSTD_decompressBound() :
1299
+ /*! ZSTD_decompressBound() :
1166
1300
  * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
1167
1301
  * `srcSize` must be the _exact_ size of this series
1168
1302
  * (i.e. there should be a frame boundary at `src + srcSize`)
1169
1303
  * @return : - upper-bound for the decompressed size of all data in all successive frames
1170
- * - if an error occured: ZSTD_CONTENTSIZE_ERROR
1304
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
1171
1305
  *
1172
1306
  * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
1173
1307
  * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
@@ -1183,6 +1317,92 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS
1183
1317
  * or an error code (if srcSize is too small) */
1184
1318
  ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
1185
1319
 
1320
+ typedef enum {
1321
+ ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
1322
+ ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
1323
+ } ZSTD_sequenceFormat_e;
1324
+
1325
+ /*! ZSTD_generateSequences() :
1326
+ * Generate sequences using ZSTD_compress2, given a source buffer.
1327
+ *
1328
+ * Each block will end with a dummy sequence
1329
+ * with offset == 0, matchLength == 0, and litLength == length of last literals.
1330
+ * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
1331
+ * simply acts as a block delimiter.
1332
+ *
1333
+ * zc can be used to insert custom compression params.
1334
+ * This function invokes ZSTD_compress2
1335
+ *
1336
+ * The output of this function can be fed into ZSTD_compressSequences() with CCtx
1337
+ * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
1338
+ * @return : number of sequences generated
1339
+ */
1340
+
1341
+ ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
1342
+ size_t outSeqsSize, const void* src, size_t srcSize);
1343
+
1344
+ /*! ZSTD_mergeBlockDelimiters() :
1345
+ * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
1346
+ * by merging them into into the literals of the next sequence.
1347
+ *
1348
+ * As such, the final generated result has no explicit representation of block boundaries,
1349
+ * and the final last literals segment is not represented in the sequences.
1350
+ *
1351
+ * The output of this function can be fed into ZSTD_compressSequences() with CCtx
1352
+ * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
1353
+ * @return : number of sequences left after merging
1354
+ */
1355
+ ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
1356
+
1357
+ /*! ZSTD_compressSequences() :
1358
+ * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
1359
+ * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
1360
+ * The entire source is compressed into a single frame.
1361
+ *
1362
+ * The compression behavior changes based on cctx params. In particular:
1363
+ * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
1364
+ * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
1365
+ * the block size derived from the cctx, and sequences may be split. This is the default setting.
1366
+ *
1367
+ * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
1368
+ * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
1369
+ *
1370
+ * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
1371
+ * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
1372
+ * specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
1373
+ *
1374
+ * In addition to the two adjustable experimental params, there are other important cctx params.
1375
+ * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
1376
+ * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
1377
+ * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
1378
+ * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
1379
+ *
1380
+ * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
1381
+ * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
1382
+ * and cannot emit an RLE block that disagrees with the repcode history
1383
+ * @return : final compressed size or a ZSTD error.
1384
+ */
1385
+ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
1386
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
1387
+ const void* src, size_t srcSize);
1388
+
1389
+
1390
+ /*! ZSTD_writeSkippableFrame() :
1391
+ * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
1392
+ *
1393
+ * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
1394
+ * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
1395
+ * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
1396
+ * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
1397
+ *
1398
+ * Returns an error if destination buffer is not large enough, if the source size is not representable
1399
+ * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
1400
+ *
1401
+ * @return : number of bytes written or a ZSTD error.
1402
+ */
1403
+ ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
1404
+ const void* src, size_t srcSize, unsigned magicVariant);
1405
+
1186
1406
 
1187
1407
  /***************************************
1188
1408
  * Memory management
@@ -1191,12 +1411,26 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
1191
1411
  /*! ZSTD_estimate*() :
1192
1412
  * These functions make it possible to estimate memory usage
1193
1413
  * of a future {D,C}Ctx, before its creation.
1194
- * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
1195
- * It will also consider src size to be arbitrarily "large", which is worst case.
1196
- * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
1197
- * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
1198
- * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
1199
- * Note : CCtx size estimation is only correct for single-threaded compression. */
1414
+ *
1415
+ * ZSTD_estimateCCtxSize() will provide a memory budget large enough
1416
+ * for any compression level up to selected one.
1417
+ * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
1418
+ * does not include space for a window buffer.
1419
+ * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
1420
+ * The estimate will assume the input may be arbitrarily large,
1421
+ * which is the worst case.
1422
+ *
1423
+ * When srcSize can be bound by a known and rather "small" value,
1424
+ * this fact can be used to provide a tighter estimation
1425
+ * because the CCtx compression context will need less memory.
1426
+ * This tighter estimation can be provided by more advanced functions
1427
+ * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
1428
+ * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
1429
+ * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
1430
+ *
1431
+ * Note 2 : only single-threaded compression is supported.
1432
+ * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
1433
+ */
1200
1434
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
1201
1435
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
1202
1436
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
@@ -1279,7 +1513,11 @@ ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict(
1279
1513
  typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
1280
1514
  typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
1281
1515
  typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
1282
- static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
1516
+ static
1517
+ #ifdef __GNUC__
1518
+ __attribute__((__unused__))
1519
+ #endif
1520
+ ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
1283
1521
 
1284
1522
  ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
1285
1523
  ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
@@ -1292,11 +1530,37 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
1292
1530
  ZSTD_compressionParameters cParams,
1293
1531
  ZSTD_customMem customMem);
1294
1532
 
1295
- ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
1296
- ZSTD_dictLoadMethod_e dictLoadMethod,
1297
- ZSTD_dictContentType_e dictContentType,
1298
- ZSTD_customMem customMem);
1533
+ /* ! Thread pool :
1534
+ * These prototypes make it possible to share a thread pool among multiple compression contexts.
1535
+ * This can limit resources for applications with multiple threads where each one uses
1536
+ * a threaded compression mode (via ZSTD_c_nbWorkers parameter).
1537
+ * ZSTD_createThreadPool creates a new thread pool with a given number of threads.
1538
+ * Note that the lifetime of such pool must exist while being used.
1539
+ * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
1540
+ * to use an internal thread pool).
1541
+ * ZSTD_freeThreadPool frees a thread pool.
1542
+ */
1543
+ typedef struct POOL_ctx_s ZSTD_threadPool;
1544
+ ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
1545
+ ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);
1546
+ ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
1547
+
1299
1548
 
1549
+ /*
1550
+ * This API is temporary and is expected to change or disappear in the future!
1551
+ */
1552
+ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
1553
+ const void* dict, size_t dictSize,
1554
+ ZSTD_dictLoadMethod_e dictLoadMethod,
1555
+ ZSTD_dictContentType_e dictContentType,
1556
+ const ZSTD_CCtx_params* cctxParams,
1557
+ ZSTD_customMem customMem);
1558
+
1559
+ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(
1560
+ const void* dict, size_t dictSize,
1561
+ ZSTD_dictLoadMethod_e dictLoadMethod,
1562
+ ZSTD_dictContentType_e dictContentType,
1563
+ ZSTD_customMem customMem);
1300
1564
 
1301
1565
 
1302
1566
  /***************************************
@@ -1307,9 +1571,16 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS
1307
1571
  * Create a digested dictionary for compression
1308
1572
  * Dictionary content is just referenced, not duplicated.
1309
1573
  * As a consequence, `dictBuffer` **must** outlive CDict,
1310
- * and its content must remain unmodified throughout the lifetime of CDict. */
1574
+ * and its content must remain unmodified throughout the lifetime of CDict.
1575
+ * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
1311
1576
  ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
1312
1577
 
1578
+ /*! ZSTD_getDictID_fromCDict() :
1579
+ * Provides the dictID of the dictionary loaded into `cdict`.
1580
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
1581
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
1582
+ ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
1583
+
1313
1584
  /*! ZSTD_getCParams() :
1314
1585
  * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
1315
1586
  * `estimatedSrcSize` value is optional, select 0 if not known */
@@ -1334,7 +1605,9 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
1334
1605
  ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
1335
1606
 
1336
1607
  /*! ZSTD_compress_advanced() :
1337
- * Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */
1608
+ * Note : this function is now DEPRECATED.
1609
+ * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
1610
+ * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */
1338
1611
  ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
1339
1612
  void* dst, size_t dstCapacity,
1340
1613
  const void* src, size_t srcSize,
@@ -1342,7 +1615,9 @@ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
1342
1615
  ZSTD_parameters params);
1343
1616
 
1344
1617
  /*! ZSTD_compress_usingCDict_advanced() :
1345
- * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
1618
+ * Note : this function is now REDUNDANT.
1619
+ * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
1620
+ * This prototype will be marked as deprecated and generate compilation warning in some future version */
1346
1621
  ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
1347
1622
  void* dst, size_t dstCapacity,
1348
1623
  const void* src, size_t srcSize,
@@ -1409,12 +1684,160 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
1409
1684
  */
1410
1685
  #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
1411
1686
 
1687
+ /* Tries to fit compressed block size to be around targetCBlockSize.
1688
+ * No target when targetCBlockSize == 0.
1689
+ * There is no guarantee on compressed block size (default:0) */
1690
+ #define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
1691
+
1692
+ /* User's best guess of source size.
1693
+ * Hint is not valid when srcSizeHint == 0.
1694
+ * There is no guarantee that hint is close to actual source size,
1695
+ * but compression ratio may regress significantly if guess considerably underestimates */
1696
+ #define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
1697
+
1698
+ /* Controls whether the new and experimental "dedicated dictionary search
1699
+ * structure" can be used. This feature is still rough around the edges, be
1700
+ * prepared for surprising behavior!
1701
+ *
1702
+ * How to use it:
1703
+ *
1704
+ * When using a CDict, whether to use this feature or not is controlled at
1705
+ * CDict creation, and it must be set in a CCtxParams set passed into that
1706
+ * construction (via ZSTD_createCDict_advanced2()). A compression will then
1707
+ * use the feature or not based on how the CDict was constructed; the value of
1708
+ * this param, set in the CCtx, will have no effect.
1709
+ *
1710
+ * However, when a dictionary buffer is passed into a CCtx, such as via
1711
+ * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
1712
+ * whether the CDict that is created internally can use the feature or not.
1713
+ *
1714
+ * What it does:
1715
+ *
1716
+ * Normally, the internal data structures of the CDict are analogous to what
1717
+ * would be stored in a CCtx after compressing the contents of a dictionary.
1718
+ * To an approximation, a compression using a dictionary can then use those
1719
+ * data structures to simply continue what is effectively a streaming
1720
+ * compression where the simulated compression of the dictionary left off.
1721
+ * Which is to say, the search structures in the CDict are normally the same
1722
+ * format as in the CCtx.
1723
+ *
1724
+ * It is possible to do better, since the CDict is not like a CCtx: the search
1725
+ * structures are written once during CDict creation, and then are only read
1726
+ * after that, while the search structures in the CCtx are both read and
1727
+ * written as the compression goes along. This means we can choose a search
1728
+ * structure for the dictionary that is read-optimized.
1729
+ *
1730
+ * This feature enables the use of that different structure.
1731
+ *
1732
+ * Note that some of the members of the ZSTD_compressionParameters struct have
1733
+ * different semantics and constraints in the dedicated search structure. It is
1734
+ * highly recommended that you simply set a compression level in the CCtxParams
1735
+ * you pass into the CDict creation call, and avoid messing with the cParams
1736
+ * directly.
1737
+ *
1738
+ * Effects:
1739
+ *
1740
+ * This will only have any effect when the selected ZSTD_strategy
1741
+ * implementation supports this feature. Currently, that's limited to
1742
+ * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
1743
+ *
1744
+ * Note that this means that the CDict tables can no longer be copied into the
1745
+ * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
1746
+ * useable. The dictionary can only be attached or reloaded.
1747
+ *
1748
+ * In general, you should expect compression to be faster--sometimes very much
1749
+ * so--and CDict creation to be slightly slower. Eventually, we will probably
1750
+ * make this mode the default.
1751
+ */
1752
+ #define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
1753
+
1754
+ /* ZSTD_c_stableInBuffer
1755
+ * Experimental parameter.
1756
+ * Default is 0 == disabled. Set to 1 to enable.
1757
+ *
1758
+ * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same
1759
+ * between calls, except for the modifications that zstd makes to pos (the
1760
+ * caller must not modify pos). This is checked by the compressor, and
1761
+ * compression will fail if it ever changes. This means the only flush
1762
+ * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end
1763
+ * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos)
1764
+ * MUST not be modified during compression or you will get data corruption.
1765
+ *
1766
+ * When this flag is enabled zstd won't allocate an input window buffer,
1767
+ * because the user guarantees it can reference the ZSTD_inBuffer until
1768
+ * the frame is complete. But, it will still allocate an output buffer
1769
+ * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
1770
+ * avoid the memcpy() from the input buffer to the input window buffer.
1771
+ *
1772
+ * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
1773
+ * That means this flag cannot be used with ZSTD_compressStream().
1774
+ *
1775
+ * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
1776
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
1777
+ * memory. However, compression WILL fail if you violate the preconditions.
1778
+ *
1779
+ * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST
1780
+ * not be modified during compression or you will get data corruption. This
1781
+ * is because zstd needs to reference data in the ZSTD_inBuffer to find
1782
+ * matches. Normally zstd maintains its own window buffer for this purpose,
1783
+ * but passing this flag tells zstd to use the user provided buffer.
1784
+ */
1785
+ #define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
1786
+
1787
+ /* ZSTD_c_stableOutBuffer
1788
+ * Experimental parameter.
1789
+ * Default is 0 == disabled. Set to 1 to enable.
1790
+ *
1791
+ * Tells he compressor that the ZSTD_outBuffer will not be resized between
1792
+ * calls. Specifically: (out.size - out.pos) will never grow. This gives the
1793
+ * compressor the freedom to say: If the compressed data doesn't fit in the
1794
+ * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
1795
+ * always decompress directly into the output buffer, instead of decompressing
1796
+ * into an internal buffer and copying to the output buffer.
1797
+ *
1798
+ * When this flag is enabled zstd won't allocate an output buffer, because
1799
+ * it can write directly to the ZSTD_outBuffer. It will still allocate the
1800
+ * input window buffer (see ZSTD_c_stableInBuffer).
1801
+ *
1802
+ * Zstd will check that (out.size - out.pos) never grows and return an error
1803
+ * if it does. While not strictly necessary, this should prevent surprises.
1804
+ */
1805
+ #define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
1806
+
1807
+ /* ZSTD_c_blockDelimiters
1808
+ * Default is 0 == ZSTD_sf_noBlockDelimiters.
1809
+ *
1810
+ * For use with sequence compression API: ZSTD_compressSequences().
1811
+ *
1812
+ * Designates whether or not the given array of ZSTD_Sequence contains block delimiters
1813
+ * and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
1814
+ * See the definition of ZSTD_Sequence for more specifics.
1815
+ */
1816
+ #define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
1817
+
1818
+ /* ZSTD_c_validateSequences
1819
+ * Default is 0 == disabled. Set to 1 to enable sequence validation.
1820
+ *
1821
+ * For use with sequence compression API: ZSTD_compressSequences().
1822
+ * Designates whether or not we validate sequences provided to ZSTD_compressSequences()
1823
+ * during function execution.
1824
+ *
1825
+ * Without validation, providing a sequence that does not conform to the zstd spec will cause
1826
+ * undefined behavior, and may produce a corrupted block.
1827
+ *
1828
+ * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for
1829
+ * specifics regarding offset/matchlength requirements) then the function will bail out and
1830
+ * return an error.
1831
+ *
1832
+ */
1833
+ #define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
1834
+
1412
1835
  /*! ZSTD_CCtx_getParameter() :
1413
1836
  * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
1414
1837
  * and store it into int* value.
1415
1838
  * @return : 0, or an error code (which can be tested with ZSTD_isError()).
1416
1839
  */
1417
- ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
1840
+ ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
1418
1841
 
1419
1842
 
1420
1843
  /*! ZSTD_CCtx_params :
@@ -1457,8 +1880,10 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, Z
1457
1880
  /*! ZSTD_CCtxParams_setParameter() :
1458
1881
  * Similar to ZSTD_CCtx_setParameter.
1459
1882
  * Set one compression parameter, selected by enum ZSTD_cParameter.
1460
- * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
1461
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1883
+ * Parameters must be applied to a ZSTD_CCtx using
1884
+ * ZSTD_CCtx_setParametersUsingCCtxParams().
1885
+ * @result : a code representing success or failure (which can be tested with
1886
+ * ZSTD_isError()).
1462
1887
  */
1463
1888
  ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
1464
1889
 
@@ -1467,7 +1892,7 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_c
1467
1892
  * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
1468
1893
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1469
1894
  */
1470
- ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
1895
+ ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
1471
1896
 
1472
1897
  /*! ZSTD_CCtx_setParametersUsingCCtxParams() :
1473
1898
  * Apply a set of ZSTD_CCtx_params to the compression context.
@@ -1538,11 +1963,84 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre
1538
1963
  */
1539
1964
  ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
1540
1965
 
1966
+ /*! ZSTD_DCtx_getParameter() :
1967
+ * Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
1968
+ * and store it into int* value.
1969
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
1970
+ */
1971
+ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
1972
+
1541
1973
  /* ZSTD_d_format
1542
1974
  * experimental parameter,
1543
1975
  * allowing selection between ZSTD_format_e input compression formats
1544
1976
  */
1545
1977
  #define ZSTD_d_format ZSTD_d_experimentalParam1
1978
+ /* ZSTD_d_stableOutBuffer
1979
+ * Experimental parameter.
1980
+ * Default is 0 == disabled. Set to 1 to enable.
1981
+ *
1982
+ * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
1983
+ * between calls, except for the modifications that zstd makes to pos (the
1984
+ * caller must not modify pos). This is checked by the decompressor, and
1985
+ * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
1986
+ * MUST be large enough to fit the entire decompressed frame. This will be
1987
+ * checked when the frame content size is known. The data in the ZSTD_outBuffer
1988
+ * in the range [dst, dst + pos) MUST not be modified during decompression
1989
+ * or you will get data corruption.
1990
+ *
1991
+ * When this flags is enabled zstd won't allocate an output buffer, because
1992
+ * it can write directly to the ZSTD_outBuffer, but it will still allocate
1993
+ * an input buffer large enough to fit any compressed block. This will also
1994
+ * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
1995
+ * If you need to avoid the input buffer allocation use the buffer-less
1996
+ * streaming API.
1997
+ *
1998
+ * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
1999
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
2000
+ * memory. However, decompression WILL fail if you violate the preconditions.
2001
+ *
2002
+ * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
2003
+ * not be modified during decompression or you will get data corruption. This
2004
+ * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
2005
+ * matches. Normally zstd maintains its own buffer for this purpose, but passing
2006
+ * this flag tells zstd to use the user provided buffer.
2007
+ */
2008
+ #define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
2009
+
2010
+ /* ZSTD_d_forceIgnoreChecksum
2011
+ * Experimental parameter.
2012
+ * Default is 0 == disabled. Set to 1 to enable
2013
+ *
2014
+ * Tells the decompressor to skip checksum validation during decompression, regardless
2015
+ * of whether checksumming was specified during compression. This offers some
2016
+ * slight performance benefits, and may be useful for debugging.
2017
+ * Param has values of type ZSTD_forceIgnoreChecksum_e
2018
+ */
2019
+ #define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
2020
+
2021
+ /* ZSTD_d_refMultipleDDicts
2022
+ * Experimental parameter.
2023
+ * Default is 0 == disabled. Set to 1 to enable
2024
+ *
2025
+ * If enabled and dctx is allocated on the heap, then additional memory will be allocated
2026
+ * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
2027
+ * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
2028
+ * store all references. At decompression time, the appropriate dictID is selected
2029
+ * from the set of DDicts based on the dictID in the frame.
2030
+ *
2031
+ * Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
2032
+ *
2033
+ * Param has values of byte ZSTD_refMultipleDDicts_e
2034
+ *
2035
+ * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
2036
+ * allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
2037
+ * Memory is allocated as per ZSTD_DCtx::customMem.
2038
+ *
2039
+ * Although this function allocates memory for the table, the user is still responsible for
2040
+ * memory management of the underlying ZSTD_DDict* themselves.
2041
+ */
2042
+ #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
2043
+
1546
2044
 
1547
2045
  /*! ZSTD_DCtx_setFormat() :
1548
2046
  * Instruct the decoder context about what kind of data to decode next.
@@ -1571,7 +2069,8 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
1571
2069
  ********************************************************************/
1572
2070
 
1573
2071
  /*===== Advanced Streaming compression functions =====*/
1574
- /**! ZSTD_initCStream_srcSize() :
2072
+
2073
+ /*! ZSTD_initCStream_srcSize() :
1575
2074
  * This function is deprecated, and equivalent to:
1576
2075
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1577
2076
  * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
@@ -1581,9 +2080,14 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
1581
2080
  * pledgedSrcSize must be correct. If it is not known at init time, use
1582
2081
  * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
1583
2082
  * "0" also disables frame content size field. It may be enabled in the future.
2083
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1584
2084
  */
1585
- ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
1586
- /**! ZSTD_initCStream_usingDict() :
2085
+ ZSTDLIB_API size_t
2086
+ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
2087
+ int compressionLevel,
2088
+ unsigned long long pledgedSrcSize);
2089
+
2090
+ /*! ZSTD_initCStream_usingDict() :
1587
2091
  * This function is deprecated, and is equivalent to:
1588
2092
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1589
2093
  * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
@@ -1591,42 +2095,66 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
1591
2095
  *
1592
2096
  * Creates of an internal CDict (incompatible with static CCtx), except if
1593
2097
  * dict == NULL or dictSize < 8, in which case no dict is used.
1594
- * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
2098
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
1595
2099
  * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
2100
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1596
2101
  */
1597
- ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
1598
- /**! ZSTD_initCStream_advanced() :
2102
+ ZSTDLIB_API size_t
2103
+ ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
2104
+ const void* dict, size_t dictSize,
2105
+ int compressionLevel);
2106
+
2107
+ /*! ZSTD_initCStream_advanced() :
1599
2108
  * This function is deprecated, and is approximately equivalent to:
1600
2109
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1601
- * ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is
2110
+ * // Pseudocode: Set each zstd parameter and leave the rest as-is.
2111
+ * for ((param, value) : params) {
2112
+ * ZSTD_CCtx_setParameter(zcs, param, value);
2113
+ * }
1602
2114
  * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
1603
2115
  * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
1604
2116
  *
1605
- * pledgedSrcSize must be correct. If srcSize is not known at init time, use
1606
- * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
2117
+ * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
2118
+ * pledgedSrcSize must be correct.
2119
+ * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
2120
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1607
2121
  */
1608
- ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
1609
- ZSTD_parameters params, unsigned long long pledgedSrcSize);
1610
- /**! ZSTD_initCStream_usingCDict() :
2122
+ ZSTDLIB_API size_t
2123
+ ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
2124
+ const void* dict, size_t dictSize,
2125
+ ZSTD_parameters params,
2126
+ unsigned long long pledgedSrcSize);
2127
+
2128
+ /*! ZSTD_initCStream_usingCDict() :
1611
2129
  * This function is deprecated, and equivalent to:
1612
2130
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1613
2131
  * ZSTD_CCtx_refCDict(zcs, cdict);
1614
2132
  *
1615
2133
  * note : cdict will just be referenced, and must outlive compression session
2134
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1616
2135
  */
1617
2136
  ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
1618
- /**! ZSTD_initCStream_usingCDict_advanced() :
1619
- * This function is deprecated, and is approximately equivalent to:
2137
+
2138
+ /*! ZSTD_initCStream_usingCDict_advanced() :
2139
+ * This function is DEPRECATED, and is approximately equivalent to:
1620
2140
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1621
- * ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is
2141
+ * // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
2142
+ * for ((fParam, value) : fParams) {
2143
+ * ZSTD_CCtx_setParameter(zcs, fParam, value);
2144
+ * }
1622
2145
  * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
1623
2146
  * ZSTD_CCtx_refCDict(zcs, cdict);
1624
2147
  *
1625
2148
  * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
1626
2149
  * pledgedSrcSize must be correct. If srcSize is not known at init time, use
1627
2150
  * value ZSTD_CONTENTSIZE_UNKNOWN.
2151
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1628
2152
  */
1629
- ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
2153
+ ZSTDLIB_API size_t
2154
+ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
2155
+ const ZSTD_CDict* cdict,
2156
+ ZSTD_frameParameters fParams,
2157
+ unsigned long long pledgedSrcSize);
1630
2158
 
1631
2159
  /*! ZSTD_resetCStream() :
1632
2160
  * This function is deprecated, and is equivalent to:
@@ -1641,6 +2169,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
1641
2169
  * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
1642
2170
  * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
1643
2171
  * @return : 0, or an error code (which can be tested using ZSTD_isError())
2172
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1644
2173
  */
1645
2174
  ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
1646
2175
 
@@ -1679,30 +2208,36 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
1679
2208
 
1680
2209
 
1681
2210
  /*===== Advanced Streaming decompression functions =====*/
1682
- /**
2211
+
2212
+ /*!
1683
2213
  * This function is deprecated, and is equivalent to:
1684
2214
  *
1685
2215
  * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
1686
2216
  * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
1687
2217
  *
1688
2218
  * note: no dictionary will be used if dict == NULL or dictSize < 8
2219
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1689
2220
  */
1690
2221
  ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
1691
- /**
2222
+
2223
+ /*!
1692
2224
  * This function is deprecated, and is equivalent to:
1693
2225
  *
1694
2226
  * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
1695
2227
  * ZSTD_DCtx_refDDict(zds, ddict);
1696
2228
  *
1697
2229
  * note : ddict is referenced, it must outlive decompression session
2230
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1698
2231
  */
1699
2232
  ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
1700
- /**
2233
+
2234
+ /*!
1701
2235
  * This function is deprecated, and is equivalent to:
1702
2236
  *
1703
2237
  * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
1704
2238
  *
1705
2239
  * re-use decompression parameters from previous init; saves dictionary loading
2240
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1706
2241
  */
1707
2242
  ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
1708
2243
 
@@ -1758,7 +2293,7 @@ ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstC
1758
2293
  ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
1759
2294
 
1760
2295
 
1761
- /*-
2296
+ /**
1762
2297
  Buffer-less streaming decompression (synchronous mode)
1763
2298
 
1764
2299
  A ZSTD_DCtx object is required to track streaming operations.
@@ -1843,7 +2378,7 @@ typedef struct {
1843
2378
  unsigned checksumFlag;
1844
2379
  } ZSTD_frameHeader;
1845
2380
 
1846
- /** ZSTD_getFrameHeader() :
2381
+ /*! ZSTD_getFrameHeader() :
1847
2382
  * decode Frame Header, or requires larger `srcSize`.
1848
2383
  * @return : 0, `zfhPtr` is correctly filled,
1849
2384
  * >0, `srcSize` is too small, value is wanted `srcSize` amount,
@@ -1876,8 +2411,8 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
1876
2411
 
1877
2412
  /*!
1878
2413
  Block functions produce and decode raw zstd blocks, without frame metadata.
1879
- Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
1880
- User will have to take in charge required information to regenerate data, such as compressed and content sizes.
2414
+ Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
2415
+ But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
1881
2416
 
1882
2417
  A few rules to respect :
1883
2418
  - Compressing and decompressing require a context structure
@@ -1888,12 +2423,14 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
1888
2423
  + copyCCtx() and copyDCtx() can be used too
1889
2424
  - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
1890
2425
  + If input is larger than a block size, it's necessary to split input data into multiple blocks
1891
- + For inputs larger than a single block, really consider using regular ZSTD_compress() instead.
1892
- Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
1893
- - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
1894
- In which case, nothing is produced into `dst` !
1895
- + User must test for such outcome and deal directly with uncompressed data
1896
- + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
2426
+ + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
2427
+ Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
2428
+ - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
2429
+ ===> In which case, nothing is produced into `dst` !
2430
+ + User __must__ test for such outcome and deal directly with uncompressed data
2431
+ + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
2432
+ Doing so would mess up with statistics history, leading to potential data corruption.
2433
+ + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
1897
2434
  + In case of multiple successive blocks, should some of them be uncompressed,
1898
2435
  decoder must be informed of their existence in order to follow proper history.
1899
2436
  Use ZSTD_insertBlock() for such a case.