zstd-ruby 1.4.0.0 → 1.4.9.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (95) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +35 -0
  3. data/README.md +2 -2
  4. data/ext/zstdruby/libzstd/Makefile +274 -107
  5. data/ext/zstdruby/libzstd/README.md +75 -16
  6. data/ext/zstdruby/libzstd/common/bitstream.h +59 -51
  7. data/ext/zstdruby/libzstd/common/compiler.h +154 -5
  8. data/ext/zstdruby/libzstd/common/cpu.h +1 -3
  9. data/ext/zstdruby/libzstd/common/debug.c +11 -31
  10. data/ext/zstdruby/libzstd/common/debug.h +22 -49
  11. data/ext/zstdruby/libzstd/common/entropy_common.c +201 -75
  12. data/ext/zstdruby/libzstd/common/error_private.c +3 -1
  13. data/ext/zstdruby/libzstd/common/error_private.h +7 -3
  14. data/ext/zstdruby/libzstd/common/fse.h +50 -42
  15. data/ext/zstdruby/libzstd/common/fse_decompress.c +134 -50
  16. data/ext/zstdruby/libzstd/common/huf.h +41 -38
  17. data/ext/zstdruby/libzstd/common/mem.h +68 -22
  18. data/ext/zstdruby/libzstd/common/pool.c +30 -20
  19. data/ext/zstdruby/libzstd/common/pool.h +3 -3
  20. data/ext/zstdruby/libzstd/common/threading.c +51 -4
  21. data/ext/zstdruby/libzstd/common/threading.h +36 -4
  22. data/ext/zstdruby/libzstd/common/xxhash.c +39 -89
  23. data/ext/zstdruby/libzstd/common/xxhash.h +12 -32
  24. data/ext/zstdruby/libzstd/common/zstd_common.c +10 -10
  25. data/ext/zstdruby/libzstd/common/zstd_deps.h +111 -0
  26. data/ext/zstdruby/libzstd/common/zstd_errors.h +3 -1
  27. data/ext/zstdruby/libzstd/common/zstd_internal.h +231 -72
  28. data/ext/zstdruby/libzstd/common/zstd_trace.c +42 -0
  29. data/ext/zstdruby/libzstd/common/zstd_trace.h +152 -0
  30. data/ext/zstdruby/libzstd/compress/fse_compress.c +47 -63
  31. data/ext/zstdruby/libzstd/compress/hist.c +41 -63
  32. data/ext/zstdruby/libzstd/compress/hist.h +13 -33
  33. data/ext/zstdruby/libzstd/compress/huf_compress.c +288 -172
  34. data/ext/zstdruby/libzstd/compress/zstd_compress.c +2504 -1626
  35. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +446 -85
  36. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +158 -0
  37. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +29 -0
  38. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +433 -0
  39. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +54 -0
  40. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +849 -0
  41. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
  42. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +561 -0
  43. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +82 -60
  44. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
  45. data/ext/zstdruby/libzstd/compress/zstd_fast.c +106 -80
  46. data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
  47. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +411 -105
  48. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +21 -1
  49. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +296 -207
  50. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +14 -3
  51. data/ext/zstdruby/libzstd/compress/zstd_ldm_geartab.h +103 -0
  52. data/ext/zstdruby/libzstd/compress/zstd_opt.c +260 -148
  53. data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
  54. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +153 -440
  55. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +29 -110
  56. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +356 -238
  57. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +20 -16
  58. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +3 -3
  59. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +641 -238
  60. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +600 -371
  61. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +8 -5
  62. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +40 -9
  63. data/ext/zstdruby/libzstd/deprecated/zbuff.h +9 -8
  64. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
  65. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
  66. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
  67. data/ext/zstdruby/libzstd/dictBuilder/cover.c +197 -78
  68. data/ext/zstdruby/libzstd/dictBuilder/cover.h +52 -7
  69. data/ext/zstdruby/libzstd/dictBuilder/divsufsort.c +1 -1
  70. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +84 -66
  71. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +58 -36
  72. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +60 -31
  73. data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
  74. data/ext/zstdruby/libzstd/dll/example/README.md +16 -22
  75. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +8 -4
  76. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +115 -111
  77. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +1 -1
  78. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +28 -14
  79. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +1 -1
  80. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +28 -14
  81. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +1 -1
  82. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +36 -19
  83. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +1 -1
  84. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +122 -107
  85. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +2 -2
  86. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +29 -23
  87. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +1 -1
  88. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +34 -24
  89. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +1 -1
  90. data/ext/zstdruby/libzstd/libzstd.pc.in +2 -1
  91. data/ext/zstdruby/libzstd/zstd.h +655 -118
  92. data/lib/zstd-ruby/version.rb +1 -1
  93. data/zstd-ruby.gemspec +1 -1
  94. metadata +20 -10
  95. data/.travis.yml +0 -14
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -19,7 +19,7 @@ extern "C" {
19
19
  * Dependencies
20
20
  ***************************************/
21
21
  #include <stddef.h> /* size_t */
22
- #include "mem.h" /* U64, U32 */
22
+ #include "../common/mem.h" /* U64, U32 */
23
23
 
24
24
 
25
25
  /* *************************************
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +14,7 @@
14
14
  #include <stddef.h> /* size_t, ptrdiff_t */
15
15
  #include <string.h> /* memcpy */
16
16
  #include <stdlib.h> /* malloc, free, qsort */
17
- #include "error_private.h"
17
+ #include "../common/error_private.h"
18
18
 
19
19
 
20
20
 
@@ -82,7 +82,11 @@ extern "C" {
82
82
  * Basic Types
83
83
  *****************************************************************/
84
84
  #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
85
- # include <stdint.h>
85
+ # if defined(_AIX)
86
+ # include <inttypes.h>
87
+ # else
88
+ # include <stdint.h> /* intptr_t */
89
+ # endif
86
90
  typedef uint8_t BYTE;
87
91
  typedef uint16_t U16;
88
92
  typedef int16_t S16;
@@ -860,7 +864,7 @@ MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
860
864
  _BitScanReverse ( &r, val );
861
865
  return (unsigned) r;
862
866
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
863
- return 31 - __builtin_clz (val);
867
+ return __builtin_clz (val) ^ 31;
864
868
  # else /* Software version */
865
869
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
866
870
  U32 v = val;
@@ -1862,7 +1866,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
1862
1866
 
1863
1867
  if (!srcSize) return ERROR(srcSize_wrong);
1864
1868
  iSize = ip[0];
1865
- //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1869
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
1866
1870
 
1867
1871
  if (iSize >= 128) { /* special header */
1868
1872
  if (iSize >= (242)) { /* RLE */
@@ -2014,7 +2018,7 @@ size_t HUFv06_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
2014
2018
  HUFv06_DEltX2* const dt = (HUFv06_DEltX2*)dtPtr;
2015
2019
 
2016
2020
  HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
2017
- //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
2021
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
2018
2022
 
2019
2023
  iSize = HUFv06_readStats(huffWeight, HUFv06_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
2020
2024
  if (HUFv06_isError(iSize)) return iSize;
@@ -2340,7 +2344,7 @@ size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2340
2344
 
2341
2345
  HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
2342
2346
  if (memLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
2343
- //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2347
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
2344
2348
 
2345
2349
  iSize = HUFv06_readStats(weightList, HUFv06_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
2346
2350
  if (HUFv06_isError(iSize)) return iSize;
@@ -2664,13 +2668,13 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2664
2668
 
2665
2669
  { U32 algoNb = 0;
2666
2670
  if (Dtime[1] < Dtime[0]) algoNb = 1;
2667
- // if (Dtime[2] < Dtime[algoNb]) algoNb = 2; /* current speed of HUFv06_decompress4X6 is not good */
2671
+ /* if (Dtime[2] < Dtime[algoNb]) algoNb = 2; */ /* current speed of HUFv06_decompress4X6 is not good */
2668
2672
  return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
2669
2673
  }
2670
2674
 
2671
- //return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
2672
- //return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
2673
- //return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); /* multi-streams quad-symbols decoding */
2675
+ /* return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
2676
+ /* return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
2677
+ /* return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams quad-symbols decoding */
2674
2678
  }
2675
2679
  /*
2676
2680
  Common functions of Zstd compression library
@@ -3025,7 +3029,7 @@ typedef struct
3025
3029
  * Provides the size of compressed block from block header `src` */
3026
3030
  static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
3027
3031
  {
3028
- const BYTE* const in = (const BYTE* const)src;
3032
+ const BYTE* const in = (const BYTE*)src;
3029
3033
  U32 cSize;
3030
3034
 
3031
3035
  if (srcSize < ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
@@ -3219,7 +3223,7 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3219
3223
  FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
3220
3224
  const void* src, size_t srcSize)
3221
3225
  {
3222
- const BYTE* const istart = (const BYTE* const)src;
3226
+ const BYTE* const istart = (const BYTE*)src;
3223
3227
  const BYTE* const iend = istart + srcSize;
3224
3228
  const BYTE* ip = istart;
3225
3229
 
@@ -3242,14 +3246,12 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3242
3246
  }
3243
3247
 
3244
3248
  /* FSE table descriptors */
3249
+ if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
3245
3250
  { U32 const LLtype = *ip >> 6;
3246
3251
  U32 const Offtype = (*ip >> 4) & 3;
3247
3252
  U32 const MLtype = (*ip >> 2) & 3;
3248
3253
  ip++;
3249
3254
 
3250
- /* check */
3251
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
3252
-
3253
3255
  /* Build DTables */
3254
3256
  { size_t const bhSize = ZSTDv06_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
3255
3257
  if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
@@ -3443,7 +3445,7 @@ static size_t ZSTDv06_decompressSequences(
3443
3445
  {
3444
3446
  const BYTE* ip = (const BYTE*)seqStart;
3445
3447
  const BYTE* const iend = ip + seqSize;
3446
- BYTE* const ostart = (BYTE* const)dst;
3448
+ BYTE* const ostart = (BYTE*)dst;
3447
3449
  BYTE* const oend = ostart + maxDstSize;
3448
3450
  BYTE* op = ostart;
3449
3451
  const BYTE* litPtr = dctx->litPtr;
@@ -3503,8 +3505,10 @@ static size_t ZSTDv06_decompressSequences(
3503
3505
  { size_t const lastLLSize = litEnd - litPtr;
3504
3506
  if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
3505
3507
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
3506
- memcpy(op, litPtr, lastLLSize);
3507
- op += lastLLSize;
3508
+ if (lastLLSize > 0) {
3509
+ memcpy(op, litPtr, lastLLSize);
3510
+ op += lastLLSize;
3511
+ }
3508
3512
  }
3509
3513
 
3510
3514
  return op-ostart;
@@ -3557,7 +3561,7 @@ static size_t ZSTDv06_decompressFrame(ZSTDv06_DCtx* dctx,
3557
3561
  {
3558
3562
  const BYTE* ip = (const BYTE*)src;
3559
3563
  const BYTE* const iend = ip + srcSize;
3560
- BYTE* const ostart = (BYTE* const)dst;
3564
+ BYTE* const ostart = (BYTE*)dst;
3561
3565
  BYTE* op = ostart;
3562
3566
  BYTE* const oend = ostart + dstCapacity;
3563
3567
  size_t remainingSize = srcSize;
@@ -3672,7 +3676,7 @@ void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cS
3672
3676
  blockProperties_t blockProperties = { bt_compressed, 0 };
3673
3677
 
3674
3678
  /* Frame Header */
3675
- { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
3679
+ { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, srcSize);
3676
3680
  if (ZSTDv06_isError(frameHeaderSize)) {
3677
3681
  ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
3678
3682
  return;
@@ -4002,7 +4006,9 @@ size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* zbd)
4002
4006
  MEM_STATIC size_t ZBUFFv06_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
4003
4007
  {
4004
4008
  size_t length = MIN(dstCapacity, srcSize);
4005
- memcpy(dst, src, length);
4009
+ if (length > 0) {
4010
+ memcpy(dst, src, length);
4011
+ }
4006
4012
  return length;
4007
4013
  }
4008
4014
 
@@ -4111,7 +4117,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4111
4117
  if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
4112
4118
  zbd->outEnd = zbd->outStart + decodedSize;
4113
4119
  zbd->stage = ZBUFFds_flush;
4114
- // break; /* ZBUFFds_flush follows */
4120
+ /* break; */ /* ZBUFFds_flush follows */
4115
4121
  }
4116
4122
  }
4117
4123
  /* fall-through */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -17,14 +17,14 @@
17
17
  #ifndef XXH_STATIC_LINKING_ONLY
18
18
  # define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
19
19
  #endif
20
- #include "xxhash.h" /* XXH64_* */
20
+ #include "../common/xxhash.h" /* XXH64_* */
21
21
  #include "zstd_v07.h"
22
22
 
23
23
  #define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
24
24
  #define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
25
25
  #define ZSTDv07_STATIC_LINKING_ONLY
26
26
 
27
- #include "error_private.h"
27
+ #include "../common/error_private.h"
28
28
 
29
29
 
30
30
  #ifdef ZSTDv07_STATIC_LINKING_ONLY
@@ -242,7 +242,11 @@ extern "C" {
242
242
  * Basic Types
243
243
  *****************************************************************/
244
244
  #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
245
- # include <stdint.h>
245
+ # if defined(_AIX)
246
+ # include <inttypes.h>
247
+ # else
248
+ # include <stdint.h> /* intptr_t */
249
+ # endif
246
250
  typedef uint8_t BYTE;
247
251
  typedef uint16_t U16;
248
252
  typedef int16_t S16;
@@ -530,7 +534,7 @@ MEM_STATIC unsigned BITv07_highbit32 (U32 val)
530
534
  _BitScanReverse ( &r, val );
531
535
  return (unsigned) r;
532
536
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
533
- return 31 - __builtin_clz (val);
537
+ return __builtin_clz (val) ^ 31;
534
538
  # else /* Software version */
535
539
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
536
540
  U32 v = val;
@@ -1314,7 +1318,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
1314
1318
 
1315
1319
  if (!srcSize) return ERROR(srcSize_wrong);
1316
1320
  iSize = ip[0];
1317
- //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1321
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
1318
1322
 
1319
1323
  if (iSize >= 128) { /* special header */
1320
1324
  if (iSize >= (242)) { /* RLE */
@@ -1784,7 +1788,7 @@ size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSi
1784
1788
  HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr;
1785
1789
 
1786
1790
  HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable));
1787
- //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
1791
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
1788
1792
 
1789
1793
  iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
1790
1794
  if (HUFv07_isError(iSize)) return iSize;
@@ -2148,7 +2152,7 @@ size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSi
2148
2152
 
2149
2153
  HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */
2150
2154
  if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
2151
- //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2155
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
2152
2156
 
2153
2157
  iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
2154
2158
  if (HUFv07_isError(iSize)) return iSize;
@@ -2530,8 +2534,8 @@ size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2530
2534
  return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
2531
2535
  }
2532
2536
 
2533
- //return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
2534
- //return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
2537
+ /* return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
2538
+ /* return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
2535
2539
  }
2536
2540
 
2537
2541
  size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
@@ -3254,7 +3258,7 @@ typedef struct
3254
3258
  * Provides the size of compressed block from block header `src` */
3255
3259
  static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
3256
3260
  {
3257
- const BYTE* const in = (const BYTE* const)src;
3261
+ const BYTE* const in = (const BYTE*)src;
3258
3262
  U32 cSize;
3259
3263
 
3260
3264
  if (srcSize < ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
@@ -3272,7 +3276,9 @@ static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProper
3272
3276
  static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
3273
3277
  {
3274
3278
  if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
3275
- memcpy(dst, src, srcSize);
3279
+ if (srcSize > 0) {
3280
+ memcpy(dst, src, srcSize);
3281
+ }
3276
3282
  return srcSize;
3277
3283
  }
3278
3284
 
@@ -3447,7 +3453,7 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
3447
3453
  FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable,
3448
3454
  const void* src, size_t srcSize)
3449
3455
  {
3450
- const BYTE* const istart = (const BYTE* const)src;
3456
+ const BYTE* const istart = (const BYTE*)src;
3451
3457
  const BYTE* const iend = istart + srcSize;
3452
3458
  const BYTE* ip = istart;
3453
3459
 
@@ -3470,14 +3476,12 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
3470
3476
  }
3471
3477
 
3472
3478
  /* FSE table descriptors */
3479
+ if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
3473
3480
  { U32 const LLtype = *ip >> 6;
3474
3481
  U32 const OFtype = (*ip >> 4) & 3;
3475
3482
  U32 const MLtype = (*ip >> 2) & 3;
3476
3483
  ip++;
3477
3484
 
3478
- /* check */
3479
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
3480
-
3481
3485
  /* Build DTables */
3482
3486
  { size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
3483
3487
  if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected);
@@ -3668,7 +3672,7 @@ static size_t ZSTDv07_decompressSequences(
3668
3672
  {
3669
3673
  const BYTE* ip = (const BYTE*)seqStart;
3670
3674
  const BYTE* const iend = ip + seqSize;
3671
- BYTE* const ostart = (BYTE* const)dst;
3675
+ BYTE* const ostart = (BYTE*)dst;
3672
3676
  BYTE* const oend = ostart + maxDstSize;
3673
3677
  BYTE* op = ostart;
3674
3678
  const BYTE* litPtr = dctx->litPtr;
@@ -3714,10 +3718,12 @@ static size_t ZSTDv07_decompressSequences(
3714
3718
 
3715
3719
  /* last literal segment */
3716
3720
  { size_t const lastLLSize = litEnd - litPtr;
3717
- //if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
3721
+ /* if (litPtr > litEnd) return ERROR(corruption_detected); */ /* too many literals already used */
3718
3722
  if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
3719
- memcpy(op, litPtr, lastLLSize);
3720
- op += lastLLSize;
3723
+ if (lastLLSize > 0) {
3724
+ memcpy(op, litPtr, lastLLSize);
3725
+ op += lastLLSize;
3726
+ }
3721
3727
  }
3722
3728
 
3723
3729
  return op-ostart;
@@ -3778,7 +3784,9 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
3778
3784
  static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
3779
3785
  {
3780
3786
  if (length > dstCapacity) return ERROR(dstSize_tooSmall);
3781
- memset(dst, byte, length);
3787
+ if (length > 0) {
3788
+ memset(dst, byte, length);
3789
+ }
3782
3790
  return length;
3783
3791
  }
3784
3792
 
@@ -3791,7 +3799,7 @@ static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx,
3791
3799
  {
3792
3800
  const BYTE* ip = (const BYTE*)src;
3793
3801
  const BYTE* const iend = ip + srcSize;
3794
- BYTE* const ostart = (BYTE* const)dst;
3802
+ BYTE* const ostart = (BYTE*)dst;
3795
3803
  BYTE* const oend = ostart + dstCapacity;
3796
3804
  BYTE* op = ostart;
3797
3805
  size_t remainingSize = srcSize;
@@ -3918,7 +3926,7 @@ void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cS
3918
3926
  }
3919
3927
 
3920
3928
  /* Frame Header */
3921
- { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
3929
+ { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, srcSize);
3922
3930
  if (ZSTDv07_isError(frameHeaderSize)) {
3923
3931
  ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
3924
3932
  return;
@@ -4380,7 +4388,9 @@ size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd)
4380
4388
  MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
4381
4389
  {
4382
4390
  size_t const length = MIN(dstCapacity, srcSize);
4383
- memcpy(dst, src, length);
4391
+ if (length > 0) {
4392
+ memcpy(dst, src, length);
4393
+ }
4384
4394
  return length;
4385
4395
  }
4386
4396
 
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -3,8 +3,9 @@
3
3
  # BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
4
4
 
5
5
  prefix=@PREFIX@
6
- libdir=@LIBDIR@
6
+ exec_prefix=@EXEC_PREFIX@
7
7
  includedir=@INCLUDEDIR@
8
+ libdir=@LIBDIR@
8
9
 
9
10
  Name: zstd
10
11
  Description: fast lossless compression algorithm library
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -15,6 +15,7 @@ extern "C" {
15
15
  #define ZSTD_H_235446
16
16
 
17
17
  /* ====== Dependency ======*/
18
+ #include <limits.h> /* INT_MAX */
18
19
  #include <stddef.h> /* size_t */
19
20
 
20
21
 
@@ -71,27 +72,32 @@ extern "C" {
71
72
  /*------ Version ------*/
72
73
  #define ZSTD_VERSION_MAJOR 1
73
74
  #define ZSTD_VERSION_MINOR 4
74
- #define ZSTD_VERSION_RELEASE 0
75
-
75
+ #define ZSTD_VERSION_RELEASE 9
76
76
  #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
77
- ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */
77
+
78
+ /*! ZSTD_versionNumber() :
79
+ * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */
80
+ ZSTDLIB_API unsigned ZSTD_versionNumber(void);
78
81
 
79
82
  #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
80
83
  #define ZSTD_QUOTE(str) #str
81
84
  #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
82
85
  #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
83
- ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */
84
86
 
85
- /***************************************
86
- * Default constant
87
- ***************************************/
87
+ /*! ZSTD_versionString() :
88
+ * Return runtime library version, like "1.4.5". Requires v1.3.0+. */
89
+ ZSTDLIB_API const char* ZSTD_versionString(void);
90
+
91
+ /* *************************************
92
+ * Default constant
93
+ ***************************************/
88
94
  #ifndef ZSTD_CLEVEL_DEFAULT
89
95
  # define ZSTD_CLEVEL_DEFAULT 3
90
96
  #endif
91
97
 
92
- /***************************************
93
- * Constants
94
- ***************************************/
98
+ /* *************************************
99
+ * Constants
100
+ ***************************************/
95
101
 
96
102
  /* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
97
103
  #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
@@ -183,17 +189,26 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres
183
189
  ***************************************/
184
190
  /*= Compression context
185
191
  * When compressing many times,
186
- * it is recommended to allocate a context just once, and re-use it for each successive compression operation.
192
+ * it is recommended to allocate a context just once,
193
+ * and re-use it for each successive compression operation.
187
194
  * This will make workload friendlier for system's memory.
188
- * Use one context per thread for parallel execution in multi-threaded environments. */
195
+ * Note : re-using context is just a speed / resource optimization.
196
+ * It doesn't change the compression ratio, which remains identical.
197
+ * Note 2 : In multi-threaded environments,
198
+ * use one different context per thread for parallel execution.
199
+ */
189
200
  typedef struct ZSTD_CCtx_s ZSTD_CCtx;
190
201
  ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
191
202
  ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
192
203
 
193
204
  /*! ZSTD_compressCCtx() :
194
- * Same as ZSTD_compress(), using an explicit ZSTD_CCtx
195
- * The function will compress at requested compression level,
196
- * ignoring any other parameter */
205
+ * Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
206
+ * Important : in order to behave similarly to `ZSTD_compress()`,
207
+ * this function compresses at requested compression level,
208
+ * __ignoring any other parameter__ .
209
+ * If any advanced parameter was set using the advanced API,
210
+ * they will all be reset. Only `compressionLevel` remains.
211
+ */
197
212
  ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
198
213
  void* dst, size_t dstCapacity,
199
214
  const void* src, size_t srcSize,
@@ -228,7 +243,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
228
243
  * using ZSTD_CCtx_set*() functions.
229
244
  * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
230
245
  * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
231
- * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()
246
+ * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
232
247
  *
233
248
  * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
234
249
  *
@@ -256,18 +271,29 @@ typedef enum {
256
271
 
257
272
  /* compression parameters
258
273
  * Note: When compressing with a ZSTD_CDict these parameters are superseded
259
- * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
260
- * for more info (superseded-by-cdict). */
261
- ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
274
+ * by the parameters used to construct the ZSTD_CDict.
275
+ * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
276
+ ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
277
+ * Note that exact compression parameters are dynamically determined,
278
+ * depending on both compression level and srcSize (when known).
262
279
  * Default level is ZSTD_CLEVEL_DEFAULT==3.
263
280
  * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
264
281
  * Note 1 : it's possible to pass a negative compression level.
265
- * Note 2 : setting a level sets all default values of other compression parameters */
282
+ * Note 2 : setting a level does not automatically set all other compression parameters
283
+ * to default. Setting this will however eventually dynamically impact the compression
284
+ * parameters which have not been manually set. The manually set
285
+ * ones will 'stick'. */
286
+ /* Advanced compression parameters :
287
+ * It's possible to pin down compression parameters to some specific values.
288
+ * In which case, these values are no longer dynamically selected by the compressor */
266
289
  ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
290
+ * This will set a memory budget for streaming decompression,
291
+ * with larger values requiring more memory
292
+ * and typically compressing more.
267
293
  * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
268
294
  * Special: value 0 means "use default windowLog".
269
295
  * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
270
- * requires explicitly allowing such window size at decompression stage if using streaming. */
296
+ * requires explicitly allowing such size at streaming decompression stage. */
271
297
  ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
272
298
  * Resulting memory usage is (1 << (hashLog+2)).
273
299
  * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
@@ -278,13 +304,13 @@ typedef enum {
278
304
  * Resulting memory usage is (1 << (chainLog+2)).
279
305
  * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
280
306
  * Larger tables result in better and slower compression.
281
- * This parameter is useless when using "fast" strategy.
307
+ * This parameter is useless for "fast" strategy.
282
308
  * It's still useful when using "dfast" strategy,
283
309
  * in which case it defines a secondary probe table.
284
310
  * Special: value 0 means "use default chainLog". */
285
311
  ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
286
312
  * More attempts result in better and slower compression.
287
- * This parameter is useless when using "fast" and "dFast" strategies.
313
+ * This parameter is useless for "fast" and "dFast" strategies.
288
314
  * Special: value 0 means "use default searchLog". */
289
315
  ZSTD_c_minMatch=105, /* Minimum size of searched matches.
290
316
  * Note that Zstandard can still find matches of smaller size,
@@ -313,7 +339,9 @@ typedef enum {
313
339
  * for large inputs, by finding large matches at long distance.
314
340
  * It increases memory usage and window size.
315
341
  * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
316
- * except when expressly set to a different value. */
342
+ * except when expressly set to a different value.
343
+ * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
344
+ * compression strategy >= ZSTD_btopt (== compression level 16+) */
317
345
  ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
318
346
  * Larger values increase memory usage and compression ratio,
319
347
  * but decrease compression speed.
@@ -339,26 +367,30 @@ typedef enum {
339
367
  ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
340
368
  * Content size must be known at the beginning of compression.
341
369
  * This is automatically the case when using ZSTD_compress2(),
342
- * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
370
+ * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
343
371
  ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
344
372
  ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
345
373
 
346
374
  /* multi-threading parameters */
347
- /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
348
- * They return an error otherwise. */
375
+ /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
376
+ * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
377
+ * In a situation where it's unknown if the linked library supports multi-threading or not,
378
+ * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
379
+ */
349
380
  ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
350
- * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :
381
+ * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
351
382
  * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
352
- * while compression work is performed in parallel, within worker threads.
383
+ * while compression is performed in parallel, within worker thread(s).
353
384
  * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
354
385
  * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
355
386
  * More workers improve speed, but also increase memory usage.
356
- * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
387
+ * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
388
+ * compression is performed inside Caller's thread, and all invocations are blocking */
357
389
  ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
358
390
  * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
359
391
  * 0 means default, which is dynamically determined based on compression parameters.
360
392
  * Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
361
- * The minimum size is automatically and transparently enforced */
393
+ * The minimum size is automatically and transparently enforced. */
362
394
  ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
363
395
  * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
364
396
  * It helps preserve compression ratio, while each job is compressed in parallel.
@@ -380,6 +412,13 @@ typedef enum {
380
412
  * ZSTD_c_forceMaxWindow
381
413
  * ZSTD_c_forceAttachDict
382
414
  * ZSTD_c_literalCompressionMode
415
+ * ZSTD_c_targetCBlockSize
416
+ * ZSTD_c_srcSizeHint
417
+ * ZSTD_c_enableDedicatedDictSearch
418
+ * ZSTD_c_stableInBuffer
419
+ * ZSTD_c_stableOutBuffer
420
+ * ZSTD_c_blockDelimiters
421
+ * ZSTD_c_validateSequences
383
422
  * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
384
423
  * note : never ever use experimentalParam? names directly;
385
424
  * also, the enums values themselves are unstable and can still change.
@@ -389,6 +428,13 @@ typedef enum {
389
428
  ZSTD_c_experimentalParam3=1000,
390
429
  ZSTD_c_experimentalParam4=1001,
391
430
  ZSTD_c_experimentalParam5=1002,
431
+ ZSTD_c_experimentalParam6=1003,
432
+ ZSTD_c_experimentalParam7=1004,
433
+ ZSTD_c_experimentalParam8=1005,
434
+ ZSTD_c_experimentalParam9=1006,
435
+ ZSTD_c_experimentalParam10=1007,
436
+ ZSTD_c_experimentalParam11=1008,
437
+ ZSTD_c_experimentalParam12=1009
392
438
  } ZSTD_cParameter;
393
439
 
394
440
  typedef struct {
@@ -497,11 +543,17 @@ typedef enum {
497
543
  /* note : additional experimental parameters are also available
498
544
  * within the experimental section of the API.
499
545
  * At the time of this writing, they include :
500
- * ZSTD_c_format
546
+ * ZSTD_d_format
547
+ * ZSTD_d_stableOutBuffer
548
+ * ZSTD_d_forceIgnoreChecksum
549
+ * ZSTD_d_refMultipleDDicts
501
550
  * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
502
551
  * note : never ever use experimentalParam? names directly
503
552
  */
504
- ZSTD_d_experimentalParam1=1000
553
+ ZSTD_d_experimentalParam1=1000,
554
+ ZSTD_d_experimentalParam2=1001,
555
+ ZSTD_d_experimentalParam3=1002,
556
+ ZSTD_d_experimentalParam4=1003
505
557
 
506
558
  } ZSTD_dParameter;
507
559
 
@@ -637,8 +689,9 @@ typedef enum {
637
689
  * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
638
690
  * - output->pos must be <= dstCapacity, input->pos must be <= srcSize
639
691
  * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
692
+ * - endOp must be a valid directive
640
693
  * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
641
- * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
694
+ * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
642
695
  * and then immediately returns, just indicating that there is some data remaining to be flushed.
643
696
  * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
644
697
  * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
@@ -657,17 +710,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
657
710
  ZSTD_inBuffer* input,
658
711
  ZSTD_EndDirective endOp);
659
712
 
713
+
714
+ /* These buffer sizes are softly recommended.
715
+ * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
716
+ * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
717
+ * reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
718
+ *
719
+ * However, note that these recommendations are from the perspective of a C caller program.
720
+ * If the streaming interface is invoked from some other language,
721
+ * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
722
+ * a major performance rule is to reduce crossing such interface to an absolute minimum.
723
+ * It's not rare that performance ends being spent more into the interface, rather than compression itself.
724
+ * In which cases, prefer using large buffers, as large as practical,
725
+ * for both input and output, to reduce the nb of roundtrips.
726
+ */
660
727
  ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
661
- ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
728
+ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
662
729
 
663
- /*******************************************************************************
664
- * This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and
665
- * ZSTD_compressStream2(). It is redundant, but is still fully supported.
730
+
731
+ /* *****************************************************************************
732
+ * This following is a legacy streaming API.
733
+ * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
734
+ * It is redundant, but remains fully supported.
666
735
  * Advanced parameters and dictionary compression can only be used through the
667
736
  * new API.
668
737
  ******************************************************************************/
669
738
 
670
- /**
739
+ /*!
671
740
  * Equivalent to:
672
741
  *
673
742
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -675,16 +744,16 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
675
744
  * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
676
745
  */
677
746
  ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
678
- /**
747
+ /*!
679
748
  * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
680
749
  * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
681
750
  * the next read size (if non-zero and not an error). ZSTD_compressStream2()
682
- * returns the number of bytes left to flush (if non-zero and not an error).
751
+ * returns the minimum nb of bytes left to flush (if non-zero and not an error).
683
752
  */
684
753
  ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
685
- /** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
754
+ /*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
686
755
  ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
687
- /** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
756
+ /*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
688
757
  ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
689
758
 
690
759
 
@@ -725,7 +794,7 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
725
794
 
726
795
  /* This function is redundant with the advanced API and equivalent to:
727
796
  *
728
- * ZSTD_DCtx_reset(zds);
797
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
729
798
  * ZSTD_DCtx_refDDict(zds, NULL);
730
799
  */
731
800
  ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
@@ -770,12 +839,17 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
770
839
  typedef struct ZSTD_CDict_s ZSTD_CDict;
771
840
 
772
841
  /*! ZSTD_createCDict() :
773
- * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
774
- * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
842
+ * When compressing multiple messages or blocks using the same dictionary,
843
+ * it's recommended to digest the dictionary only once, since it's a costly operation.
844
+ * ZSTD_createCDict() will create a state from digesting a dictionary.
845
+ * The resulting state can be used for future compression operations with very limited startup cost.
775
846
  * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
776
- * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
777
- * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
778
- * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
847
+ * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
848
+ * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
849
+ * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
850
+ * in which case the only thing that it transports is the @compressionLevel.
851
+ * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
852
+ * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
779
853
  ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
780
854
  int compressionLevel);
781
855
 
@@ -876,7 +950,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
876
950
  * Reference a prepared dictionary, to be used for all next compressed frames.
877
951
  * Note that compression parameters are enforced from within CDict,
878
952
  * and supersede any compression parameter previously set within CCtx.
879
- * The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
953
+ * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
880
954
  * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
881
955
  * The dictionary will remain valid for future compressed frames using same CCtx.
882
956
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
@@ -902,7 +976,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
902
976
  * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
903
977
  * It's a CPU consuming operation, with non-negligible impact on latency.
904
978
  * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
905
- * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
979
+ * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
906
980
  * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
907
981
  ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
908
982
  const void* prefix, size_t prefixSize);
@@ -927,6 +1001,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
927
1001
  /*! ZSTD_DCtx_refDDict() :
928
1002
  * Reference a prepared dictionary, to be used to decompress next frames.
929
1003
  * The dictionary remains active for decompression of future frames using same DCtx.
1004
+ *
1005
+ * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
1006
+ * will store the DDict references in a table, and the DDict used for decompression
1007
+ * will be determined at decompression time, as per the dict ID in the frame.
1008
+ * The memory for the table is allocated on the first call to refDDict, and can be
1009
+ * freed with ZSTD_freeDCtx().
1010
+ *
930
1011
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
931
1012
  * Note 1 : Currently, only one dictionary can be managed.
932
1013
  * Referencing a new dictionary effectively "discards" any previous one.
@@ -946,7 +1027,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
946
1027
  * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
947
1028
  * Prefix buffer must remain unmodified up to the end of frame,
948
1029
  * reached when ZSTD_decompressStream() returns 0.
949
- * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
1030
+ * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
950
1031
  * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
951
1032
  * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
952
1033
  * A full dictionary is more costly, as it requires building tables.
@@ -969,7 +1050,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
969
1050
  #endif /* ZSTD_H_235446 */
970
1051
 
971
1052
 
972
- /****************************************************************************************
1053
+ /* **************************************************************************************
973
1054
  * ADVANCED AND EXPERIMENTAL FUNCTIONS
974
1055
  ****************************************************************************************
975
1056
  * The definitions in the following section are considered experimental.
@@ -991,8 +1072,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
991
1072
  * Some of them might be removed in the future (especially when redundant with existing stable functions)
992
1073
  * ***************************************************************************************/
993
1074
 
994
- #define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */
995
- #define ZSTD_FRAMEHEADERSIZE_MIN 6
1075
+ #define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
1076
+ #define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
996
1077
  #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
997
1078
  #define ZSTD_SKIPPABLEHEADERSIZE 8
998
1079
 
@@ -1037,6 +1118,12 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
1037
1118
  #define ZSTD_LDM_HASHRATELOG_MIN 0
1038
1119
  #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
1039
1120
 
1121
+ /* Advanced parameter bounds */
1122
+ #define ZSTD_TARGETCBLOCKSIZE_MIN 64
1123
+ #define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
1124
+ #define ZSTD_SRCSIZEHINT_MIN 0
1125
+ #define ZSTD_SRCSIZEHINT_MAX INT_MAX
1126
+
1040
1127
  /* internal */
1041
1128
  #define ZSTD_HASHLOG3_MAX 17
1042
1129
 
@@ -1045,6 +1132,43 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
1045
1132
 
1046
1133
  typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
1047
1134
 
1135
+ typedef struct {
1136
+ unsigned int offset; /* The offset of the match. (NOT the same as the offset code)
1137
+ * If offset == 0 and matchLength == 0, this sequence represents the last
1138
+ * literals in the block of litLength size.
1139
+ */
1140
+
1141
+ unsigned int litLength; /* Literal length of the sequence. */
1142
+ unsigned int matchLength; /* Match length of the sequence. */
1143
+
1144
+ /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
1145
+ * In this case, we will treat the sequence as a marker for a block boundary.
1146
+ */
1147
+
1148
+ unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'.
1149
+ * Ranges from [0, 3].
1150
+ *
1151
+ * Repeat offsets are essentially previous offsets from previous sequences sorted in
1152
+ * recency order. For more detail, see doc/zstd_compression_format.md
1153
+ *
1154
+ * If rep == 0, then 'offset' does not contain a repeat offset.
1155
+ * If rep > 0:
1156
+ * If litLength != 0:
1157
+ * rep == 1 --> offset == repeat_offset_1
1158
+ * rep == 2 --> offset == repeat_offset_2
1159
+ * rep == 3 --> offset == repeat_offset_3
1160
+ * If litLength == 0:
1161
+ * rep == 1 --> offset == repeat_offset_2
1162
+ * rep == 2 --> offset == repeat_offset_3
1163
+ * rep == 3 --> offset == repeat_offset_1 - 1
1164
+ *
1165
+ * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
1166
+ * 'rep', but repeat offsets do not necessarily need to be calculated from an external
1167
+ * sequence provider's perspective. For example, ZSTD_compressSequences() does not
1168
+ * use this 'rep' field at all (as of now).
1169
+ */
1170
+ } ZSTD_Sequence;
1171
+
1048
1172
  typedef struct {
1049
1173
  unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
1050
1174
  unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
@@ -1074,32 +1198,35 @@ typedef enum {
1074
1198
 
1075
1199
  typedef enum {
1076
1200
  ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
1077
- ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
1201
+ ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
1078
1202
  } ZSTD_dictLoadMethod_e;
1079
1203
 
1080
1204
  typedef enum {
1081
- /* Opened question : should we have a format ZSTD_f_auto ?
1082
- * Today, it would mean exactly the same as ZSTD_f_zstd1.
1083
- * But, in the future, should several formats become supported,
1084
- * on the compression side, it would mean "default format".
1085
- * On the decompression side, it would mean "automatic format detection",
1086
- * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
1087
- * Since meaning is a little different, another option could be to define different enums for compression and decompression.
1088
- * This question could be kept for later, when there are actually multiple formats to support,
1089
- * but there is also the question of pinning enum values, and pinning value `0` is especially important */
1090
1205
  ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
1091
- ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number.
1206
+ ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
1092
1207
  * Useful to save 4 bytes per generated frame.
1093
1208
  * Decoder cannot recognise automatically this format, requiring this instruction. */
1094
1209
  } ZSTD_format_e;
1095
1210
 
1211
+ typedef enum {
1212
+ /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
1213
+ ZSTD_d_validateChecksum = 0,
1214
+ ZSTD_d_ignoreChecksum = 1
1215
+ } ZSTD_forceIgnoreChecksum_e;
1216
+
1217
+ typedef enum {
1218
+ /* Note: this enum controls ZSTD_d_refMultipleDDicts */
1219
+ ZSTD_rmd_refSingleDDict = 0,
1220
+ ZSTD_rmd_refMultipleDDicts = 1
1221
+ } ZSTD_refMultipleDDicts_e;
1222
+
1096
1223
  typedef enum {
1097
1224
  /* Note: this enum and the behavior it controls are effectively internal
1098
1225
  * implementation details of the compressor. They are expected to continue
1099
1226
  * to evolve and should be considered only in the context of extremely
1100
1227
  * advanced performance tuning.
1101
1228
  *
1102
- * Zstd currently supports the use of a CDict in two ways:
1229
+ * Zstd currently supports the use of a CDict in three ways:
1103
1230
  *
1104
1231
  * - The contents of the CDict can be copied into the working context. This
1105
1232
  * means that the compression can search both the dictionary and input
@@ -1115,6 +1242,12 @@ typedef enum {
1115
1242
  * working context's tables can be reused). For small inputs, this can be
1116
1243
  * faster than copying the CDict's tables.
1117
1244
  *
1245
+ * - The CDict's tables are not used at all, and instead we use the working
1246
+ * context alone to reload the dictionary and use params based on the source
1247
+ * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
1248
+ * This method is effective when the dictionary sizes are very small relative
1249
+ * to the input size, and the input size is fairly large to begin with.
1250
+ *
1118
1251
  * Zstd has a simple internal heuristic that selects which strategy to use
1119
1252
  * at the beginning of a compression. However, if experimentation shows that
1120
1253
  * Zstd is making poor choices, it is possible to override that choice with
@@ -1123,6 +1256,7 @@ typedef enum {
1123
1256
  ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
1124
1257
  ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
1125
1258
  ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
1259
+ ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
1126
1260
  } ZSTD_dictAttachPref_e;
1127
1261
 
1128
1262
  typedef enum {
@@ -1131,7 +1265,7 @@ typedef enum {
1131
1265
  * levels will be compressed. */
1132
1266
  ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
1133
1267
  * emitted if Huffman compression is not profitable. */
1134
- ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */
1268
+ ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
1135
1269
  } ZSTD_literalCompressionMode_e;
1136
1270
 
1137
1271
 
@@ -1162,12 +1296,12 @@ typedef enum {
1162
1296
  * however it does mean that all frame data must be present and valid. */
1163
1297
  ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
1164
1298
 
1165
- /** ZSTD_decompressBound() :
1299
+ /*! ZSTD_decompressBound() :
1166
1300
  * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
1167
1301
  * `srcSize` must be the _exact_ size of this series
1168
1302
  * (i.e. there should be a frame boundary at `src + srcSize`)
1169
1303
  * @return : - upper-bound for the decompressed size of all data in all successive frames
1170
- * - if an error occured: ZSTD_CONTENTSIZE_ERROR
1304
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
1171
1305
  *
1172
1306
  * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
1173
1307
  * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
@@ -1183,6 +1317,92 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS
1183
1317
  * or an error code (if srcSize is too small) */
1184
1318
  ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
1185
1319
 
1320
+ typedef enum {
1321
+ ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
1322
+ ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
1323
+ } ZSTD_sequenceFormat_e;
1324
+
1325
+ /*! ZSTD_generateSequences() :
1326
+ * Generate sequences using ZSTD_compress2, given a source buffer.
1327
+ *
1328
+ * Each block will end with a dummy sequence
1329
+ * with offset == 0, matchLength == 0, and litLength == length of last literals.
1330
+ * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
1331
+ * simply acts as a block delimiter.
1332
+ *
1333
+ * zc can be used to insert custom compression params.
1334
+ * This function invokes ZSTD_compress2
1335
+ *
1336
+ * The output of this function can be fed into ZSTD_compressSequences() with CCtx
1337
+ * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
1338
+ * @return : number of sequences generated
1339
+ */
1340
+
1341
+ ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
1342
+ size_t outSeqsSize, const void* src, size_t srcSize);
1343
+
1344
+ /*! ZSTD_mergeBlockDelimiters() :
1345
+ * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
1346
+ * by merging them into into the literals of the next sequence.
1347
+ *
1348
+ * As such, the final generated result has no explicit representation of block boundaries,
1349
+ * and the final last literals segment is not represented in the sequences.
1350
+ *
1351
+ * The output of this function can be fed into ZSTD_compressSequences() with CCtx
1352
+ * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
1353
+ * @return : number of sequences left after merging
1354
+ */
1355
+ ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
1356
+
1357
+ /*! ZSTD_compressSequences() :
1358
+ * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
1359
+ * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
1360
+ * The entire source is compressed into a single frame.
1361
+ *
1362
+ * The compression behavior changes based on cctx params. In particular:
1363
+ * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
1364
+ * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
1365
+ * the block size derived from the cctx, and sequences may be split. This is the default setting.
1366
+ *
1367
+ * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
1368
+ * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
1369
+ *
1370
+ * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
1371
+ * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
1372
+ * specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
1373
+ *
1374
+ * In addition to the two adjustable experimental params, there are other important cctx params.
1375
+ * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
1376
+ * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
1377
+ * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
1378
+ * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
1379
+ *
1380
+ * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
1381
+ * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
1382
+ * and cannot emit an RLE block that disagrees with the repcode history
1383
+ * @return : final compressed size or a ZSTD error.
1384
+ */
1385
+ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
1386
+ const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
1387
+ const void* src, size_t srcSize);
1388
+
1389
+
1390
+ /*! ZSTD_writeSkippableFrame() :
1391
+ * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
1392
+ *
1393
+ * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
1394
+ * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
1395
+ * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
1396
+ * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
1397
+ *
1398
+ * Returns an error if destination buffer is not large enough, if the source size is not representable
1399
+ * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
1400
+ *
1401
+ * @return : number of bytes written or a ZSTD error.
1402
+ */
1403
+ ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
1404
+ const void* src, size_t srcSize, unsigned magicVariant);
1405
+
1186
1406
 
1187
1407
  /***************************************
1188
1408
  * Memory management
@@ -1191,12 +1411,26 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
1191
1411
  /*! ZSTD_estimate*() :
1192
1412
  * These functions make it possible to estimate memory usage
1193
1413
  * of a future {D,C}Ctx, before its creation.
1194
- * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
1195
- * It will also consider src size to be arbitrarily "large", which is worst case.
1196
- * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
1197
- * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
1198
- * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
1199
- * Note : CCtx size estimation is only correct for single-threaded compression. */
1414
+ *
1415
+ * ZSTD_estimateCCtxSize() will provide a memory budget large enough
1416
+ * for any compression level up to selected one.
1417
+ * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
1418
+ * does not include space for a window buffer.
1419
+ * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
1420
+ * The estimate will assume the input may be arbitrarily large,
1421
+ * which is the worst case.
1422
+ *
1423
+ * When srcSize can be bound by a known and rather "small" value,
1424
+ * this fact can be used to provide a tighter estimation
1425
+ * because the CCtx compression context will need less memory.
1426
+ * This tighter estimation can be provided by more advanced functions
1427
+ * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
1428
+ * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
1429
+ * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
1430
+ *
1431
+ * Note 2 : only single-threaded compression is supported.
1432
+ * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
1433
+ */
1200
1434
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
1201
1435
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
1202
1436
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
@@ -1279,7 +1513,11 @@ ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict(
1279
1513
  typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
1280
1514
  typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
1281
1515
  typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
1282
- static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
1516
+ static
1517
+ #ifdef __GNUC__
1518
+ __attribute__((__unused__))
1519
+ #endif
1520
+ ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
1283
1521
 
1284
1522
  ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
1285
1523
  ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
@@ -1292,11 +1530,37 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
1292
1530
  ZSTD_compressionParameters cParams,
1293
1531
  ZSTD_customMem customMem);
1294
1532
 
1295
- ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
1296
- ZSTD_dictLoadMethod_e dictLoadMethod,
1297
- ZSTD_dictContentType_e dictContentType,
1298
- ZSTD_customMem customMem);
1533
+ /* ! Thread pool :
1534
+ * These prototypes make it possible to share a thread pool among multiple compression contexts.
1535
+ * This can limit resources for applications with multiple threads where each one uses
1536
+ * a threaded compression mode (via ZSTD_c_nbWorkers parameter).
1537
+ * ZSTD_createThreadPool creates a new thread pool with a given number of threads.
1538
+ * Note that the lifetime of such pool must exist while being used.
1539
+ * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
1540
+ * to use an internal thread pool).
1541
+ * ZSTD_freeThreadPool frees a thread pool.
1542
+ */
1543
+ typedef struct POOL_ctx_s ZSTD_threadPool;
1544
+ ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
1545
+ ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);
1546
+ ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
1547
+
1299
1548
 
1549
+ /*
1550
+ * This API is temporary and is expected to change or disappear in the future!
1551
+ */
1552
+ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
1553
+ const void* dict, size_t dictSize,
1554
+ ZSTD_dictLoadMethod_e dictLoadMethod,
1555
+ ZSTD_dictContentType_e dictContentType,
1556
+ const ZSTD_CCtx_params* cctxParams,
1557
+ ZSTD_customMem customMem);
1558
+
1559
+ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(
1560
+ const void* dict, size_t dictSize,
1561
+ ZSTD_dictLoadMethod_e dictLoadMethod,
1562
+ ZSTD_dictContentType_e dictContentType,
1563
+ ZSTD_customMem customMem);
1300
1564
 
1301
1565
 
1302
1566
  /***************************************
@@ -1307,9 +1571,16 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS
1307
1571
  * Create a digested dictionary for compression
1308
1572
  * Dictionary content is just referenced, not duplicated.
1309
1573
  * As a consequence, `dictBuffer` **must** outlive CDict,
1310
- * and its content must remain unmodified throughout the lifetime of CDict. */
1574
+ * and its content must remain unmodified throughout the lifetime of CDict.
1575
+ * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
1311
1576
  ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
1312
1577
 
1578
+ /*! ZSTD_getDictID_fromCDict() :
1579
+ * Provides the dictID of the dictionary loaded into `cdict`.
1580
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
1581
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
1582
+ ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
1583
+
1313
1584
  /*! ZSTD_getCParams() :
1314
1585
  * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
1315
1586
  * `estimatedSrcSize` value is optional, select 0 if not known */
@@ -1334,7 +1605,9 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
1334
1605
  ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
1335
1606
 
1336
1607
  /*! ZSTD_compress_advanced() :
1337
- * Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */
1608
+ * Note : this function is now DEPRECATED.
1609
+ * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
1610
+ * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */
1338
1611
  ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
1339
1612
  void* dst, size_t dstCapacity,
1340
1613
  const void* src, size_t srcSize,
@@ -1342,7 +1615,9 @@ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
1342
1615
  ZSTD_parameters params);
1343
1616
 
1344
1617
  /*! ZSTD_compress_usingCDict_advanced() :
1345
- * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
1618
+ * Note : this function is now REDUNDANT.
1619
+ * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
1620
+ * This prototype will be marked as deprecated and generate compilation warning in some future version */
1346
1621
  ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
1347
1622
  void* dst, size_t dstCapacity,
1348
1623
  const void* src, size_t srcSize,
@@ -1409,12 +1684,160 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
1409
1684
  */
1410
1685
  #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
1411
1686
 
1687
+ /* Tries to fit compressed block size to be around targetCBlockSize.
1688
+ * No target when targetCBlockSize == 0.
1689
+ * There is no guarantee on compressed block size (default:0) */
1690
+ #define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
1691
+
1692
+ /* User's best guess of source size.
1693
+ * Hint is not valid when srcSizeHint == 0.
1694
+ * There is no guarantee that hint is close to actual source size,
1695
+ * but compression ratio may regress significantly if guess considerably underestimates */
1696
+ #define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
1697
+
1698
+ /* Controls whether the new and experimental "dedicated dictionary search
1699
+ * structure" can be used. This feature is still rough around the edges, be
1700
+ * prepared for surprising behavior!
1701
+ *
1702
+ * How to use it:
1703
+ *
1704
+ * When using a CDict, whether to use this feature or not is controlled at
1705
+ * CDict creation, and it must be set in a CCtxParams set passed into that
1706
+ * construction (via ZSTD_createCDict_advanced2()). A compression will then
1707
+ * use the feature or not based on how the CDict was constructed; the value of
1708
+ * this param, set in the CCtx, will have no effect.
1709
+ *
1710
+ * However, when a dictionary buffer is passed into a CCtx, such as via
1711
+ * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
1712
+ * whether the CDict that is created internally can use the feature or not.
1713
+ *
1714
+ * What it does:
1715
+ *
1716
+ * Normally, the internal data structures of the CDict are analogous to what
1717
+ * would be stored in a CCtx after compressing the contents of a dictionary.
1718
+ * To an approximation, a compression using a dictionary can then use those
1719
+ * data structures to simply continue what is effectively a streaming
1720
+ * compression where the simulated compression of the dictionary left off.
1721
+ * Which is to say, the search structures in the CDict are normally the same
1722
+ * format as in the CCtx.
1723
+ *
1724
+ * It is possible to do better, since the CDict is not like a CCtx: the search
1725
+ * structures are written once during CDict creation, and then are only read
1726
+ * after that, while the search structures in the CCtx are both read and
1727
+ * written as the compression goes along. This means we can choose a search
1728
+ * structure for the dictionary that is read-optimized.
1729
+ *
1730
+ * This feature enables the use of that different structure.
1731
+ *
1732
+ * Note that some of the members of the ZSTD_compressionParameters struct have
1733
+ * different semantics and constraints in the dedicated search structure. It is
1734
+ * highly recommended that you simply set a compression level in the CCtxParams
1735
+ * you pass into the CDict creation call, and avoid messing with the cParams
1736
+ * directly.
1737
+ *
1738
+ * Effects:
1739
+ *
1740
+ * This will only have any effect when the selected ZSTD_strategy
1741
+ * implementation supports this feature. Currently, that's limited to
1742
+ * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
1743
+ *
1744
+ * Note that this means that the CDict tables can no longer be copied into the
1745
+ * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
1746
+ * useable. The dictionary can only be attached or reloaded.
1747
+ *
1748
+ * In general, you should expect compression to be faster--sometimes very much
1749
+ * so--and CDict creation to be slightly slower. Eventually, we will probably
1750
+ * make this mode the default.
1751
+ */
1752
+ #define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
1753
+
1754
+ /* ZSTD_c_stableInBuffer
1755
+ * Experimental parameter.
1756
+ * Default is 0 == disabled. Set to 1 to enable.
1757
+ *
1758
+ * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same
1759
+ * between calls, except for the modifications that zstd makes to pos (the
1760
+ * caller must not modify pos). This is checked by the compressor, and
1761
+ * compression will fail if it ever changes. This means the only flush
1762
+ * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end
1763
+ * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos)
1764
+ * MUST not be modified during compression or you will get data corruption.
1765
+ *
1766
+ * When this flag is enabled zstd won't allocate an input window buffer,
1767
+ * because the user guarantees it can reference the ZSTD_inBuffer until
1768
+ * the frame is complete. But, it will still allocate an output buffer
1769
+ * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
1770
+ * avoid the memcpy() from the input buffer to the input window buffer.
1771
+ *
1772
+ * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
1773
+ * That means this flag cannot be used with ZSTD_compressStream().
1774
+ *
1775
+ * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
1776
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
1777
+ * memory. However, compression WILL fail if you violate the preconditions.
1778
+ *
1779
+ * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST
1780
+ * not be modified during compression or you will get data corruption. This
1781
+ * is because zstd needs to reference data in the ZSTD_inBuffer to find
1782
+ * matches. Normally zstd maintains its own window buffer for this purpose,
1783
+ * but passing this flag tells zstd to use the user provided buffer.
1784
+ */
1785
+ #define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
1786
+
1787
+ /* ZSTD_c_stableOutBuffer
1788
+ * Experimental parameter.
1789
+ * Default is 0 == disabled. Set to 1 to enable.
1790
+ *
1791
+ * Tells he compressor that the ZSTD_outBuffer will not be resized between
1792
+ * calls. Specifically: (out.size - out.pos) will never grow. This gives the
1793
+ * compressor the freedom to say: If the compressed data doesn't fit in the
1794
+ * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
1795
+ * always decompress directly into the output buffer, instead of decompressing
1796
+ * into an internal buffer and copying to the output buffer.
1797
+ *
1798
+ * When this flag is enabled zstd won't allocate an output buffer, because
1799
+ * it can write directly to the ZSTD_outBuffer. It will still allocate the
1800
+ * input window buffer (see ZSTD_c_stableInBuffer).
1801
+ *
1802
+ * Zstd will check that (out.size - out.pos) never grows and return an error
1803
+ * if it does. While not strictly necessary, this should prevent surprises.
1804
+ */
1805
+ #define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
1806
+
1807
+ /* ZSTD_c_blockDelimiters
1808
+ * Default is 0 == ZSTD_sf_noBlockDelimiters.
1809
+ *
1810
+ * For use with sequence compression API: ZSTD_compressSequences().
1811
+ *
1812
+ * Designates whether or not the given array of ZSTD_Sequence contains block delimiters
1813
+ * and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
1814
+ * See the definition of ZSTD_Sequence for more specifics.
1815
+ */
1816
+ #define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
1817
+
1818
+ /* ZSTD_c_validateSequences
1819
+ * Default is 0 == disabled. Set to 1 to enable sequence validation.
1820
+ *
1821
+ * For use with sequence compression API: ZSTD_compressSequences().
1822
+ * Designates whether or not we validate sequences provided to ZSTD_compressSequences()
1823
+ * during function execution.
1824
+ *
1825
+ * Without validation, providing a sequence that does not conform to the zstd spec will cause
1826
+ * undefined behavior, and may produce a corrupted block.
1827
+ *
1828
+ * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for
1829
+ * specifics regarding offset/matchlength requirements) then the function will bail out and
1830
+ * return an error.
1831
+ *
1832
+ */
1833
+ #define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
1834
+
1412
1835
  /*! ZSTD_CCtx_getParameter() :
1413
1836
  * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
1414
1837
  * and store it into int* value.
1415
1838
  * @return : 0, or an error code (which can be tested with ZSTD_isError()).
1416
1839
  */
1417
- ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
1840
+ ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
1418
1841
 
1419
1842
 
1420
1843
  /*! ZSTD_CCtx_params :
@@ -1457,8 +1880,10 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, Z
1457
1880
  /*! ZSTD_CCtxParams_setParameter() :
1458
1881
  * Similar to ZSTD_CCtx_setParameter.
1459
1882
  * Set one compression parameter, selected by enum ZSTD_cParameter.
1460
- * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
1461
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1883
+ * Parameters must be applied to a ZSTD_CCtx using
1884
+ * ZSTD_CCtx_setParametersUsingCCtxParams().
1885
+ * @result : a code representing success or failure (which can be tested with
1886
+ * ZSTD_isError()).
1462
1887
  */
1463
1888
  ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
1464
1889
 
@@ -1467,7 +1892,7 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_c
1467
1892
  * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
1468
1893
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1469
1894
  */
1470
- ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
1895
+ ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
1471
1896
 
1472
1897
  /*! ZSTD_CCtx_setParametersUsingCCtxParams() :
1473
1898
  * Apply a set of ZSTD_CCtx_params to the compression context.
@@ -1538,11 +1963,84 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre
1538
1963
  */
1539
1964
  ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
1540
1965
 
1966
+ /*! ZSTD_DCtx_getParameter() :
1967
+ * Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
1968
+ * and store it into int* value.
1969
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
1970
+ */
1971
+ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
1972
+
1541
1973
  /* ZSTD_d_format
1542
1974
  * experimental parameter,
1543
1975
  * allowing selection between ZSTD_format_e input compression formats
1544
1976
  */
1545
1977
  #define ZSTD_d_format ZSTD_d_experimentalParam1
1978
+ /* ZSTD_d_stableOutBuffer
1979
+ * Experimental parameter.
1980
+ * Default is 0 == disabled. Set to 1 to enable.
1981
+ *
1982
+ * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
1983
+ * between calls, except for the modifications that zstd makes to pos (the
1984
+ * caller must not modify pos). This is checked by the decompressor, and
1985
+ * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
1986
+ * MUST be large enough to fit the entire decompressed frame. This will be
1987
+ * checked when the frame content size is known. The data in the ZSTD_outBuffer
1988
+ * in the range [dst, dst + pos) MUST not be modified during decompression
1989
+ * or you will get data corruption.
1990
+ *
1991
+ * When this flags is enabled zstd won't allocate an output buffer, because
1992
+ * it can write directly to the ZSTD_outBuffer, but it will still allocate
1993
+ * an input buffer large enough to fit any compressed block. This will also
1994
+ * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
1995
+ * If you need to avoid the input buffer allocation use the buffer-less
1996
+ * streaming API.
1997
+ *
1998
+ * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
1999
+ * this flag is ALWAYS memory safe, and will never access out-of-bounds
2000
+ * memory. However, decompression WILL fail if you violate the preconditions.
2001
+ *
2002
+ * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
2003
+ * not be modified during decompression or you will get data corruption. This
2004
+ * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
2005
+ * matches. Normally zstd maintains its own buffer for this purpose, but passing
2006
+ * this flag tells zstd to use the user provided buffer.
2007
+ */
2008
+ #define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
2009
+
2010
+ /* ZSTD_d_forceIgnoreChecksum
2011
+ * Experimental parameter.
2012
+ * Default is 0 == disabled. Set to 1 to enable
2013
+ *
2014
+ * Tells the decompressor to skip checksum validation during decompression, regardless
2015
+ * of whether checksumming was specified during compression. This offers some
2016
+ * slight performance benefits, and may be useful for debugging.
2017
+ * Param has values of type ZSTD_forceIgnoreChecksum_e
2018
+ */
2019
+ #define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
2020
+
2021
+ /* ZSTD_d_refMultipleDDicts
2022
+ * Experimental parameter.
2023
+ * Default is 0 == disabled. Set to 1 to enable
2024
+ *
2025
+ * If enabled and dctx is allocated on the heap, then additional memory will be allocated
2026
+ * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
2027
+ * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
2028
+ * store all references. At decompression time, the appropriate dictID is selected
2029
+ * from the set of DDicts based on the dictID in the frame.
2030
+ *
2031
+ * Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
2032
+ *
2033
+ * Param has values of byte ZSTD_refMultipleDDicts_e
2034
+ *
2035
+ * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
2036
+ * allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
2037
+ * Memory is allocated as per ZSTD_DCtx::customMem.
2038
+ *
2039
+ * Although this function allocates memory for the table, the user is still responsible for
2040
+ * memory management of the underlying ZSTD_DDict* themselves.
2041
+ */
2042
+ #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
2043
+
1546
2044
 
1547
2045
  /*! ZSTD_DCtx_setFormat() :
1548
2046
  * Instruct the decoder context about what kind of data to decode next.
@@ -1571,7 +2069,8 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
1571
2069
  ********************************************************************/
1572
2070
 
1573
2071
  /*===== Advanced Streaming compression functions =====*/
1574
- /**! ZSTD_initCStream_srcSize() :
2072
+
2073
+ /*! ZSTD_initCStream_srcSize() :
1575
2074
  * This function is deprecated, and equivalent to:
1576
2075
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1577
2076
  * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
@@ -1581,9 +2080,14 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
1581
2080
  * pledgedSrcSize must be correct. If it is not known at init time, use
1582
2081
  * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
1583
2082
  * "0" also disables frame content size field. It may be enabled in the future.
2083
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1584
2084
  */
1585
- ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
1586
- /**! ZSTD_initCStream_usingDict() :
2085
+ ZSTDLIB_API size_t
2086
+ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
2087
+ int compressionLevel,
2088
+ unsigned long long pledgedSrcSize);
2089
+
2090
+ /*! ZSTD_initCStream_usingDict() :
1587
2091
  * This function is deprecated, and is equivalent to:
1588
2092
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1589
2093
  * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
@@ -1591,42 +2095,66 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
1591
2095
  *
1592
2096
  * Creates of an internal CDict (incompatible with static CCtx), except if
1593
2097
  * dict == NULL or dictSize < 8, in which case no dict is used.
1594
- * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
2098
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
1595
2099
  * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
2100
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1596
2101
  */
1597
- ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
1598
- /**! ZSTD_initCStream_advanced() :
2102
+ ZSTDLIB_API size_t
2103
+ ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
2104
+ const void* dict, size_t dictSize,
2105
+ int compressionLevel);
2106
+
2107
+ /*! ZSTD_initCStream_advanced() :
1599
2108
  * This function is deprecated, and is approximately equivalent to:
1600
2109
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1601
- * ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is
2110
+ * // Pseudocode: Set each zstd parameter and leave the rest as-is.
2111
+ * for ((param, value) : params) {
2112
+ * ZSTD_CCtx_setParameter(zcs, param, value);
2113
+ * }
1602
2114
  * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
1603
2115
  * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
1604
2116
  *
1605
- * pledgedSrcSize must be correct. If srcSize is not known at init time, use
1606
- * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
2117
+ * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
2118
+ * pledgedSrcSize must be correct.
2119
+ * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
2120
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1607
2121
  */
1608
- ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
1609
- ZSTD_parameters params, unsigned long long pledgedSrcSize);
1610
- /**! ZSTD_initCStream_usingCDict() :
2122
+ ZSTDLIB_API size_t
2123
+ ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
2124
+ const void* dict, size_t dictSize,
2125
+ ZSTD_parameters params,
2126
+ unsigned long long pledgedSrcSize);
2127
+
2128
+ /*! ZSTD_initCStream_usingCDict() :
1611
2129
  * This function is deprecated, and equivalent to:
1612
2130
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1613
2131
  * ZSTD_CCtx_refCDict(zcs, cdict);
1614
2132
  *
1615
2133
  * note : cdict will just be referenced, and must outlive compression session
2134
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1616
2135
  */
1617
2136
  ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
1618
- /**! ZSTD_initCStream_usingCDict_advanced() :
1619
- * This function is deprecated, and is approximately equivalent to:
2137
+
2138
+ /*! ZSTD_initCStream_usingCDict_advanced() :
2139
+ * This function is DEPRECATED, and is approximately equivalent to:
1620
2140
  * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1621
- * ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is
2141
+ * // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
2142
+ * for ((fParam, value) : fParams) {
2143
+ * ZSTD_CCtx_setParameter(zcs, fParam, value);
2144
+ * }
1622
2145
  * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
1623
2146
  * ZSTD_CCtx_refCDict(zcs, cdict);
1624
2147
  *
1625
2148
  * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
1626
2149
  * pledgedSrcSize must be correct. If srcSize is not known at init time, use
1627
2150
  * value ZSTD_CONTENTSIZE_UNKNOWN.
2151
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1628
2152
  */
1629
- ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
2153
+ ZSTDLIB_API size_t
2154
+ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
2155
+ const ZSTD_CDict* cdict,
2156
+ ZSTD_frameParameters fParams,
2157
+ unsigned long long pledgedSrcSize);
1630
2158
 
1631
2159
  /*! ZSTD_resetCStream() :
1632
2160
  * This function is deprecated, and is equivalent to:
@@ -1641,6 +2169,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
1641
2169
  * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
1642
2170
  * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
1643
2171
  * @return : 0, or an error code (which can be tested using ZSTD_isError())
2172
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1644
2173
  */
1645
2174
  ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
1646
2175
 
@@ -1679,30 +2208,36 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
1679
2208
 
1680
2209
 
1681
2210
  /*===== Advanced Streaming decompression functions =====*/
1682
- /**
2211
+
2212
+ /*!
1683
2213
  * This function is deprecated, and is equivalent to:
1684
2214
  *
1685
2215
  * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
1686
2216
  * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
1687
2217
  *
1688
2218
  * note: no dictionary will be used if dict == NULL or dictSize < 8
2219
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1689
2220
  */
1690
2221
  ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
1691
- /**
2222
+
2223
+ /*!
1692
2224
  * This function is deprecated, and is equivalent to:
1693
2225
  *
1694
2226
  * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
1695
2227
  * ZSTD_DCtx_refDDict(zds, ddict);
1696
2228
  *
1697
2229
  * note : ddict is referenced, it must outlive decompression session
2230
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1698
2231
  */
1699
2232
  ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
1700
- /**
2233
+
2234
+ /*!
1701
2235
  * This function is deprecated, and is equivalent to:
1702
2236
  *
1703
2237
  * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
1704
2238
  *
1705
2239
  * re-use decompression parameters from previous init; saves dictionary loading
2240
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
1706
2241
  */
1707
2242
  ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
1708
2243
 
@@ -1758,7 +2293,7 @@ ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstC
1758
2293
  ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
1759
2294
 
1760
2295
 
1761
- /*-
2296
+ /**
1762
2297
  Buffer-less streaming decompression (synchronous mode)
1763
2298
 
1764
2299
  A ZSTD_DCtx object is required to track streaming operations.
@@ -1843,7 +2378,7 @@ typedef struct {
1843
2378
  unsigned checksumFlag;
1844
2379
  } ZSTD_frameHeader;
1845
2380
 
1846
- /** ZSTD_getFrameHeader() :
2381
+ /*! ZSTD_getFrameHeader() :
1847
2382
  * decode Frame Header, or requires larger `srcSize`.
1848
2383
  * @return : 0, `zfhPtr` is correctly filled,
1849
2384
  * >0, `srcSize` is too small, value is wanted `srcSize` amount,
@@ -1876,8 +2411,8 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
1876
2411
 
1877
2412
  /*!
1878
2413
  Block functions produce and decode raw zstd blocks, without frame metadata.
1879
- Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
1880
- User will have to take in charge required information to regenerate data, such as compressed and content sizes.
2414
+ Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
2415
+ But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
1881
2416
 
1882
2417
  A few rules to respect :
1883
2418
  - Compressing and decompressing require a context structure
@@ -1888,12 +2423,14 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
1888
2423
  + copyCCtx() and copyDCtx() can be used too
1889
2424
  - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
1890
2425
  + If input is larger than a block size, it's necessary to split input data into multiple blocks
1891
- + For inputs larger than a single block, really consider using regular ZSTD_compress() instead.
1892
- Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
1893
- - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
1894
- In which case, nothing is produced into `dst` !
1895
- + User must test for such outcome and deal directly with uncompressed data
1896
- + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
2426
+ + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
2427
+ Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
2428
+ - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
2429
+ ===> In which case, nothing is produced into `dst` !
2430
+ + User __must__ test for such outcome and deal directly with uncompressed data
2431
+ + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
2432
+ Doing so would mess up with statistics history, leading to potential data corruption.
2433
+ + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
1897
2434
  + In case of multiple successive blocks, should some of them be uncompressed,
1898
2435
  decoder must be informed of their existence in order to follow proper history.
1899
2436
  Use ZSTD_insertBlock() for such a case.