extzstd 0.2 → 0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja.md +13 -0
  3. data/README.md +17 -14
  4. data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/Makefile +99 -53
  7. data/contrib/zstd/README.md +59 -39
  8. data/contrib/zstd/TESTING.md +1 -1
  9. data/contrib/zstd/appveyor.yml +17 -6
  10. data/contrib/zstd/lib/BUCK +29 -2
  11. data/contrib/zstd/lib/Makefile +118 -21
  12. data/contrib/zstd/lib/README.md +84 -44
  13. data/contrib/zstd/lib/common/bitstream.h +17 -33
  14. data/contrib/zstd/lib/common/compiler.h +62 -8
  15. data/contrib/zstd/lib/common/cpu.h +215 -0
  16. data/contrib/zstd/lib/common/debug.c +44 -0
  17. data/contrib/zstd/lib/common/debug.h +134 -0
  18. data/contrib/zstd/lib/common/entropy_common.c +16 -1
  19. data/contrib/zstd/lib/common/error_private.c +7 -0
  20. data/contrib/zstd/lib/common/fse.h +48 -44
  21. data/contrib/zstd/lib/common/fse_decompress.c +3 -3
  22. data/contrib/zstd/lib/common/huf.h +169 -113
  23. data/contrib/zstd/lib/common/mem.h +20 -2
  24. data/contrib/zstd/lib/common/pool.c +135 -49
  25. data/contrib/zstd/lib/common/pool.h +40 -21
  26. data/contrib/zstd/lib/common/threading.c +2 -2
  27. data/contrib/zstd/lib/common/threading.h +12 -12
  28. data/contrib/zstd/lib/common/xxhash.c +3 -2
  29. data/contrib/zstd/lib/common/zstd_common.c +3 -6
  30. data/contrib/zstd/lib/common/zstd_errors.h +17 -7
  31. data/contrib/zstd/lib/common/zstd_internal.h +76 -48
  32. data/contrib/zstd/lib/compress/fse_compress.c +89 -209
  33. data/contrib/zstd/lib/compress/hist.c +203 -0
  34. data/contrib/zstd/lib/compress/hist.h +95 -0
  35. data/contrib/zstd/lib/compress/huf_compress.c +188 -80
  36. data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
  37. data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
  38. data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
  39. data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
  40. data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
  41. data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
  42. data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
  43. data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
  44. data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
  45. data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
  46. data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
  47. data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
  48. data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
  49. data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
  50. data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
  51. data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
  52. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  53. data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
  54. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
  55. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
  56. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
  57. data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
  58. data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
  59. data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
  60. data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
  61. data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
  62. data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
  63. data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
  64. data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
  65. data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
  66. data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
  67. data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
  68. data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
  69. data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
  70. data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
  71. data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
  72. data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
  73. data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
  74. data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
  75. data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
  76. data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
  77. data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
  78. data/contrib/zstd/lib/zstd.h +1346 -832
  79. data/ext/extzstd.c +27 -19
  80. data/ext/extzstd_stream.c +20 -4
  81. data/ext/zstd_compress.c +1 -0
  82. data/ext/zstd_decompress.c +4 -0
  83. data/ext/zstd_dictbuilder.c +4 -0
  84. data/ext/zstd_dictbuilder_fastcover.c +5 -0
  85. data/lib/extzstd.rb +52 -220
  86. data/lib/extzstd/version.rb +1 -1
  87. metadata +21 -7
  88. data/contrib/zstd/circle.yml +0 -63
@@ -43,12 +43,17 @@ ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
43
43
  const void* src, size_t compressedSize);
44
44
 
45
45
  /**
46
- ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
47
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
48
- return : the number of bytes that would be read to decompress this frame
49
- or an errorCode if it fails (which can be tested using ZSTDv06_isError())
46
+ ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format
47
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
48
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
49
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
50
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
51
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
52
+
53
+ note : assumes `cSize` and `dBound` are _not_ NULL.
50
54
  */
51
- size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t compressedSize);
55
+ void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
56
+ size_t* cSize, unsigned long long* dBound);
52
57
 
53
58
  /* *************************************
54
59
  * Helper functions
@@ -348,7 +348,7 @@ MEM_STATIC U32 MEM_swap32(U32 in)
348
348
  {
349
349
  #if defined(_MSC_VER) /* Visual Studio */
350
350
  return _byteswap_ulong(in);
351
- #elif defined (__GNUC__)
351
+ #elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
352
352
  return __builtin_bswap32(in);
353
353
  #else
354
354
  return ((in << 24) & 0xff000000 ) |
@@ -362,7 +362,7 @@ MEM_STATIC U64 MEM_swap64(U64 in)
362
362
  {
363
363
  #if defined(_MSC_VER) /* Visual Studio */
364
364
  return _byteswap_uint64(in);
365
- #elif defined (__GNUC__)
365
+ #elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
366
366
  return __builtin_bswap64(in);
367
367
  #else
368
368
  return ((in << 56) & 0xff00000000000000ULL) |
@@ -2628,7 +2628,7 @@ const char* ZBUFFv07_getErrorName(size_t errorCode) { return ERR_getErrorName(er
2628
2628
 
2629
2629
 
2630
2630
 
2631
- void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size)
2631
+ static void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size)
2632
2632
  {
2633
2633
  void* address = malloc(size);
2634
2634
  (void)opaque;
@@ -2636,7 +2636,7 @@ void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size)
2636
2636
  return address;
2637
2637
  }
2638
2638
 
2639
- void ZSTDv07_defaultFreeFunction(void* opaque, void* address)
2639
+ static void ZSTDv07_defaultFreeFunction(void* opaque, void* address)
2640
2640
  {
2641
2641
  (void)opaque;
2642
2642
  /* if (address) printf("free %p opaque=%p \n", address, opaque); */
@@ -2740,6 +2740,8 @@ typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t;
2740
2740
  #define FSEv07_ENCODING_STATIC 2
2741
2741
  #define FSEv07_ENCODING_DYNAMIC 3
2742
2742
 
2743
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2744
+
2743
2745
  static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2744
2746
  1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
2745
2747
  13,14,15,16 };
@@ -3150,10 +3152,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
3150
3152
  const BYTE* ip = (const BYTE*)src;
3151
3153
 
3152
3154
  if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min;
3155
+ memset(fparamsPtr, 0, sizeof(*fparamsPtr));
3153
3156
  if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
3154
3157
  if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) {
3155
3158
  if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */
3156
- memset(fparamsPtr, 0, sizeof(*fparamsPtr));
3157
3159
  fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
3158
3160
  fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
3159
3161
  return 0;
@@ -3175,11 +3177,13 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
3175
3177
  U32 windowSize = 0;
3176
3178
  U32 dictID = 0;
3177
3179
  U64 frameContentSize = 0;
3178
- if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
3180
+ if ((fhdByte & 0x08) != 0) /* reserved bits, which must be zero */
3181
+ return ERROR(frameParameter_unsupported);
3179
3182
  if (!directMode) {
3180
3183
  BYTE const wlByte = ip[pos++];
3181
3184
  U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN;
3182
- if (windowLog > ZSTDv07_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
3185
+ if (windowLog > ZSTDv07_WINDOWLOG_MAX)
3186
+ return ERROR(frameParameter_unsupported);
3183
3187
  windowSize = (1U << windowLog);
3184
3188
  windowSize += (windowSize >> 3) * (wlByte&7);
3185
3189
  }
@@ -3201,7 +3205,8 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
3201
3205
  case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
3202
3206
  }
3203
3207
  if (!windowSize) windowSize = (U32)frameContentSize;
3204
- if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
3208
+ if (windowSize > windowSizeMax)
3209
+ return ERROR(frameParameter_unsupported);
3205
3210
  fparamsPtr->frameContentSize = frameContentSize;
3206
3211
  fparamsPtr->windowSize = windowSize;
3207
3212
  fparamsPtr->dictID = dictID;
@@ -3220,11 +3225,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
3220
3225
  - frame header not completely provided (`srcSize` too small) */
3221
3226
  unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize)
3222
3227
  {
3223
- { ZSTDv07_frameParams fparams;
3224
- size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize);
3225
- if (frResult!=0) return 0;
3226
- return fparams.frameContentSize;
3227
- }
3228
+ ZSTDv07_frameParams fparams;
3229
+ size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize);
3230
+ if (frResult!=0) return 0;
3231
+ return fparams.frameContentSize;
3228
3232
  }
3229
3233
 
3230
3234
 
@@ -3248,7 +3252,7 @@ typedef struct
3248
3252
 
3249
3253
  /*! ZSTDv07_getcBlockSize() :
3250
3254
  * Provides the size of compressed block from block header `src` */
3251
- size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
3255
+ static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
3252
3256
  {
3253
3257
  const BYTE* const in = (const BYTE* const)src;
3254
3258
  U32 cSize;
@@ -3275,7 +3279,7 @@ static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* sr
3275
3279
 
3276
3280
  /*! ZSTDv07_decodeLiteralsBlock() :
3277
3281
  @return : nb of bytes read from src (< srcSize ) */
3278
- size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
3282
+ static size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
3279
3283
  const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
3280
3284
  {
3281
3285
  const BYTE* const istart = (const BYTE*) src;
@@ -3409,7 +3413,7 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
3409
3413
  @return : nb bytes read from src,
3410
3414
  or an error code if it fails, testable with ZSTDv07_isError()
3411
3415
  */
3412
- size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLog,
3416
+ static size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLog,
3413
3417
  const void* src, size_t srcSize,
3414
3418
  const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
3415
3419
  {
@@ -3439,7 +3443,7 @@ size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLo
3439
3443
  }
3440
3444
 
3441
3445
 
3442
- size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
3446
+ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
3443
3447
  FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable,
3444
3448
  const void* src, size_t srcSize)
3445
3449
  {
@@ -3629,7 +3633,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
3629
3633
  if (sequence.offset < 8) {
3630
3634
  /* close range match, overlap */
3631
3635
  static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3632
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
3636
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3633
3637
  int const sub2 = dec64table[sequence.offset];
3634
3638
  op[0] = match[0];
3635
3639
  op[1] = match[1];
@@ -3771,7 +3775,7 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
3771
3775
  }
3772
3776
 
3773
3777
 
3774
- size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
3778
+ static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
3775
3779
  {
3776
3780
  if (length > dstCapacity) return ERROR(dstSize_tooSmall);
3777
3781
  memset(dst, byte, length);
@@ -3851,7 +3855,7 @@ static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx,
3851
3855
  * It avoids reloading the dictionary each time.
3852
3856
  * `preparedDCtx` must have been properly initialized using ZSTDv07_decompressBegin_usingDict().
3853
3857
  * Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
3854
- size_t ZSTDv07_decompress_usingPreparedDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* refDCtx,
3858
+ static size_t ZSTDv07_decompress_usingPreparedDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* refDCtx,
3855
3859
  void* dst, size_t dstCapacity,
3856
3860
  const void* src, size_t srcSize)
3857
3861
  {
@@ -3893,19 +3897,40 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
3893
3897
  #endif
3894
3898
  }
3895
3899
 
3896
- size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
3900
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3901
+ assumes `cSize` and `dBound` are _not_ NULL */
3902
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3903
+ {
3904
+ *cSize = ret;
3905
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3906
+ }
3907
+
3908
+ void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3897
3909
  {
3898
3910
  const BYTE* ip = (const BYTE*)src;
3899
3911
  size_t remainingSize = srcSize;
3912
+ size_t nbBlocks = 0;
3900
3913
 
3901
3914
  /* check */
3902
- if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
3915
+ if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) {
3916
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3917
+ return;
3918
+ }
3903
3919
 
3904
3920
  /* Frame Header */
3905
3921
  { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
3906
- if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize;
3907
- if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) return ERROR(prefix_unknown);
3908
- if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
3922
+ if (ZSTDv07_isError(frameHeaderSize)) {
3923
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
3924
+ return;
3925
+ }
3926
+ if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
3927
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3928
+ return;
3929
+ }
3930
+ if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) {
3931
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3932
+ return;
3933
+ }
3909
3934
  ip += frameHeaderSize; remainingSize -= frameHeaderSize;
3910
3935
  }
3911
3936
 
@@ -3913,20 +3938,28 @@ size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
3913
3938
  while (1) {
3914
3939
  blockProperties_t blockProperties;
3915
3940
  size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties);
3916
- if (ZSTDv07_isError(cBlockSize)) return cBlockSize;
3941
+ if (ZSTDv07_isError(cBlockSize)) {
3942
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3943
+ return;
3944
+ }
3917
3945
 
3918
3946
  ip += ZSTDv07_blockHeaderSize;
3919
3947
  remainingSize -= ZSTDv07_blockHeaderSize;
3920
3948
 
3921
3949
  if (blockProperties.blockType == bt_end) break;
3922
3950
 
3923
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3951
+ if (cBlockSize > remainingSize) {
3952
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3953
+ return;
3954
+ }
3924
3955
 
3925
3956
  ip += cBlockSize;
3926
3957
  remainingSize -= cBlockSize;
3958
+ nbBlocks++;
3927
3959
  }
3928
3960
 
3929
- return ip - (const BYTE*)src;
3961
+ *cSize = ip - (const BYTE*)src;
3962
+ *dBound = nbBlocks * ZSTDv07_BLOCKSIZE_ABSOLUTEMAX;
3930
3963
  }
3931
3964
 
3932
3965
  /*_******************************
@@ -4146,7 +4179,7 @@ struct ZSTDv07_DDict_s {
4146
4179
  ZSTDv07_DCtx* refContext;
4147
4180
  }; /* typedef'd tp ZSTDv07_CDict within zstd.h */
4148
4181
 
4149
- ZSTDv07_DDict* ZSTDv07_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv07_customMem customMem)
4182
+ static ZSTDv07_DDict* ZSTDv07_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv07_customMem customMem)
4150
4183
  {
4151
4184
  if (!customMem.customAlloc && !customMem.customFree)
4152
4185
  customMem = defaultCustomMem;
@@ -50,12 +50,17 @@ ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
50
50
  const void* src, size_t compressedSize);
51
51
 
52
52
  /**
53
- ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame
54
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
55
- return : the number of bytes that would be read to decompress this frame
56
- or an errorCode if it fails (which can be tested using ZSTDv07_isError())
53
+ ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.7.x format
54
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
55
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
56
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
57
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
58
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
59
+
60
+ note : assumes `cSize` and `dBound` are _not_ NULL.
57
61
  */
58
- size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t compressedSize);
62
+ void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
63
+ size_t* cSize, unsigned long long* dBound);
59
64
 
60
65
  /*====== Helper functions ======*/
61
66
  ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
@@ -35,40 +35,73 @@ extern "C" {
35
35
  #endif
36
36
 
37
37
 
38
- /*******************************************************************************************************
38
+ /*******************************************************************************
39
39
  Introduction
40
40
 
41
- zstd, short for Zstandard, is a fast lossless compression algorithm,
42
- targeting real-time compression scenarios at zlib-level and better compression ratios.
43
- The zstd compression library provides in-memory compression and decompression functions.
44
- The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
45
- Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
41
+ zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
42
+ real-time compression scenarios at zlib-level and better compression ratios.
43
+ The zstd compression library provides in-memory compression and decompression
44
+ functions.
45
+
46
+ The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
47
+ which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
48
+ caution, as they require more memory. The library also offers negative
49
+ compression levels, which extend the range of speed vs. ratio preferences.
50
+ The lower the level, the faster the speed (at the cost of compression).
51
+
46
52
  Compression can be done in:
47
53
  - a single step (described as Simple API)
48
- - a single step, reusing a context (described as Explicit memory management)
54
+ - a single step, reusing a context (described as Explicit context)
49
55
  - unbounded multiple steps (described as Streaming compression)
50
- The compression ratio achievable on small data can be highly improved using a dictionary in:
56
+
57
+ The compression ratio achievable on small data can be highly improved using
58
+ a dictionary. Dictionary compression can be performed in:
51
59
  - a single step (described as Simple dictionary API)
52
- - a single step, reusing a dictionary (described as Fast dictionary API)
60
+ - a single step, reusing a dictionary (described as Bulk-processing
61
+ dictionary API)
62
+
63
+ Advanced experimental functions can be accessed using
64
+ `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
53
65
 
54
- Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
55
- Advanced experimental APIs shall never be used with a dynamic library.
56
- They are not "stable", their definition may change in the future. Only static linking is allowed.
57
- *********************************************************************************************************/
66
+ Advanced experimental APIs should never be used with a dynamically-linked
67
+ library. They are not "stable"; their definitions or signatures may change in
68
+ the future. Only static linking is allowed.
69
+ *******************************************************************************/
58
70
 
59
71
  /*------ Version ------*/
60
72
  #define ZSTD_VERSION_MAJOR 1
61
- #define ZSTD_VERSION_MINOR 3
62
- #define ZSTD_VERSION_RELEASE 3
73
+ #define ZSTD_VERSION_MINOR 4
74
+ #define ZSTD_VERSION_RELEASE 0
63
75
 
64
76
  #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
65
- ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */
77
+ ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */
66
78
 
67
79
  #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
68
80
  #define ZSTD_QUOTE(str) #str
69
81
  #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
70
82
  #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
71
- ZSTDLIB_API const char* ZSTD_versionString(void); /* v1.3.0 */
83
+ ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */
84
+
85
+ /***************************************
86
+ * Default constant
87
+ ***************************************/
88
+ #ifndef ZSTD_CLEVEL_DEFAULT
89
+ # define ZSTD_CLEVEL_DEFAULT 3
90
+ #endif
91
+
92
+ /***************************************
93
+ * Constants
94
+ ***************************************/
95
+
96
+ /* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
97
+ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
98
+ #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */
99
+ #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
100
+ #define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0
101
+
102
+ #define ZSTD_BLOCKSIZELOG_MAX 17
103
+ #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
104
+
72
105
 
73
106
 
74
107
  /***************************************
@@ -92,11 +125,11 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
92
125
  ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
93
126
  const void* src, size_t compressedSize);
94
127
 
95
- /*! ZSTD_getFrameContentSize() : v1.3.0
128
+ /*! ZSTD_getFrameContentSize() : requires v1.3.0+
96
129
  * `src` should point to the start of a ZSTD encoded frame.
97
130
  * `srcSize` must be at least as large as the frame header.
98
131
  * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
99
- * @return : - decompressed size of the frame in `src`, if known
132
+ * @return : - decompressed size of `src` frame content, if known
100
133
  * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
101
134
  * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
102
135
  * note 1 : a 0 return value means the frame is valid but "empty".
@@ -106,7 +139,8 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
106
139
  * Optionally, application can rely on some implicit limit,
107
140
  * as ZSTD_decompress() only needs an upper bound of decompressed size.
108
141
  * (For example, data could be necessarily cut into blocks <= 16 KB).
109
- * note 3 : decompressed size is always present when compression is done with ZSTD_compress()
142
+ * note 3 : decompressed size is always present when compression is completed using single-pass functions,
143
+ * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
110
144
  * note 4 : decompressed size can be very large (64-bits value),
111
145
  * potentially larger than what local system can handle as a single memory segment.
112
146
  * In which case, it's necessary to use streaming mode to decompress data.
@@ -120,26 +154,32 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
120
154
 
121
155
  /*! ZSTD_getDecompressedSize() :
122
156
  * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
123
- * Both functions work the same way,
124
- * but ZSTD_getDecompressedSize() blends
125
- * "empty", "unknown" and "error" results in the same return value (0),
126
- * while ZSTD_getFrameContentSize() distinguishes them.
127
- *
128
- * 'src' is the start of a zstd compressed frame.
129
- * @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. */
157
+ * Both functions work the same way, but ZSTD_getDecompressedSize() blends
158
+ * "empty", "unknown" and "error" results to the same return value (0),
159
+ * while ZSTD_getFrameContentSize() gives them separate return values.
160
+ * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
130
161
  ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
131
162
 
163
+ /*! ZSTD_findFrameCompressedSize() :
164
+ * `src` should point to the start of a ZSTD frame or skippable frame.
165
+ * `srcSize` must be >= first frame size
166
+ * @return : the compressed size of the first frame starting at `src`,
167
+ * suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
168
+ * or an error code if input is invalid */
169
+ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
170
+
132
171
 
133
172
  /*====== Helper functions ======*/
134
173
  #define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
135
- ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
174
+ ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
136
175
  ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
137
176
  ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
177
+ ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */
138
178
  ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
139
179
 
140
180
 
141
181
  /***************************************
142
- * Explicit memory management
182
+ * Explicit context
143
183
  ***************************************/
144
184
  /*= Compression context
145
185
  * When compressing many times,
@@ -151,8 +191,10 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
151
191
  ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
152
192
 
153
193
  /*! ZSTD_compressCCtx() :
154
- * Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
155
- ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx,
194
+ * Same as ZSTD_compress(), using an explicit ZSTD_CCtx
195
+ * The function will compress at requested compression level,
196
+ * ignoring any other parameter */
197
+ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
156
198
  void* dst, size_t dstCapacity,
157
199
  const void* src, size_t srcSize,
158
200
  int compressionLevel);
@@ -168,82 +210,326 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
168
210
  ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
169
211
 
170
212
  /*! ZSTD_decompressDCtx() :
171
- * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
172
- ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
213
+ * Same as ZSTD_decompress(),
214
+ * requires an allocated ZSTD_DCtx.
215
+ * Compatible with sticky parameters.
216
+ */
217
+ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
173
218
  void* dst, size_t dstCapacity,
174
219
  const void* src, size_t srcSize);
175
220
 
176
221
 
177
- /**************************
178
- * Simple dictionary API
179
- ***************************/
180
- /*! ZSTD_compress_usingDict() :
181
- * Compression using a predefined Dictionary (see dictBuilder/zdict.h).
182
- * Note : This function loads the dictionary, resulting in significant startup delay.
183
- * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
184
- ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
185
- void* dst, size_t dstCapacity,
186
- const void* src, size_t srcSize,
187
- const void* dict,size_t dictSize,
188
- int compressionLevel);
222
+ /***************************************
223
+ * Advanced compression API
224
+ ***************************************/
189
225
 
190
- /*! ZSTD_decompress_usingDict() :
191
- * Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
192
- * Dictionary must be identical to the one used during compression.
193
- * Note : This function loads the dictionary, resulting in significant startup delay.
194
- * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
195
- ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
196
- void* dst, size_t dstCapacity,
197
- const void* src, size_t srcSize,
198
- const void* dict,size_t dictSize);
226
+ /* API design :
227
+ * Parameters are pushed one by one into an existing context,
228
+ * using ZSTD_CCtx_set*() functions.
229
+ * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
230
+ * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
231
+ * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()
232
+ *
233
+ * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
234
+ *
235
+ * This API supercedes all other "advanced" API entry points in the experimental section.
236
+ * In the future, we expect to remove from experimental API entry points which are redundant with this API.
237
+ */
199
238
 
200
239
 
201
- /**********************************
202
- * Bulk processing dictionary API
203
- *********************************/
204
- typedef struct ZSTD_CDict_s ZSTD_CDict;
240
+ /* Compression strategies, listed from fastest to strongest */
241
+ typedef enum { ZSTD_fast=1,
242
+ ZSTD_dfast=2,
243
+ ZSTD_greedy=3,
244
+ ZSTD_lazy=4,
245
+ ZSTD_lazy2=5,
246
+ ZSTD_btlazy2=6,
247
+ ZSTD_btopt=7,
248
+ ZSTD_btultra=8,
249
+ ZSTD_btultra2=9
250
+ /* note : new strategies _might_ be added in the future.
251
+ Only the order (from fast to strong) is guaranteed */
252
+ } ZSTD_strategy;
205
253
 
206
- /*! ZSTD_createCDict() :
207
- * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
208
- * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
209
- * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
210
- * `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict */
211
- ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
212
- int compressionLevel);
213
254
 
214
- /*! ZSTD_freeCDict() :
215
- * Function frees memory allocated by ZSTD_createCDict(). */
216
- ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
255
+ typedef enum {
217
256
 
218
- /*! ZSTD_compress_usingCDict() :
219
- * Compression using a digested Dictionary.
220
- * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
221
- * Note that compression level is decided during dictionary creation.
222
- * Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
223
- ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
224
- void* dst, size_t dstCapacity,
225
- const void* src, size_t srcSize,
226
- const ZSTD_CDict* cdict);
257
+ /* compression parameters
258
+ * Note: When compressing with a ZSTD_CDict these parameters are superseded
259
+ * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
260
+ * for more info (superseded-by-cdict). */
261
+ ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
262
+ * Default level is ZSTD_CLEVEL_DEFAULT==3.
263
+ * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
264
+ * Note 1 : it's possible to pass a negative compression level.
265
+ * Note 2 : setting a level sets all default values of other compression parameters */
266
+ ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
267
+ * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
268
+ * Special: value 0 means "use default windowLog".
269
+ * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
270
+ * requires explicitly allowing such window size at decompression stage if using streaming. */
271
+ ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
272
+ * Resulting memory usage is (1 << (hashLog+2)).
273
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
274
+ * Larger tables improve compression ratio of strategies <= dFast,
275
+ * and improve speed of strategies > dFast.
276
+ * Special: value 0 means "use default hashLog". */
277
+ ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2.
278
+ * Resulting memory usage is (1 << (chainLog+2)).
279
+ * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
280
+ * Larger tables result in better and slower compression.
281
+ * This parameter is useless when using "fast" strategy.
282
+ * It's still useful when using "dfast" strategy,
283
+ * in which case it defines a secondary probe table.
284
+ * Special: value 0 means "use default chainLog". */
285
+ ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
286
+ * More attempts result in better and slower compression.
287
+ * This parameter is useless when using "fast" and "dFast" strategies.
288
+ * Special: value 0 means "use default searchLog". */
289
+ ZSTD_c_minMatch=105, /* Minimum size of searched matches.
290
+ * Note that Zstandard can still find matches of smaller size,
291
+ * it just tweaks its search algorithm to look for this size and larger.
292
+ * Larger values increase compression and decompression speed, but decrease ratio.
293
+ * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
294
+ * Note that currently, for all strategies < btopt, effective minimum is 4.
295
+ * , for all strategies > fast, effective maximum is 6.
296
+ * Special: value 0 means "use default minMatchLength". */
297
+ ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
298
+ * For strategies btopt, btultra & btultra2:
299
+ * Length of Match considered "good enough" to stop search.
300
+ * Larger values make compression stronger, and slower.
301
+ * For strategy fast:
302
+ * Distance between match sampling.
303
+ * Larger values make compression faster, and weaker.
304
+ * Special: value 0 means "use default targetLength". */
305
+ ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition.
306
+ * The higher the value of selected strategy, the more complex it is,
307
+ * resulting in stronger and slower compression.
308
+ * Special: value 0 means "use default strategy". */
309
+
310
+ /* LDM mode parameters */
311
+ ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
312
+ * This parameter is designed to improve compression ratio
313
+ * for large inputs, by finding large matches at long distance.
314
+ * It increases memory usage and window size.
315
+ * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
316
+ * except when expressly set to a different value. */
317
+ ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
318
+ * Larger values increase memory usage and compression ratio,
319
+ * but decrease compression speed.
320
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
321
+ * default: windowlog - 7.
322
+ * Special: value 0 means "automatically determine hashlog". */
323
+ ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher.
324
+ * Larger/too small values usually decrease compression ratio.
325
+ * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
326
+ * Special: value 0 means "use default value" (default: 64). */
327
+ ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
328
+ * Larger values improve collision resolution but decrease compression speed.
329
+ * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
330
+ * Special: value 0 means "use default value" (default: 3). */
331
+ ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
332
+ * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
333
+ * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
334
+ * Larger values improve compression speed.
335
+ * Deviating far from default value will likely result in a compression ratio decrease.
336
+ * Special: value 0 means "automatically determine hashRateLog". */
227
337
 
338
+ /* frame parameters */
339
+ ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
340
+ * Content size must be known at the beginning of compression.
341
+ * This is automatically the case when using ZSTD_compress2(),
342
+ * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
343
+ ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
344
+ ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
228
345
 
229
- typedef struct ZSTD_DDict_s ZSTD_DDict;
346
+ /* multi-threading parameters */
347
+ /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
348
+ * They return an error otherwise. */
349
+ ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
350
+ * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :
351
+ * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
352
+ * while compression work is performed in parallel, within worker threads.
353
+ * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
354
+ * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
355
+ * More workers improve speed, but also increase memory usage.
356
+ * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
357
+ ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
358
+ * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
359
+ * 0 means default, which is dynamically determined based on compression parameters.
360
+ * Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
361
+ * The minimum size is automatically and transparently enforced */
362
+ ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
363
+ * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
364
+ * It helps preserve compression ratio, while each job is compressed in parallel.
365
+ * This value is enforced only when nbWorkers >= 1.
366
+ * Larger values increase compression ratio, but decrease speed.
367
+ * Possible values range from 0 to 9 :
368
+ * - 0 means "default" : value will be determined by the library, depending on strategy
369
+ * - 1 means "no overlap"
370
+ * - 9 means "full overlap", using a full window size.
371
+ * Each intermediate rank increases/decreases load size by a factor 2 :
372
+ * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default
373
+ * default value varies between 6 and 9, depending on strategy */
374
+
375
+ /* note : additional experimental parameters are also available
376
+ * within the experimental section of the API.
377
+ * At the time of this writing, they include :
378
+ * ZSTD_c_rsyncable
379
+ * ZSTD_c_format
380
+ * ZSTD_c_forceMaxWindow
381
+ * ZSTD_c_forceAttachDict
382
+ * ZSTD_c_literalCompressionMode
383
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
384
+ * note : never ever use experimentalParam? names directly;
385
+ * also, the enums values themselves are unstable and can still change.
386
+ */
387
+ ZSTD_c_experimentalParam1=500,
388
+ ZSTD_c_experimentalParam2=10,
389
+ ZSTD_c_experimentalParam3=1000,
390
+ ZSTD_c_experimentalParam4=1001,
391
+ ZSTD_c_experimentalParam5=1002,
392
+ } ZSTD_cParameter;
230
393
 
231
- /*! ZSTD_createDDict() :
232
- * Create a digested dictionary, ready to start decompression operation without startup delay.
233
- * dictBuffer can be released after DDict creation, as its content is copied inside DDict */
234
- ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
394
+ typedef struct {
395
+ size_t error;
396
+ int lowerBound;
397
+ int upperBound;
398
+ } ZSTD_bounds;
399
+
400
+ /*! ZSTD_cParam_getBounds() :
401
+ * All parameters must belong to an interval with lower and upper bounds,
402
+ * otherwise they will either trigger an error or be automatically clamped.
403
+ * @return : a structure, ZSTD_bounds, which contains
404
+ * - an error status field, which must be tested using ZSTD_isError()
405
+ * - lower and upper bounds, both inclusive
406
+ */
407
+ ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
235
408
 
236
- /*! ZSTD_freeDDict() :
237
- * Function frees memory allocated with ZSTD_createDDict() */
238
- ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
409
+ /*! ZSTD_CCtx_setParameter() :
410
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
411
+ * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
412
+ * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
413
+ * Setting a parameter is generally only possible during frame initialization (before starting compression).
414
+ * Exception : when using multi-threading mode (nbWorkers >= 1),
415
+ * the following parameters can be updated _during_ compression (within same frame):
416
+ * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
417
+ * new parameters will be active for next job only (after a flush()).
418
+ * @return : an error code (which can be tested using ZSTD_isError()).
419
+ */
420
+ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
239
421
 
240
- /*! ZSTD_decompress_usingDDict() :
241
- * Decompression using a digested Dictionary.
242
- * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
243
- ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
244
- void* dst, size_t dstCapacity,
245
- const void* src, size_t srcSize,
246
- const ZSTD_DDict* ddict);
422
+ /*! ZSTD_CCtx_setPledgedSrcSize() :
423
+ * Total input data size to be compressed as a single frame.
424
+ * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
425
+ * This value will also be controlled at end of frame, and trigger an error if not respected.
426
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
427
+ * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
428
+ * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
429
+ * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
430
+ * Note 2 : pledgedSrcSize is only valid once, for the next frame.
431
+ * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
432
+ * Note 3 : Whenever all input data is provided and consumed in a single round,
433
+ * for example with ZSTD_compress2(),
434
+ * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
435
+ * this value is automatically overridden by srcSize instead.
436
+ */
437
+ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
438
+
439
+ typedef enum {
440
+ ZSTD_reset_session_only = 1,
441
+ ZSTD_reset_parameters = 2,
442
+ ZSTD_reset_session_and_parameters = 3
443
+ } ZSTD_ResetDirective;
444
+
445
+ /*! ZSTD_CCtx_reset() :
446
+ * There are 2 different things that can be reset, independently or jointly :
447
+ * - The session : will stop compressing current frame, and make CCtx ready to start a new one.
448
+ * Useful after an error, or to interrupt any ongoing compression.
449
+ * Any internal data not yet flushed is cancelled.
450
+ * Compression parameters and dictionary remain unchanged.
451
+ * They will be used to compress next frame.
452
+ * Resetting session never fails.
453
+ * - The parameters : changes all parameters back to "default".
454
+ * This removes any reference to any dictionary too.
455
+ * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
456
+ * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
457
+ * - Both : similar to resetting the session, followed by resetting parameters.
458
+ */
459
+ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
460
+
461
+ /*! ZSTD_compress2() :
462
+ * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
463
+ * ZSTD_compress2() always starts a new frame.
464
+ * Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
465
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
466
+ * - The function is always blocking, returns when compression is completed.
467
+ * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
468
+ * @return : compressed size written into `dst` (<= `dstCapacity),
469
+ * or an error code if it fails (which can be tested using ZSTD_isError()).
470
+ */
471
+ ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
472
+ void* dst, size_t dstCapacity,
473
+ const void* src, size_t srcSize);
474
+
475
+
476
+ /***************************************
477
+ * Advanced decompression API
478
+ ***************************************/
479
+
480
+ /* The advanced API pushes parameters one by one into an existing DCtx context.
481
+ * Parameters are sticky, and remain valid for all following frames
482
+ * using the same DCtx context.
483
+ * It's possible to reset parameters to default values using ZSTD_DCtx_reset().
484
+ * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().
485
+ * Therefore, no new decompression function is necessary.
486
+ */
487
+
488
+ typedef enum {
489
+
490
+ ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
491
+ * the streaming API will refuse to allocate memory buffer
492
+ * in order to protect the host from unreasonable memory requirements.
493
+ * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
494
+ * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
495
+ * Special: value 0 means "use default maximum windowLog". */
496
+
497
+ /* note : additional experimental parameters are also available
498
+ * within the experimental section of the API.
499
+ * At the time of this writing, they include :
500
+ * ZSTD_c_format
501
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
502
+ * note : never ever use experimentalParam? names directly
503
+ */
504
+ ZSTD_d_experimentalParam1=1000
505
+
506
+ } ZSTD_dParameter;
507
+
508
+ /*! ZSTD_dParam_getBounds() :
509
+ * All parameters must belong to an interval with lower and upper bounds,
510
+ * otherwise they will either trigger an error or be automatically clamped.
511
+ * @return : a structure, ZSTD_bounds, which contains
512
+ * - an error status field, which must be tested using ZSTD_isError()
513
+ * - both lower and upper bounds, inclusive
514
+ */
515
+ ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
516
+
517
+ /*! ZSTD_DCtx_setParameter() :
518
+ * Set one compression parameter, selected by enum ZSTD_dParameter.
519
+ * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
520
+ * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
521
+ * Setting a parameter is only possible during frame initialization (before starting decompression).
522
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
523
+ */
524
+ ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
525
+
526
+ /*! ZSTD_DCtx_reset() :
527
+ * Return a DCtx to clean state.
528
+ * Session and parameters can be reset jointly or separately.
529
+ * Parameters can only be reset when no active frame is being decompressed.
530
+ * @return : 0, or an error code, which can be tested with ZSTD_isError()
531
+ */
532
+ ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
247
533
 
248
534
 
249
535
  /****************************
@@ -270,56 +556,136 @@ typedef struct ZSTD_outBuffer_s {
270
556
  * A ZSTD_CStream object is required to track streaming operation.
271
557
  * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
272
558
  * ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
273
- * It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
274
- * since it will play nicer with system's memory, by re-using already allocated memory.
275
- * Use one separate ZSTD_CStream per thread for parallel execution.
559
+ * It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
276
560
  *
277
- * Start a new compression by initializing ZSTD_CStream.
278
- * Use ZSTD_initCStream() to start a new compression operation.
279
- * Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section)
561
+ * For parallel execution, use one separate ZSTD_CStream per thread.
280
562
  *
281
- * Use ZSTD_compressStream() repetitively to consume input stream.
282
- * The function will automatically update both `pos` fields.
283
- * Note that it may not consume the entire input, in which case `pos < size`,
284
- * and it's up to the caller to present again remaining data.
285
- * @return : a size hint, preferred nb of bytes to use as input for next function call
286
- * or an error code, which can be tested using ZSTD_isError().
287
- * Note 1 : it's just a hint, to help latency a little, any other value will work fine.
288
- * Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
563
+ * note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
564
+ *
565
+ * Parameters are sticky : when starting a new compression on the same context,
566
+ * it will re-use the same sticky parameters as previous compression session.
567
+ * When in doubt, it's recommended to fully initialize the context before usage.
568
+ * Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
569
+ * ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
570
+ * set more specific parameters, the pledged source size, or load a dictionary.
289
571
  *
290
- * At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream().
291
- * `output->pos` will be updated.
292
- * Note that some content might still be left within internal buffer if `output->size` is too small.
293
- * @return : nb of bytes still present within internal buffer (0 if it's empty)
572
+ * Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
573
+ * consume input stream. The function will automatically update both `pos`
574
+ * fields within `input` and `output`.
575
+ * Note that the function may not consume the entire input, for example, because
576
+ * the output buffer is already full, in which case `input.pos < input.size`.
577
+ * The caller must check if input has been entirely consumed.
578
+ * If not, the caller must make some room to receive more compressed data,
579
+ * and then present again remaining input data.
580
+ * note: ZSTD_e_continue is guaranteed to make some forward progress when called,
581
+ * but doesn't guarantee maximal forward progress. This is especially relevant
582
+ * when compressing with multiple threads. The call won't block if it can
583
+ * consume some input, but if it can't it will wait for some, but not all,
584
+ * output to be flushed.
585
+ * @return : provides a minimum amount of data remaining to be flushed from internal buffers
586
+ * or an error code, which can be tested using ZSTD_isError().
587
+ *
588
+ * At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
589
+ * using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
590
+ * Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
591
+ * In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
592
+ * You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
593
+ * operation.
594
+ * note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
595
+ * block until the flush is complete or the output buffer is full.
596
+ * @return : 0 if internal buffers are entirely flushed,
597
+ * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
294
598
  * or an error code, which can be tested using ZSTD_isError().
295
599
  *
296
- * ZSTD_endStream() instructs to finish a frame.
600
+ * Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
297
601
  * It will perform a flush and write frame epilogue.
298
602
  * The epilogue is required for decoders to consider a frame completed.
299
- * ZSTD_endStream() may not be able to flush full data if `output->size` is too small.
300
- * In which case, call again ZSTD_endStream() to complete the flush.
603
+ * flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
604
+ * You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
605
+ * start a new frame.
606
+ * note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
607
+ * block until the flush is complete or the output buffer is full.
301
608
  * @return : 0 if frame fully completed and fully flushed,
302
- or >0 if some data is still present within internal buffer
303
- (value is minimum size estimation for remaining data to flush, but it could be more)
609
+ * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
304
610
  * or an error code, which can be tested using ZSTD_isError().
305
611
  *
306
612
  * *******************************************************************/
307
613
 
308
614
  typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
309
- /* Continue to distinguish them for compatibility with versions <= v1.2.0 */
615
+ /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
310
616
  /*===== ZSTD_CStream management functions =====*/
311
617
  ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
312
618
  ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
313
619
 
314
620
  /*===== Streaming compression functions =====*/
315
- ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
316
- ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
317
- ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
318
- ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
621
+ typedef enum {
622
+ ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
623
+ ZSTD_e_flush=1, /* flush any data provided so far,
624
+ * it creates (at least) one new block, that can be decoded immediately on reception;
625
+ * frame will continue: any future data can still reference previously compressed data, improving compression.
626
+ * note : multithreaded compression will block to flush as much output as possible. */
627
+ ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.
628
+ * note that frame is only closed after compressed data is fully flushed (return value == 0).
629
+ * After that point, any additional data starts a new frame.
630
+ * note : each frame is independent (does not reference any content from previous frame).
631
+ : note : multithreaded compression will block to flush as much output as possible. */
632
+ } ZSTD_EndDirective;
633
+
634
+ /*! ZSTD_compressStream2() :
635
+ * Behaves about the same as ZSTD_compressStream, with additional control on end directive.
636
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
637
+ * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
638
+ * - output->pos must be <= dstCapacity, input->pos must be <= srcSize
639
+ * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
640
+ * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
641
+ * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
642
+ * and then immediately returns, just indicating that there is some data remaining to be flushed.
643
+ * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
644
+ * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
645
+ * - @return provides a minimum amount of data remaining to be flushed from internal buffers
646
+ * or an error code, which can be tested using ZSTD_isError().
647
+ * if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
648
+ * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
649
+ * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
650
+ * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
651
+ * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
652
+ * Before starting a new compression job, or changing compression parameters,
653
+ * it is required to fully flush internal buffers.
654
+ */
655
+ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
656
+ ZSTD_outBuffer* output,
657
+ ZSTD_inBuffer* input,
658
+ ZSTD_EndDirective endOp);
319
659
 
320
660
  ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
321
661
  ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
322
662
 
663
+ /*******************************************************************************
664
+ * This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and
665
+ * ZSTD_compressStream2(). It is redundant, but is still fully supported.
666
+ * Advanced parameters and dictionary compression can only be used through the
667
+ * new API.
668
+ ******************************************************************************/
669
+
670
+ /**
671
+ * Equivalent to:
672
+ *
673
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
674
+ * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
675
+ * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
676
+ */
677
+ ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
678
+ /**
679
+ * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
680
+ * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
681
+ * the next read size (if non-zero and not an error). ZSTD_compressStream2()
682
+ * returns the number of bytes left to flush (if non-zero and not an error).
683
+ */
684
+ ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
685
+ /** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
686
+ ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
687
+ /** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
688
+ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
323
689
 
324
690
 
325
691
  /*-***************************************************************************
@@ -329,136 +695,457 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
329
695
  * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
330
696
  * ZSTD_DStream objects can be re-used multiple times.
331
697
  *
332
- * Use ZSTD_initDStream() to start a new decompression operation,
333
- * or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
334
- * @return : recommended first input size
698
+ * Use ZSTD_initDStream() to start a new decompression operation.
699
+ * @return : recommended first input size
700
+ * Alternatively, use advanced API to set specific properties.
335
701
  *
336
702
  * Use ZSTD_decompressStream() repetitively to consume your input.
337
703
  * The function will update both `pos` fields.
338
704
  * If `input.pos < input.size`, some input has not been consumed.
339
705
  * It's up to the caller to present again remaining data.
706
+ * The function tries to flush all data decoded immediately, respecting output buffer size.
340
707
  * If `output.pos < output.size`, decoder has flushed everything it could.
341
- * @return : 0 when a frame is completely decoded and fully flushed,
342
- * an error code, which can be tested using ZSTD_isError(),
343
- * any other value > 0, which means there is still some decoding to do to complete current frame.
344
- * The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
708
+ * But if `output.pos == output.size`, there might be some data left within internal buffers.,
709
+ * In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
710
+ * Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
711
+ * @return : 0 when a frame is completely decoded and fully flushed,
712
+ * or an error code, which can be tested using ZSTD_isError(),
713
+ * or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
714
+ * the return value is a suggested next input size (just a hint for better latency)
715
+ * that will never request more than the remaining frame size.
345
716
  * *******************************************************************************/
346
717
 
347
718
  typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
348
- /* Continue to distinguish them for compatibility with versions <= v1.2.0 */
719
+ /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
349
720
  /*===== ZSTD_DStream management functions =====*/
350
721
  ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
351
722
  ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
352
723
 
353
724
  /*===== Streaming decompression functions =====*/
725
+
726
+ /* This function is redundant with the advanced API and equivalent to:
727
+ *
728
+ * ZSTD_DCtx_reset(zds);
729
+ * ZSTD_DCtx_refDDict(zds, NULL);
730
+ */
354
731
  ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
732
+
355
733
  ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
356
734
 
357
735
  ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
358
736
  ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
359
737
 
360
- #endif /* ZSTD_H_235446 */
361
738
 
739
+ /**************************
740
+ * Simple dictionary API
741
+ ***************************/
742
+ /*! ZSTD_compress_usingDict() :
743
+ * Compression at an explicit compression level using a Dictionary.
744
+ * A dictionary can be any arbitrary data segment (also called a prefix),
745
+ * or a buffer with specified information (see dictBuilder/zdict.h).
746
+ * Note : This function loads the dictionary, resulting in significant startup delay.
747
+ * It's intended for a dictionary used only once.
748
+ * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
749
+ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
750
+ void* dst, size_t dstCapacity,
751
+ const void* src, size_t srcSize,
752
+ const void* dict,size_t dictSize,
753
+ int compressionLevel);
754
+
755
+ /*! ZSTD_decompress_usingDict() :
756
+ * Decompression using a known Dictionary.
757
+ * Dictionary must be identical to the one used during compression.
758
+ * Note : This function loads the dictionary, resulting in significant startup delay.
759
+ * It's intended for a dictionary used only once.
760
+ * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
761
+ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
762
+ void* dst, size_t dstCapacity,
763
+ const void* src, size_t srcSize,
764
+ const void* dict,size_t dictSize);
362
765
 
363
766
 
364
- /****************************************************************************************
365
- * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
366
- * The definitions in this section are considered experimental.
367
- * They should never be used with a dynamic library, as prototypes may change in the future.
368
- * They are provided for advanced scenarios.
369
- * Use them only in association with static linking.
370
- * ***************************************************************************************/
767
+ /***********************************
768
+ * Bulk processing dictionary API
769
+ **********************************/
770
+ typedef struct ZSTD_CDict_s ZSTD_CDict;
371
771
 
372
- #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
373
- #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
772
+ /*! ZSTD_createCDict() :
773
+ * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
774
+ * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
775
+ * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
776
+ * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
777
+ * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
778
+ * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
779
+ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
780
+ int compressionLevel);
374
781
 
375
- /* --- Constants ---*/
376
- #define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
377
- #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
378
- #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
379
-
380
- #define ZSTD_WINDOWLOG_MAX_32 30
381
- #define ZSTD_WINDOWLOG_MAX_64 31
382
- #define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
383
- #define ZSTD_WINDOWLOG_MIN 10
384
- #define ZSTD_HASHLOG_MAX MIN(ZSTD_WINDOWLOG_MAX, 30)
385
- #define ZSTD_HASHLOG_MIN 6
386
- #define ZSTD_CHAINLOG_MAX MIN(ZSTD_WINDOWLOG_MAX+1, 30)
387
- #define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
388
- #define ZSTD_HASHLOG3_MAX 17
389
- #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
390
- #define ZSTD_SEARCHLOG_MIN 1
391
- #define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
392
- #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
393
- #define ZSTD_TARGETLENGTH_MIN 4 /* only useful for btopt */
394
- #define ZSTD_TARGETLENGTH_MAX 999 /* only useful for btopt */
395
- #define ZSTD_LDM_MINMATCH_MIN 4
396
- #define ZSTD_LDM_MINMATCH_MAX 4096
397
- #define ZSTD_LDM_BUCKETSIZELOG_MAX 8
398
-
399
- #define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size to know frame header size */
400
- #define ZSTD_FRAMEHEADERSIZE_MIN 6
401
- #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
402
- static const size_t ZSTD_frameHeaderSize_prefix = ZSTD_FRAMEHEADERSIZE_PREFIX;
403
- static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
404
- static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
405
- static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
782
+ /*! ZSTD_freeCDict() :
783
+ * Function frees memory allocated by ZSTD_createCDict(). */
784
+ ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
406
785
 
786
+ /*! ZSTD_compress_usingCDict() :
787
+ * Compression using a digested Dictionary.
788
+ * Recommended when same dictionary is used multiple times.
789
+ * Note : compression level is _decided at dictionary creation time_,
790
+ * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
791
+ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
792
+ void* dst, size_t dstCapacity,
793
+ const void* src, size_t srcSize,
794
+ const ZSTD_CDict* cdict);
407
795
 
408
- /*--- Advanced types ---*/
409
- typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
410
- ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */
411
796
 
412
- typedef struct {
413
- unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
414
- unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
415
- unsigned hashLog; /**< dispatch table : larger == faster, more memory */
416
- unsigned searchLog; /**< nb of searches : larger == more compression, slower */
417
- unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */
418
- unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
419
- ZSTD_strategy strategy;
420
- } ZSTD_compressionParameters;
797
+ typedef struct ZSTD_DDict_s ZSTD_DDict;
421
798
 
422
- typedef struct {
423
- unsigned contentSizeFlag; /**< 1: content size will be in frame header (when known) */
424
- unsigned checksumFlag; /**< 1: generate a 32-bits checksum at end of frame, for error detection */
425
- unsigned noDictIDFlag; /**< 1: no dictID will be saved into frame header (if dictionary compression) */
426
- } ZSTD_frameParameters;
799
+ /*! ZSTD_createDDict() :
800
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
801
+ * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
802
+ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
427
803
 
428
- typedef struct {
429
- ZSTD_compressionParameters cParams;
430
- ZSTD_frameParameters fParams;
431
- } ZSTD_parameters;
804
+ /*! ZSTD_freeDDict() :
805
+ * Function frees memory allocated with ZSTD_createDDict() */
806
+ ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
432
807
 
433
- typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
808
+ /*! ZSTD_decompress_usingDDict() :
809
+ * Decompression using a digested Dictionary.
810
+ * Recommended when same dictionary is used multiple times. */
811
+ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
812
+ void* dst, size_t dstCapacity,
813
+ const void* src, size_t srcSize,
814
+ const ZSTD_DDict* ddict);
434
815
 
435
- /*--- Custom memory allocation functions ---*/
436
- typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
437
- typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
438
- typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
439
- /* use this constant to defer to stdlib's functions */
440
- static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };
441
816
 
817
+ /********************************
818
+ * Dictionary helper functions
819
+ *******************************/
442
820
 
443
- /***************************************
444
- * Frame size functions
445
- ***************************************/
821
+ /*! ZSTD_getDictID_fromDict() :
822
+ * Provides the dictID stored within dictionary.
823
+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
824
+ * It can still be loaded, but as a content-only dictionary. */
825
+ ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
446
826
 
447
- /*! ZSTD_findFrameCompressedSize() :
448
- * `src` should point to the start of a ZSTD encoded frame or skippable frame
449
- * `srcSize` must be >= first frame size
450
- * @return : the compressed size of the first frame starting at `src`,
451
- * suitable to pass to `ZSTD_decompress` or similar,
452
- * or an error code if input is invalid */
453
- ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
827
+ /*! ZSTD_getDictID_fromDDict() :
828
+ * Provides the dictID of the dictionary loaded into `ddict`.
829
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
830
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
831
+ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
454
832
 
455
- /*! ZSTD_findDecompressedSize() :
456
- * `src` should point the start of a series of ZSTD encoded and/or skippable frames
457
- * `srcSize` must be the _exact_ size of this series
458
- * (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`)
459
- * @return : - decompressed size of all data in all successive frames
460
- * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
461
- * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
833
+ /*! ZSTD_getDictID_fromFrame() :
834
+ * Provides the dictID required to decompressed the frame stored within `src`.
835
+ * If @return == 0, the dictID could not be decoded.
836
+ * This could for one of the following reasons :
837
+ * - The frame does not require a dictionary to be decoded (most common case).
838
+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
839
+ * Note : this use case also happens when using a non-conformant dictionary.
840
+ * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
841
+ * - This is not a Zstandard frame.
842
+ * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
843
+ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
844
+
845
+
846
+ /*******************************************************************************
847
+ * Advanced dictionary and prefix API
848
+ *
849
+ * This API allows dictionaries to be used with ZSTD_compress2(),
850
+ * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and
851
+ * only reset with the context is reset with ZSTD_reset_parameters or
852
+ * ZSTD_reset_session_and_parameters. Prefixes are single-use.
853
+ ******************************************************************************/
854
+
855
+
856
+ /*! ZSTD_CCtx_loadDictionary() :
857
+ * Create an internal CDict from `dict` buffer.
858
+ * Decompression will have to use same dictionary.
859
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
860
+ * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
861
+ * meaning "return to no-dictionary mode".
862
+ * Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
863
+ * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
864
+ * Note 2 : Loading a dictionary involves building tables.
865
+ * It's also a CPU consuming operation, with non-negligible impact on latency.
866
+ * Tables are dependent on compression parameters, and for this reason,
867
+ * compression parameters can no longer be changed after loading a dictionary.
868
+ * Note 3 :`dict` content will be copied internally.
869
+ * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
870
+ * In such a case, dictionary buffer must outlive its users.
871
+ * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
872
+ * to precisely select how dictionary content must be interpreted. */
873
+ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
874
+
875
+ /*! ZSTD_CCtx_refCDict() :
876
+ * Reference a prepared dictionary, to be used for all next compressed frames.
877
+ * Note that compression parameters are enforced from within CDict,
878
+ * and supersede any compression parameter previously set within CCtx.
879
+ * The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
880
+ * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
881
+ * The dictionary will remain valid for future compressed frames using same CCtx.
882
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
883
+ * Special : Referencing a NULL CDict means "return to no-dictionary mode".
884
+ * Note 1 : Currently, only one dictionary can be managed.
885
+ * Referencing a new dictionary effectively "discards" any previous one.
886
+ * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
887
+ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
888
+
889
+ /*! ZSTD_CCtx_refPrefix() :
890
+ * Reference a prefix (single-usage dictionary) for next compressed frame.
891
+ * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
892
+ * Decompression will need same prefix to properly regenerate data.
893
+ * Compressing with a prefix is similar in outcome as performing a diff and compressing it,
894
+ * but performs much faster, especially during decompression (compression speed is tunable with compression level).
895
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
896
+ * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
897
+ * Note 1 : Prefix buffer is referenced. It **must** outlive compression.
898
+ * Its content must remain unmodified during compression.
899
+ * Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
900
+ * ensure that the window size is large enough to contain the entire source.
901
+ * See ZSTD_c_windowLog.
902
+ * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
903
+ * It's a CPU consuming operation, with non-negligible impact on latency.
904
+ * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
905
+ * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
906
+ * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
907
+ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
908
+ const void* prefix, size_t prefixSize);
909
+
910
+ /*! ZSTD_DCtx_loadDictionary() :
911
+ * Create an internal DDict from dict buffer,
912
+ * to be used to decompress next frames.
913
+ * The dictionary remains valid for all future frames, until explicitly invalidated.
914
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
915
+ * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
916
+ * meaning "return to no-dictionary mode".
917
+ * Note 1 : Loading a dictionary involves building tables,
918
+ * which has a non-negligible impact on CPU usage and latency.
919
+ * It's recommended to "load once, use many times", to amortize the cost
920
+ * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
921
+ * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
922
+ * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
923
+ * how dictionary content is loaded and interpreted.
924
+ */
925
+ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
926
+
927
+ /*! ZSTD_DCtx_refDDict() :
928
+ * Reference a prepared dictionary, to be used to decompress next frames.
929
+ * The dictionary remains active for decompression of future frames using same DCtx.
930
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
931
+ * Note 1 : Currently, only one dictionary can be managed.
932
+ * Referencing a new dictionary effectively "discards" any previous one.
933
+ * Special: referencing a NULL DDict means "return to no-dictionary mode".
934
+ * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
935
+ */
936
+ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
937
+
938
+ /*! ZSTD_DCtx_refPrefix() :
939
+ * Reference a prefix (single-usage dictionary) to decompress next frame.
940
+ * This is the reverse operation of ZSTD_CCtx_refPrefix(),
941
+ * and must use the same prefix as the one used during compression.
942
+ * Prefix is **only used once**. Reference is discarded at end of frame.
943
+ * End of frame is reached when ZSTD_decompressStream() returns 0.
944
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
945
+ * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
946
+ * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
947
+ * Prefix buffer must remain unmodified up to the end of frame,
948
+ * reached when ZSTD_decompressStream() returns 0.
949
+ * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
950
+ * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
951
+ * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
952
+ * A full dictionary is more costly, as it requires building tables.
953
+ */
954
+ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
955
+ const void* prefix, size_t prefixSize);
956
+
957
+ /* === Memory management === */
958
+
959
+ /*! ZSTD_sizeof_*() :
960
+ * These functions give the _current_ memory usage of selected object.
961
+ * Note that object memory usage can evolve (increase or decrease) over time. */
962
+ ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
963
+ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
964
+ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
965
+ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
966
+ ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
967
+ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
968
+
969
+ #endif /* ZSTD_H_235446 */
970
+
971
+
972
+ /****************************************************************************************
973
+ * ADVANCED AND EXPERIMENTAL FUNCTIONS
974
+ ****************************************************************************************
975
+ * The definitions in the following section are considered experimental.
976
+ * They are provided for advanced scenarios.
977
+ * They should never be used with a dynamic library, as prototypes may change in the future.
978
+ * Use them only in association with static linking.
979
+ * ***************************************************************************************/
980
+
981
+ #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
982
+ #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
983
+
984
+ /****************************************************************************************
985
+ * experimental API (static linking only)
986
+ ****************************************************************************************
987
+ * The following symbols and constants
988
+ * are not planned to join "stable API" status in the near future.
989
+ * They can still change in future versions.
990
+ * Some of them are planned to remain in the static_only section indefinitely.
991
+ * Some of them might be removed in the future (especially when redundant with existing stable functions)
992
+ * ***************************************************************************************/
993
+
994
+ #define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */
995
+ #define ZSTD_FRAMEHEADERSIZE_MIN 6
996
+ #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
997
+ #define ZSTD_SKIPPABLEHEADERSIZE 8
998
+
999
+ /* compression parameter bounds */
1000
+ #define ZSTD_WINDOWLOG_MAX_32 30
1001
+ #define ZSTD_WINDOWLOG_MAX_64 31
1002
+ #define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
1003
+ #define ZSTD_WINDOWLOG_MIN 10
1004
+ #define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
1005
+ #define ZSTD_HASHLOG_MIN 6
1006
+ #define ZSTD_CHAINLOG_MAX_32 29
1007
+ #define ZSTD_CHAINLOG_MAX_64 30
1008
+ #define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
1009
+ #define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
1010
+ #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
1011
+ #define ZSTD_SEARCHLOG_MIN 1
1012
+ #define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
1013
+ #define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */
1014
+ #define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
1015
+ #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
1016
+ #define ZSTD_STRATEGY_MIN ZSTD_fast
1017
+ #define ZSTD_STRATEGY_MAX ZSTD_btultra2
1018
+
1019
+
1020
+ #define ZSTD_OVERLAPLOG_MIN 0
1021
+ #define ZSTD_OVERLAPLOG_MAX 9
1022
+
1023
+ #define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame
1024
+ * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,
1025
+ * to preserve host's memory from unreasonable requirements.
1026
+ * This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
1027
+ * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */
1028
+
1029
+
1030
+ /* LDM parameter bounds */
1031
+ #define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN
1032
+ #define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX
1033
+ #define ZSTD_LDM_MINMATCH_MIN 4
1034
+ #define ZSTD_LDM_MINMATCH_MAX 4096
1035
+ #define ZSTD_LDM_BUCKETSIZELOG_MIN 1
1036
+ #define ZSTD_LDM_BUCKETSIZELOG_MAX 8
1037
+ #define ZSTD_LDM_HASHRATELOG_MIN 0
1038
+ #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
1039
+
1040
+ /* internal */
1041
+ #define ZSTD_HASHLOG3_MAX 17
1042
+
1043
+
1044
+ /* --- Advanced types --- */
1045
+
1046
+ typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
1047
+
1048
+ typedef struct {
1049
+ unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
1050
+ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
1051
+ unsigned hashLog; /**< dispatch table : larger == faster, more memory */
1052
+ unsigned searchLog; /**< nb of searches : larger == more compression, slower */
1053
+ unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */
1054
+ unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
1055
+ ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */
1056
+ } ZSTD_compressionParameters;
1057
+
1058
+ typedef struct {
1059
+ int contentSizeFlag; /**< 1: content size will be in frame header (when known) */
1060
+ int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */
1061
+ int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */
1062
+ } ZSTD_frameParameters;
1063
+
1064
+ typedef struct {
1065
+ ZSTD_compressionParameters cParams;
1066
+ ZSTD_frameParameters fParams;
1067
+ } ZSTD_parameters;
1068
+
1069
+ typedef enum {
1070
+ ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
1071
+ ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
1072
+ ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
1073
+ } ZSTD_dictContentType_e;
1074
+
1075
+ typedef enum {
1076
+ ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
1077
+ ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
1078
+ } ZSTD_dictLoadMethod_e;
1079
+
1080
+ typedef enum {
1081
+ /* Opened question : should we have a format ZSTD_f_auto ?
1082
+ * Today, it would mean exactly the same as ZSTD_f_zstd1.
1083
+ * But, in the future, should several formats become supported,
1084
+ * on the compression side, it would mean "default format".
1085
+ * On the decompression side, it would mean "automatic format detection",
1086
+ * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
1087
+ * Since meaning is a little different, another option could be to define different enums for compression and decompression.
1088
+ * This question could be kept for later, when there are actually multiple formats to support,
1089
+ * but there is also the question of pinning enum values, and pinning value `0` is especially important */
1090
+ ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
1091
+ ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number.
1092
+ * Useful to save 4 bytes per generated frame.
1093
+ * Decoder cannot recognise automatically this format, requiring this instruction. */
1094
+ } ZSTD_format_e;
1095
+
1096
+ typedef enum {
1097
+ /* Note: this enum and the behavior it controls are effectively internal
1098
+ * implementation details of the compressor. They are expected to continue
1099
+ * to evolve and should be considered only in the context of extremely
1100
+ * advanced performance tuning.
1101
+ *
1102
+ * Zstd currently supports the use of a CDict in two ways:
1103
+ *
1104
+ * - The contents of the CDict can be copied into the working context. This
1105
+ * means that the compression can search both the dictionary and input
1106
+ * while operating on a single set of internal tables. This makes
1107
+ * the compression faster per-byte of input. However, the initial copy of
1108
+ * the CDict's tables incurs a fixed cost at the beginning of the
1109
+ * compression. For small compressions (< 8 KB), that copy can dominate
1110
+ * the cost of the compression.
1111
+ *
1112
+ * - The CDict's tables can be used in-place. In this model, compression is
1113
+ * slower per input byte, because the compressor has to search two sets of
1114
+ * tables. However, this model incurs no start-up cost (as long as the
1115
+ * working context's tables can be reused). For small inputs, this can be
1116
+ * faster than copying the CDict's tables.
1117
+ *
1118
+ * Zstd has a simple internal heuristic that selects which strategy to use
1119
+ * at the beginning of a compression. However, if experimentation shows that
1120
+ * Zstd is making poor choices, it is possible to override that choice with
1121
+ * this enum.
1122
+ */
1123
+ ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
1124
+ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
1125
+ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
1126
+ } ZSTD_dictAttachPref_e;
1127
+
1128
+ typedef enum {
1129
+ ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level.
1130
+ * Negative compression levels will be uncompressed, and positive compression
1131
+ * levels will be compressed. */
1132
+ ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
1133
+ * emitted if Huffman compression is not profitable. */
1134
+ ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */
1135
+ } ZSTD_literalCompressionMode_e;
1136
+
1137
+
1138
+ /***************************************
1139
+ * Frame size functions
1140
+ ***************************************/
1141
+
1142
+ /*! ZSTD_findDecompressedSize() :
1143
+ * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
1144
+ * `srcSize` must be the _exact_ size of this series
1145
+ * (i.e. there should be a frame boundary at `src + srcSize`)
1146
+ * @return : - decompressed size of all data in all successive frames
1147
+ * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
1148
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
462
1149
  *
463
1150
  * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
464
1151
  * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
@@ -475,27 +1162,32 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
475
1162
  * however it does mean that all frame data must be present and valid. */
476
1163
  ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
477
1164
 
1165
+ /** ZSTD_decompressBound() :
1166
+ * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
1167
+ * `srcSize` must be the _exact_ size of this series
1168
+ * (i.e. there should be a frame boundary at `src + srcSize`)
1169
+ * @return : - upper-bound for the decompressed size of all data in all successive frames
1170
+ * - if an error occured: ZSTD_CONTENTSIZE_ERROR
1171
+ *
1172
+ * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
1173
+ * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
1174
+ * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
1175
+ * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
1176
+ * upper-bound = # blocks * min(128 KB, Window_Size)
1177
+ */
1178
+ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
1179
+
478
1180
  /*! ZSTD_frameHeaderSize() :
479
- * `src` should point to the start of a ZSTD frame
480
- * `srcSize` must be >= ZSTD_frameHeaderSize_prefix.
481
- * @return : size of the Frame Header */
1181
+ * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
1182
+ * @return : size of the Frame Header,
1183
+ * or an error code (if srcSize is too small) */
482
1184
  ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
483
1185
 
484
1186
 
485
1187
  /***************************************
486
- * Context memory usage
1188
+ * Memory management
487
1189
  ***************************************/
488
1190
 
489
- /*! ZSTD_sizeof_*() :
490
- * These functions give the current memory usage of selected object.
491
- * Object memory usage can evolve when re-used multiple times. */
492
- ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
493
- ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
494
- ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
495
- ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
496
- ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
497
- ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
498
-
499
1191
  /*! ZSTD_estimate*() :
500
1192
  * These functions make it possible to estimate memory usage
501
1193
  * of a future {D,C}Ctx, before its creation.
@@ -503,8 +1195,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
503
1195
  * It will also consider src size to be arbitrarily "large", which is worst case.
504
1196
  * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
505
1197
  * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
506
- * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1.
507
- * Note : CCtx estimation is only correct for single-threaded compression */
1198
+ * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
1199
+ * Note : CCtx size estimation is only correct for single-threaded compression. */
508
1200
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
509
1201
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
510
1202
  ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
@@ -515,8 +1207,8 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
515
1207
  * It will also consider src size to be arbitrarily "large", which is worst case.
516
1208
  * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
517
1209
  * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
518
- * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1.
519
- * Note : CStream estimation is only correct for single-threaded compression.
1210
+ * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
1211
+ * Note : CStream size estimation is only correct for single-threaded compression.
520
1212
  * ZSTD_DStream memory budget depends on window Size.
521
1213
  * This information can be passed manually, using ZSTD_estimateDStreamSize,
522
1214
  * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
@@ -529,121 +1221,280 @@ ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_para
529
1221
  ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
530
1222
  ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
531
1223
 
532
- typedef enum {
533
- ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
534
- ZSTD_dlm_byRef, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
535
- } ZSTD_dictLoadMethod_e;
536
-
537
1224
  /*! ZSTD_estimate?DictSize() :
538
1225
  * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
539
- * ZSTD_estimateCStreamSize_advanced_usingCParams() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
540
- * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller
1226
+ * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
1227
+ * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
541
1228
  */
542
1229
  ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
543
1230
  ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
544
1231
  ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
545
1232
 
546
-
547
- /***************************************
548
- * Advanced compression functions
549
- ***************************************/
550
- /*! ZSTD_createCCtx_advanced() :
551
- * Create a ZSTD compression context using external alloc and free functions */
552
- ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
553
-
554
- /*! ZSTD_initStaticCCtx() : initialize a fixed-size zstd compression context
555
- * workspace: The memory area to emplace the context into.
556
- * Provided pointer must 8-bytes aligned.
557
- * It must outlive context usage.
558
- * workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
559
- * to determine how large workspace must be to support scenario.
560
- * @return : pointer to ZSTD_CCtx* (same address as workspace, but different type),
561
- * or NULL if error (typically size too small)
562
- * Note : zstd will never resize nor malloc() when using a static cctx.
563
- * If it needs more memory than available, it will simply error out.
1233
+ /*! ZSTD_initStatic*() :
1234
+ * Initialize an object using a pre-allocated fixed-size buffer.
1235
+ * workspace: The memory area to emplace the object into.
1236
+ * Provided pointer *must be 8-bytes aligned*.
1237
+ * Buffer must outlive object.
1238
+ * workspaceSize: Use ZSTD_estimate*Size() to determine
1239
+ * how large workspace must be to support target scenario.
1240
+ * @return : pointer to object (same address as workspace, just different type),
1241
+ * or NULL if error (size too small, incorrect alignment, etc.)
1242
+ * Note : zstd will never resize nor malloc() when using a static buffer.
1243
+ * If the object requires more memory than available,
1244
+ * zstd will just error out (typically ZSTD_error_memory_allocation).
564
1245
  * Note 2 : there is no corresponding "free" function.
565
- * Since workspace was allocated externally, it must be freed externally too.
566
- * Limitation 1 : currently not compatible with internal CDict creation, such as
567
- * ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
568
- * Limitation 2 : currently not compatible with multi-threading
1246
+ * Since workspace is allocated externally, it must be freed externally too.
1247
+ * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level
1248
+ * into its associated cParams.
1249
+ * Limitation 1 : currently not compatible with internal dictionary creation, triggered by
1250
+ * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict().
1251
+ * Limitation 2 : static cctx currently not compatible with multi-threading.
1252
+ * Limitation 3 : static dctx is incompatible with legacy support.
569
1253
  */
570
- ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
1254
+ ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
1255
+ ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
571
1256
 
1257
+ ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
1258
+ ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
1259
+
1260
+ ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict(
1261
+ void* workspace, size_t workspaceSize,
1262
+ const void* dict, size_t dictSize,
1263
+ ZSTD_dictLoadMethod_e dictLoadMethod,
1264
+ ZSTD_dictContentType_e dictContentType,
1265
+ ZSTD_compressionParameters cParams);
1266
+
1267
+ ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict(
1268
+ void* workspace, size_t workspaceSize,
1269
+ const void* dict, size_t dictSize,
1270
+ ZSTD_dictLoadMethod_e dictLoadMethod,
1271
+ ZSTD_dictContentType_e dictContentType);
572
1272
 
573
- /*! ZSTD_createCDict_byReference() :
574
- * Create a digested dictionary for compression
575
- * Dictionary content is simply referenced, and therefore stays in dictBuffer.
576
- * It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
577
- ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
578
1273
 
579
- typedef enum { ZSTD_dm_auto=0, /* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
580
- ZSTD_dm_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
581
- ZSTD_dm_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */
582
- } ZSTD_dictMode_e;
583
- /*! ZSTD_createCDict_advanced() :
584
- * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
1274
+ /*! Custom memory allocation :
1275
+ * These prototypes make it possible to pass your own allocation/free functions.
1276
+ * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
1277
+ * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
1278
+ */
1279
+ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
1280
+ typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
1281
+ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
1282
+ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
1283
+
1284
+ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
1285
+ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
1286
+ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
1287
+ ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
1288
+
585
1289
  ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
586
1290
  ZSTD_dictLoadMethod_e dictLoadMethod,
587
- ZSTD_dictMode_e dictMode,
1291
+ ZSTD_dictContentType_e dictContentType,
588
1292
  ZSTD_compressionParameters cParams,
589
1293
  ZSTD_customMem customMem);
590
1294
 
591
- /*! ZSTD_initStaticCDict() :
592
- * Generate a digested dictionary in provided memory area.
593
- * workspace: The memory area to emplace the dictionary into.
594
- * Provided pointer must 8-bytes aligned.
595
- * It must outlive dictionary usage.
596
- * workspaceSize: Use ZSTD_estimateCDictSize()
597
- * to determine how large workspace must be.
598
- * cParams : use ZSTD_getCParams() to transform a compression level
599
- * into its relevants cParams.
600
- * @return : pointer to ZSTD_CDict* (same address as workspace, but different type),
601
- * or NULL if error (typically, size too small).
602
- * Note : there is no corresponding "free" function.
603
- * Since workspace was allocated externally, it must be freed externally.
604
- */
605
- ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict(
606
- void* workspace, size_t workspaceSize,
607
- const void* dict, size_t dictSize,
608
- ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode,
609
- ZSTD_compressionParameters cParams);
1295
+ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
1296
+ ZSTD_dictLoadMethod_e dictLoadMethod,
1297
+ ZSTD_dictContentType_e dictContentType,
1298
+ ZSTD_customMem customMem);
1299
+
1300
+
1301
+
1302
+ /***************************************
1303
+ * Advanced compression functions
1304
+ ***************************************/
1305
+
1306
+ /*! ZSTD_createCDict_byReference() :
1307
+ * Create a digested dictionary for compression
1308
+ * Dictionary content is just referenced, not duplicated.
1309
+ * As a consequence, `dictBuffer` **must** outlive CDict,
1310
+ * and its content must remain unmodified throughout the lifetime of CDict. */
1311
+ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
610
1312
 
611
1313
  /*! ZSTD_getCParams() :
612
- * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
613
- * `estimatedSrcSize` value is optional, select 0 if not known */
1314
+ * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
1315
+ * `estimatedSrcSize` value is optional, select 0 if not known */
614
1316
  ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
615
1317
 
616
1318
  /*! ZSTD_getParams() :
617
- * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
618
- * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
1319
+ * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
1320
+ * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
619
1321
  ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
620
1322
 
621
1323
  /*! ZSTD_checkCParams() :
622
- * Ensure param values remain within authorized range */
1324
+ * Ensure param values remain within authorized range.
1325
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
623
1326
  ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
624
1327
 
625
1328
  /*! ZSTD_adjustCParams() :
626
1329
  * optimize params for a given `srcSize` and `dictSize`.
627
- * both values are optional, select `0` if unknown. */
1330
+ * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
1331
+ * `dictSize` must be `0` when there is no dictionary.
1332
+ * cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
1333
+ * This function never fails (wide contract) */
628
1334
  ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
629
1335
 
630
1336
  /*! ZSTD_compress_advanced() :
631
- * Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter */
632
- ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
633
- void* dst, size_t dstCapacity,
634
- const void* src, size_t srcSize,
635
- const void* dict,size_t dictSize,
636
- ZSTD_parameters params);
1337
+ * Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */
1338
+ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
1339
+ void* dst, size_t dstCapacity,
1340
+ const void* src, size_t srcSize,
1341
+ const void* dict,size_t dictSize,
1342
+ ZSTD_parameters params);
637
1343
 
638
1344
  /*! ZSTD_compress_usingCDict_advanced() :
639
- * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
1345
+ * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
640
1346
  ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
641
- void* dst, size_t dstCapacity,
642
- const void* src, size_t srcSize,
643
- const ZSTD_CDict* cdict, ZSTD_frameParameters fParams);
1347
+ void* dst, size_t dstCapacity,
1348
+ const void* src, size_t srcSize,
1349
+ const ZSTD_CDict* cdict,
1350
+ ZSTD_frameParameters fParams);
1351
+
1352
+
1353
+ /*! ZSTD_CCtx_loadDictionary_byReference() :
1354
+ * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
1355
+ * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
1356
+ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
1357
+
1358
+ /*! ZSTD_CCtx_loadDictionary_advanced() :
1359
+ * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
1360
+ * how to load the dictionary (by copy ? by reference ?)
1361
+ * and how to interpret it (automatic ? force raw mode ? full mode only ?) */
1362
+ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
1363
+
1364
+ /*! ZSTD_CCtx_refPrefix_advanced() :
1365
+ * Same as ZSTD_CCtx_refPrefix(), but gives finer control over
1366
+ * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
1367
+ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
1368
+
1369
+ /* === experimental parameters === */
1370
+ /* these parameters can be used with ZSTD_setParameter()
1371
+ * they are not guaranteed to remain supported in the future */
1372
+
1373
+ /* Enables rsyncable mode,
1374
+ * which makes compressed files more rsync friendly
1375
+ * by adding periodic synchronization points to the compressed data.
1376
+ * The target average block size is ZSTD_c_jobSize / 2.
1377
+ * It's possible to modify the job size to increase or decrease
1378
+ * the granularity of the synchronization point.
1379
+ * Once the jobSize is smaller than the window size,
1380
+ * it will result in compression ratio degradation.
1381
+ * NOTE 1: rsyncable mode only works when multithreading is enabled.
1382
+ * NOTE 2: rsyncable performs poorly in combination with long range mode,
1383
+ * since it will decrease the effectiveness of synchronization points,
1384
+ * though mileage may vary.
1385
+ * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.
1386
+ * If the selected compression level is already running significantly slower,
1387
+ * the overall speed won't be significantly impacted.
1388
+ */
1389
+ #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1
1390
+
1391
+ /* Select a compression format.
1392
+ * The value must be of type ZSTD_format_e.
1393
+ * See ZSTD_format_e enum definition for details */
1394
+ #define ZSTD_c_format ZSTD_c_experimentalParam2
1395
+
1396
+ /* Force back-reference distances to remain < windowSize,
1397
+ * even when referencing into Dictionary content (default:0) */
1398
+ #define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3
1399
+
1400
+ /* Controls whether the contents of a CDict
1401
+ * are used in place, or copied into the working context.
1402
+ * Accepts values from the ZSTD_dictAttachPref_e enum.
1403
+ * See the comments on that enum for an explanation of the feature. */
1404
+ #define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
1405
+
1406
+ /* Controls how the literals are compressed (default is auto).
1407
+ * The value must be of type ZSTD_literalCompressionMode_e.
1408
+ * See ZSTD_literalCompressionMode_t enum definition for details.
1409
+ */
1410
+ #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
1411
+
1412
+ /*! ZSTD_CCtx_getParameter() :
1413
+ * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
1414
+ * and store it into int* value.
1415
+ * @return : 0, or an error code (which can be tested with ZSTD_isError()).
1416
+ */
1417
+ ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
1418
+
644
1419
 
1420
+ /*! ZSTD_CCtx_params :
1421
+ * Quick howto :
1422
+ * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
1423
+ * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
1424
+ * an existing ZSTD_CCtx_params structure.
1425
+ * This is similar to
1426
+ * ZSTD_CCtx_setParameter().
1427
+ * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
1428
+ * an existing CCtx.
1429
+ * These parameters will be applied to
1430
+ * all subsequent frames.
1431
+ * - ZSTD_compressStream2() : Do compression using the CCtx.
1432
+ * - ZSTD_freeCCtxParams() : Free the memory.
1433
+ *
1434
+ * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
1435
+ * for static allocation of CCtx for single-threaded compression.
1436
+ */
1437
+ ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
1438
+ ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
645
1439
 
646
- /*--- Advanced decompression functions ---*/
1440
+ /*! ZSTD_CCtxParams_reset() :
1441
+ * Reset params to default values.
1442
+ */
1443
+ ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
1444
+
1445
+ /*! ZSTD_CCtxParams_init() :
1446
+ * Initializes the compression parameters of cctxParams according to
1447
+ * compression level. All other parameters are reset to their default values.
1448
+ */
1449
+ ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
1450
+
1451
+ /*! ZSTD_CCtxParams_init_advanced() :
1452
+ * Initializes the compression and frame parameters of cctxParams according to
1453
+ * params. All other parameters are reset to their default values.
1454
+ */
1455
+ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
1456
+
1457
+ /*! ZSTD_CCtxParams_setParameter() :
1458
+ * Similar to ZSTD_CCtx_setParameter.
1459
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
1460
+ * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
1461
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1462
+ */
1463
+ ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
1464
+
1465
+ /*! ZSTD_CCtxParams_getParameter() :
1466
+ * Similar to ZSTD_CCtx_getParameter.
1467
+ * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
1468
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1469
+ */
1470
+ ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
1471
+
1472
+ /*! ZSTD_CCtx_setParametersUsingCCtxParams() :
1473
+ * Apply a set of ZSTD_CCtx_params to the compression context.
1474
+ * This can be done even after compression is started,
1475
+ * if nbWorkers==0, this will have no impact until a new compression is started.
1476
+ * if nbWorkers>=1, new parameters will be picked up at next job,
1477
+ * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
1478
+ */
1479
+ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
1480
+ ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
1481
+
1482
+ /*! ZSTD_compressStream2_simpleArgs() :
1483
+ * Same as ZSTD_compressStream2(),
1484
+ * but using only integral types as arguments.
1485
+ * This variant might be helpful for binders from dynamic languages
1486
+ * which have troubles handling structures containing memory pointers.
1487
+ */
1488
+ ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs (
1489
+ ZSTD_CCtx* cctx,
1490
+ void* dst, size_t dstCapacity, size_t* dstPos,
1491
+ const void* src, size_t srcSize, size_t* srcPos,
1492
+ ZSTD_EndDirective endOp);
1493
+
1494
+
1495
+ /***************************************
1496
+ * Advanced decompression functions
1497
+ ***************************************/
647
1498
 
648
1499
  /*! ZSTD_isFrame() :
649
1500
  * Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -652,28 +1503,6 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
652
1503
  * Note 3 : Skippable Frame Identifiers are considered valid. */
653
1504
  ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
654
1505
 
655
- /*! ZSTD_createDCtx_advanced() :
656
- * Create a ZSTD decompression context using external alloc and free functions */
657
- ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
658
-
659
- /*! ZSTD_initStaticDCtx() : initialize a fixed-size zstd decompression context
660
- * workspace: The memory area to emplace the context into.
661
- * Provided pointer must 8-bytes aligned.
662
- * It must outlive context usage.
663
- * workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
664
- * to determine how large workspace must be to support scenario.
665
- * @return : pointer to ZSTD_DCtx* (same address as workspace, but different type),
666
- * or NULL if error (typically size too small)
667
- * Note : zstd will never resize nor malloc() when using a static dctx.
668
- * If it needs more memory than available, it will simply error out.
669
- * Note 2 : static dctx is incompatible with legacy support
670
- * Note 3 : there is no corresponding "free" function.
671
- * Since workspace was allocated externally, it must be freed externally.
672
- * Limitation : currently not compatible with internal DDict creation,
673
- * such as ZSTD_initDStream_usingDict().
674
- */
675
- ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
676
-
677
1506
  /*! ZSTD_createDDict_byReference() :
678
1507
  * Create a digested dictionary, ready to start decompression operation without startup delay.
679
1508
  * Dictionary content is referenced, and therefore stays in dictBuffer.
@@ -681,86 +1510,201 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize
681
1510
  * it must remain read accessible throughout the lifetime of DDict */
682
1511
  ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
683
1512
 
684
- /*! ZSTD_createDDict_advanced() :
685
- * Create a ZSTD_DDict using external alloc and free, optionally by reference */
686
- ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
687
- ZSTD_dictLoadMethod_e dictLoadMethod,
688
- ZSTD_customMem customMem);
1513
+ /*! ZSTD_DCtx_loadDictionary_byReference() :
1514
+ * Same as ZSTD_DCtx_loadDictionary(),
1515
+ * but references `dict` content instead of copying it into `dctx`.
1516
+ * This saves memory if `dict` remains around.,
1517
+ * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
1518
+ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
1519
+
1520
+ /*! ZSTD_DCtx_loadDictionary_advanced() :
1521
+ * Same as ZSTD_DCtx_loadDictionary(),
1522
+ * but gives direct control over
1523
+ * how to load the dictionary (by copy ? by reference ?)
1524
+ * and how to interpret it (automatic ? force raw mode ? full mode only ?). */
1525
+ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
1526
+
1527
+ /*! ZSTD_DCtx_refPrefix_advanced() :
1528
+ * Same as ZSTD_DCtx_refPrefix(), but gives finer control over
1529
+ * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
1530
+ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
689
1531
 
690
- /*! ZSTD_initStaticDDict() :
691
- * Generate a digested dictionary in provided memory area.
692
- * workspace: The memory area to emplace the dictionary into.
693
- * Provided pointer must 8-bytes aligned.
694
- * It must outlive dictionary usage.
695
- * workspaceSize: Use ZSTD_estimateDDictSize()
696
- * to determine how large workspace must be.
697
- * @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
698
- * Note : there is no corresponding "free" function.
699
- * Since workspace was allocated externally, it must be freed externally.
1532
+ /*! ZSTD_DCtx_setMaxWindowSize() :
1533
+ * Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
1534
+ * This protects a decoder context from reserving too much memory for itself (potential attack scenario).
1535
+ * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
1536
+ * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
1537
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()).
700
1538
  */
701
- ZSTDLIB_API ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
702
- const void* dict, size_t dictSize,
703
- ZSTD_dictLoadMethod_e dictLoadMethod);
1539
+ ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
704
1540
 
705
- /*! ZSTD_getDictID_fromDict() :
706
- * Provides the dictID stored within dictionary.
707
- * if @return == 0, the dictionary is not conformant with Zstandard specification.
708
- * It can still be loaded, but as a content-only dictionary. */
709
- ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
1541
+ /* ZSTD_d_format
1542
+ * experimental parameter,
1543
+ * allowing selection between ZSTD_format_e input compression formats
1544
+ */
1545
+ #define ZSTD_d_format ZSTD_d_experimentalParam1
710
1546
 
711
- /*! ZSTD_getDictID_fromDDict() :
712
- * Provides the dictID of the dictionary loaded into `ddict`.
713
- * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
714
- * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
715
- ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
1547
+ /*! ZSTD_DCtx_setFormat() :
1548
+ * Instruct the decoder context about what kind of data to decode next.
1549
+ * This instruction is mandatory to decode data without a fully-formed header,
1550
+ * such ZSTD_f_zstd1_magicless for example.
1551
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
1552
+ ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
716
1553
 
717
- /*! ZSTD_getDictID_fromFrame() :
718
- * Provides the dictID required to decompressed the frame stored within `src`.
719
- * If @return == 0, the dictID could not be decoded.
720
- * This could for one of the following reasons :
721
- * - The frame does not require a dictionary to be decoded (most common case).
722
- * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
723
- * Note : this use case also happens when using a non-conformant dictionary.
724
- * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
725
- * - This is not a Zstandard frame.
726
- * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
727
- ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
1554
+ /*! ZSTD_decompressStream_simpleArgs() :
1555
+ * Same as ZSTD_decompressStream(),
1556
+ * but using only integral types as arguments.
1557
+ * This can be helpful for binders from dynamic languages
1558
+ * which have troubles handling structures containing memory pointers.
1559
+ */
1560
+ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
1561
+ ZSTD_DCtx* dctx,
1562
+ void* dst, size_t dstCapacity, size_t* dstPos,
1563
+ const void* src, size_t srcSize, size_t* srcPos);
728
1564
 
729
1565
 
730
1566
  /********************************************************************
731
1567
  * Advanced streaming functions
1568
+ * Warning : most of these functions are now redundant with the Advanced API.
1569
+ * Once Advanced API reaches "stable" status,
1570
+ * redundant functions will be deprecated, and then at some point removed.
732
1571
  ********************************************************************/
733
1572
 
734
1573
  /*===== Advanced Streaming compression functions =====*/
735
- ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
736
- ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
737
- ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
738
- ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
1574
+ /**! ZSTD_initCStream_srcSize() :
1575
+ * This function is deprecated, and equivalent to:
1576
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1577
+ * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
1578
+ * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
1579
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
1580
+ *
1581
+ * pledgedSrcSize must be correct. If it is not known at init time, use
1582
+ * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
1583
+ * "0" also disables frame content size field. It may be enabled in the future.
1584
+ */
1585
+ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
1586
+ /**! ZSTD_initCStream_usingDict() :
1587
+ * This function is deprecated, and is equivalent to:
1588
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1589
+ * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
1590
+ * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
1591
+ *
1592
+ * Creates of an internal CDict (incompatible with static CCtx), except if
1593
+ * dict == NULL or dictSize < 8, in which case no dict is used.
1594
+ * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
1595
+ * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
1596
+ */
1597
+ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
1598
+ /**! ZSTD_initCStream_advanced() :
1599
+ * This function is deprecated, and is approximately equivalent to:
1600
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1601
+ * ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is
1602
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
1603
+ * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
1604
+ *
1605
+ * pledgedSrcSize must be correct. If srcSize is not known at init time, use
1606
+ * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
1607
+ */
739
1608
  ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
740
- ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
741
- ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
742
- ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */
1609
+ ZSTD_parameters params, unsigned long long pledgedSrcSize);
1610
+ /**! ZSTD_initCStream_usingCDict() :
1611
+ * This function is deprecated, and equivalent to:
1612
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1613
+ * ZSTD_CCtx_refCDict(zcs, cdict);
1614
+ *
1615
+ * note : cdict will just be referenced, and must outlive compression session
1616
+ */
1617
+ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
1618
+ /**! ZSTD_initCStream_usingCDict_advanced() :
1619
+ * This function is deprecated, and is approximately equivalent to:
1620
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1621
+ * ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is
1622
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
1623
+ * ZSTD_CCtx_refCDict(zcs, cdict);
1624
+ *
1625
+ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
1626
+ * pledgedSrcSize must be correct. If srcSize is not known at init time, use
1627
+ * value ZSTD_CONTENTSIZE_UNKNOWN.
1628
+ */
1629
+ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
743
1630
 
744
1631
  /*! ZSTD_resetCStream() :
745
- * start a new compression job, using same parameters from previous job.
746
- * This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
1632
+ * This function is deprecated, and is equivalent to:
1633
+ * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
1634
+ * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
1635
+ *
1636
+ * start a new frame, using same parameters from previous frame.
1637
+ * This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
747
1638
  * Note that zcs must be init at least once before using ZSTD_resetCStream().
748
1639
  * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
749
1640
  * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
750
1641
  * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
751
- * but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
752
- * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
1642
+ * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
1643
+ * @return : 0, or an error code (which can be tested using ZSTD_isError())
1644
+ */
753
1645
  ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
754
1646
 
755
1647
 
1648
+ typedef struct {
1649
+ unsigned long long ingested; /* nb input bytes read and buffered */
1650
+ unsigned long long consumed; /* nb input bytes actually compressed */
1651
+ unsigned long long produced; /* nb of compressed bytes generated and buffered */
1652
+ unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
1653
+ unsigned currentJobID; /* MT only : latest started job nb */
1654
+ unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */
1655
+ } ZSTD_frameProgression;
1656
+
1657
+ /* ZSTD_getFrameProgression() :
1658
+ * tells how much data has been ingested (read from input)
1659
+ * consumed (input actually compressed) and produced (output) for current frame.
1660
+ * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
1661
+ * Aggregates progression inside active worker threads.
1662
+ */
1663
+ ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
1664
+
1665
+ /*! ZSTD_toFlushNow() :
1666
+ * Tell how many bytes are ready to be flushed immediately.
1667
+ * Useful for multithreading scenarios (nbWorkers >= 1).
1668
+ * Probe the oldest active job, defined as oldest job not yet entirely flushed,
1669
+ * and check its output buffer.
1670
+ * @return : amount of data stored in oldest job and ready to be flushed immediately.
1671
+ * if @return == 0, it means either :
1672
+ * + there is no active job (could be checked with ZSTD_frameProgression()), or
1673
+ * + oldest job is still actively compressing data,
1674
+ * but everything it has produced has also been flushed so far,
1675
+ * therefore flush speed is limited by production speed of oldest job
1676
+ * irrespective of the speed of concurrent (and newer) jobs.
1677
+ */
1678
+ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
1679
+
1680
+
756
1681
  /*===== Advanced Streaming decompression functions =====*/
757
- ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
758
- ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
759
- typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
760
- ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); /* obsolete : this API will be removed in a future version */
761
- ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
762
- ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */
763
- ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
1682
+ /**
1683
+ * This function is deprecated, and is equivalent to:
1684
+ *
1685
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
1686
+ * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
1687
+ *
1688
+ * note: no dictionary will be used if dict == NULL or dictSize < 8
1689
+ */
1690
+ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
1691
+ /**
1692
+ * This function is deprecated, and is equivalent to:
1693
+ *
1694
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
1695
+ * ZSTD_DCtx_refDDict(zds, ddict);
1696
+ *
1697
+ * note : ddict is referenced, it must outlive decompression session
1698
+ */
1699
+ ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
1700
+ /**
1701
+ * This function is deprecated, and is equivalent to:
1702
+ *
1703
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
1704
+ *
1705
+ * re-use decompression parameters from previous init; saves dictionary loading
1706
+ */
1707
+ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
764
1708
 
765
1709
 
766
1710
  /*********************************************************************
@@ -898,7 +1842,17 @@ typedef struct {
898
1842
  unsigned dictID;
899
1843
  unsigned checksumFlag;
900
1844
  } ZSTD_frameHeader;
1845
+
1846
+ /** ZSTD_getFrameHeader() :
1847
+ * decode Frame Header, or requires larger `srcSize`.
1848
+ * @return : 0, `zfhPtr` is correctly filled,
1849
+ * >0, `srcSize` is too small, value is wanted `srcSize` amount,
1850
+ * or an error code, which can be tested using ZSTD_isError() */
901
1851
  ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
1852
+ /*! ZSTD_getFrameHeader_advanced() :
1853
+ * same as ZSTD_getFrameHeader(),
1854
+ * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
1855
+ ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
902
1856
  ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
903
1857
 
904
1858
  ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
@@ -915,444 +1869,6 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
915
1869
 
916
1870
 
917
1871
 
918
- /* ============================================ */
919
- /** New advanced API (experimental) */
920
- /* ============================================ */
921
-
922
- /* notes on API design :
923
- * In this proposal, parameters are pushed one by one into an existing context,
924
- * and then applied on all subsequent compression jobs.
925
- * When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT.
926
- *
927
- * This API is intended to replace all others experimental API.
928
- * It can basically do all other use cases, and even new ones.
929
- * In constrast with _advanced() variants, it stands a reasonable chance to become "stable",
930
- * after a good testing period.
931
- */
932
-
933
- /* note on naming convention :
934
- * Initially, the API favored names like ZSTD_setCCtxParameter() .
935
- * In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
936
- * The main driver is that it identifies more clearly the target object type.
937
- * It feels clearer when considering multiple targets :
938
- * ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
939
- * ZSTD_CCtxParams_setParameter() (rather than ZSTD_setCCtxParamsParameter() )
940
- * etc...
941
- */
942
-
943
- /* note on enum design :
944
- * All enum will be pinned to explicit values before reaching "stable API" status */
945
-
946
- typedef enum {
947
- /* Question : should we have a format ZSTD_f_auto ?
948
- * For the time being, it would mean exactly the same as ZSTD_f_zstd1.
949
- * But, in the future, should several formats be supported,
950
- * on the compression side, it would mean "default format".
951
- * On the decompression side, it would mean "multi format",
952
- * and ZSTD_f_zstd1 could be reserved to mean "accept *only* zstd frames".
953
- * Since meaning is a little different, another option could be to define different enums for compression and decompression.
954
- * This question could be kept for later, when there are actually multiple formats to support,
955
- * but there is also the question of pinning enum values, and pinning value `0` is especially important */
956
- ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
957
- ZSTD_f_zstd1_magicless, /* Variant of zstd frame format, without initial 4-bytes magic number.
958
- * Useful to save 4 bytes per generated frame.
959
- * Decoder cannot recognise automatically this format, requiring instructions. */
960
- } ZSTD_format_e;
961
-
962
- typedef enum {
963
- /* compression format */
964
- ZSTD_p_format = 10, /* See ZSTD_format_e enum definition.
965
- * Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
966
-
967
- /* compression parameters */
968
- ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
969
- * Default level is ZSTD_CLEVEL_DEFAULT==3.
970
- * Special: value 0 means "do not change cLevel". */
971
- ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
972
- * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
973
- * Special: value 0 means "do not change windowLog".
974
- * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
975
- * requires setting the maximum window size at least as large during decompression. */
976
- ZSTD_p_hashLog, /* Size of the probe table, as a power of 2.
977
- * Resulting table size is (1 << (hashLog+2)).
978
- * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
979
- * Larger tables improve compression ratio of strategies <= dFast,
980
- * and improve speed of strategies > dFast.
981
- * Special: value 0 means "do not change hashLog". */
982
- ZSTD_p_chainLog, /* Size of the full-search table, as a power of 2.
983
- * Resulting table size is (1 << (chainLog+2)).
984
- * Larger tables result in better and slower compression.
985
- * This parameter is useless when using "fast" strategy.
986
- * Special: value 0 means "do not change chainLog". */
987
- ZSTD_p_searchLog, /* Number of search attempts, as a power of 2.
988
- * More attempts result in better and slower compression.
989
- * This parameter is useless when using "fast" and "dFast" strategies.
990
- * Special: value 0 means "do not change searchLog". */
991
- ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller).
992
- * Larger values make faster compression and decompression, but decrease ratio.
993
- * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
994
- * Note that currently, for all strategies < btopt, effective minimum is 4.
995
- * Note that currently, for all strategies > fast, effective maximum is 6.
996
- * Special: value 0 means "do not change minMatchLength". */
997
- ZSTD_p_targetLength, /* Only useful for strategies >= btopt.
998
- * Length of Match considered "good enough" to stop search.
999
- * Larger values make compression stronger and slower.
1000
- * Special: value 0 means "do not change targetLength". */
1001
- ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
1002
- * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
1003
- * The higher the value of selected strategy, the more complex it is,
1004
- * resulting in stronger and slower compression.
1005
- * Special: value 0 means "do not change strategy". */
1006
-
1007
- /* frame parameters */
1008
- ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
1009
- * Content size must be known at the beginning of compression,
1010
- * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
1011
- ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
1012
- ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */
1013
-
1014
- /* multi-threading parameters */
1015
- ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
1016
- * More threads improve speed, but also increase memory usage.
1017
- * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
1018
- * Special: value 0 means "do not change nbThreads" */
1019
- ZSTD_p_jobSize, /* Size of a compression job. This value is only enforced in streaming (non-blocking) mode.
1020
- * Each compression job is completed in parallel, so indirectly controls the nb of active threads.
1021
- * 0 means default, which is dynamically determined based on compression parameters.
1022
- * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
1023
- * The minimum size is automatically and transparently enforced */
1024
- ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
1025
- * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
1026
-
1027
- /* advanced parameters - may not remain available after API update */
1028
- ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
1029
- * even when referencing into Dictionary content (default:0) */
1030
- ZSTD_p_enableLongDistanceMatching=1200, /* Enable long distance matching.
1031
- * This parameter is designed to improve the compression
1032
- * ratio for large inputs with long distance matches.
1033
- * This increases the memory usage as well as window size.
1034
- * Note: setting this parameter sets all the LDM parameters
1035
- * as well as ZSTD_p_windowLog. It should be set after
1036
- * ZSTD_p_compressionLevel and before ZSTD_p_windowLog and
1037
- * other LDM parameters. Setting the compression level
1038
- * after this parameter overrides the window log, though LDM
1039
- * will remain enabled until explicitly disabled. */
1040
- ZSTD_p_ldmHashLog, /* Size of the table for long distance matching, as a power of 2.
1041
- * Larger values increase memory usage and compression ratio, but decrease
1042
- * compression speed.
1043
- * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
1044
- * (default: windowlog - 7). */
1045
- ZSTD_p_ldmMinMatch, /* Minimum size of searched matches for long distance matcher.
1046
- * Larger/too small values usually decrease compression ratio.
1047
- * Must be clamped between ZSTD_LDM_MINMATCH_MIN
1048
- * and ZSTD_LDM_MINMATCH_MAX (default: 64). */
1049
- ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the LDM hash table for collision resolution.
1050
- * Larger values usually improve collision resolution but may decrease
1051
- * compression speed.
1052
- * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX (default: 3). */
1053
- ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table.
1054
- * The default is MAX(0, (windowLog - ldmHashLog)) to
1055
- * optimize hash table usage.
1056
- * Larger values improve compression speed. Deviating far from the
1057
- * default value will likely result in a decrease in compression ratio.
1058
- * Must be clamped between 0 and ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN. */
1059
-
1060
- } ZSTD_cParameter;
1061
-
1062
-
1063
- /*! ZSTD_CCtx_setParameter() :
1064
- * Set one compression parameter, selected by enum ZSTD_cParameter.
1065
- * Note : when `value` is an enum, cast it to unsigned for proper type checking.
1066
- * @result : informational value (typically, the one being set, possibly corrected),
1067
- * or an error code (which can be tested with ZSTD_isError()). */
1068
- ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
1069
-
1070
- /*! ZSTD_CCtx_setPledgedSrcSize() :
1071
- * Total input data size to be compressed as a single frame.
1072
- * This value will be controlled at the end, and result in error if not respected.
1073
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1074
- * Note 1 : 0 means zero, empty.
1075
- * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
1076
- * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
1077
- * Note 2 : If all data is provided and consumed in a single round,
1078
- * this value is overriden by srcSize instead. */
1079
- ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
1080
-
1081
- /*! ZSTD_CCtx_loadDictionary() :
1082
- * Create an internal CDict from dict buffer.
1083
- * Decompression will have to use same buffer.
1084
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1085
- * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
1086
- * meaning "return to no-dictionary mode".
1087
- * Note 1 : `dict` content will be copied internally. Use
1088
- * ZSTD_CCtx_loadDictionary_byReference() to reference dictionary
1089
- * content instead. The dictionary buffer must then outlive its
1090
- * users.
1091
- * Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
1092
- * For this reason, compression parameters cannot be changed anymore after loading a dictionary.
1093
- * It's also a CPU-heavy operation, with non-negligible impact on latency.
1094
- * Note 3 : Dictionary will be used for all future compression jobs.
1095
- * To return to "no-dictionary" situation, load a NULL dictionary
1096
- * Note 5 : Use ZSTD_CCtx_loadDictionary_advanced() to select how dictionary
1097
- * content will be interpreted.
1098
- */
1099
- ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
1100
- ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
1101
- ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);
1102
-
1103
-
1104
- /*! ZSTD_CCtx_refCDict() :
1105
- * Reference a prepared dictionary, to be used for all next compression jobs.
1106
- * Note that compression parameters are enforced from within CDict,
1107
- * and supercede any compression parameter previously set within CCtx.
1108
- * The dictionary will remain valid for future compression jobs using same CCtx.
1109
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1110
- * Special : adding a NULL CDict means "return to no-dictionary mode".
1111
- * Note 1 : Currently, only one dictionary can be managed.
1112
- * Adding a new dictionary effectively "discards" any previous one.
1113
- * Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
1114
- */
1115
- ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
1116
-
1117
- /*! ZSTD_CCtx_refPrefix() :
1118
- * Reference a prefix (single-usage dictionary) for next compression job.
1119
- * Decompression need same prefix to properly regenerate data.
1120
- * Prefix is **only used once**. Tables are discarded at end of compression job.
1121
- * Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
1122
- * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
1123
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1124
- * Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
1125
- * Note 1 : Prefix buffer is referenced. It must outlive compression job.
1126
- * Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
1127
- * It's a CPU-heavy operation, with non-negligible impact on latency.
1128
- * Note 3 : By default, the prefix is treated as raw content
1129
- * (ZSTD_dm_rawContent). Use ZSTD_CCtx_refPrefix_advanced() to alter
1130
- * dictMode. */
1131
- ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
1132
- ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode);
1133
-
1134
-
1135
-
1136
- typedef enum {
1137
- ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
1138
- ZSTD_e_flush, /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
1139
- ZSTD_e_end /* flush any remaining data and close current frame. Any additional data starts a new frame. */
1140
- } ZSTD_EndDirective;
1141
-
1142
- /*! ZSTD_compress_generic() :
1143
- * Behave about the same as ZSTD_compressStream. To note :
1144
- * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
1145
- * - Compression parameters cannot be changed once compression is started.
1146
- * - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
1147
- * - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
1148
- * - In single-thread mode (default), function is blocking : it completed its job before returning to caller.
1149
- * - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
1150
- * and then immediately returns, just indicating that there is some data remaining to be flushed.
1151
- * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
1152
- * - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
1153
- * - @return provides the minimum amount of data remaining to be flushed from internal buffers
1154
- * or an error code, which can be tested using ZSTD_isError().
1155
- * if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
1156
- * This is useful to determine if a ZSTD_e_flush or ZSTD_e_end directive is completed.
1157
- * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
1158
- * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
1159
- * Before starting a new compression job, or changing compression parameters,
1160
- * it is required to fully flush internal buffers.
1161
- */
1162
- ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
1163
- ZSTD_outBuffer* output,
1164
- ZSTD_inBuffer* input,
1165
- ZSTD_EndDirective endOp);
1166
-
1167
- /*! ZSTD_CCtx_reset() :
1168
- * Return a CCtx to clean state.
1169
- * Useful after an error, or to interrupt an ongoing compression job and start a new one.
1170
- * Any internal data not yet flushed is cancelled.
1171
- * Dictionary (if any) is dropped.
1172
- * All parameters are back to default values.
1173
- * It's possible to modify compression parameters after a reset.
1174
- */
1175
- ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); /* Not ready yet ! */
1176
-
1177
-
1178
- /*! ZSTD_compress_generic_simpleArgs() :
1179
- * Same as ZSTD_compress_generic(),
1180
- * but using only integral types as arguments.
1181
- * Argument list is larger than ZSTD_{in,out}Buffer,
1182
- * but can be helpful for binders from dynamic languages
1183
- * which have troubles handling structures containing memory pointers.
1184
- */
1185
- ZSTDLIB_API size_t ZSTD_compress_generic_simpleArgs (
1186
- ZSTD_CCtx* cctx,
1187
- void* dst, size_t dstCapacity, size_t* dstPos,
1188
- const void* src, size_t srcSize, size_t* srcPos,
1189
- ZSTD_EndDirective endOp);
1190
-
1191
-
1192
- /*! ZSTD_CCtx_params :
1193
- * Quick howto :
1194
- * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
1195
- * - ZSTD_CCtxParam_setParameter() : Push parameters one by one into
1196
- * an existing ZSTD_CCtx_params structure.
1197
- * This is similar to
1198
- * ZSTD_CCtx_setParameter().
1199
- * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
1200
- * an existing CCtx.
1201
- * These parameters will be applied to
1202
- * all subsequent compression jobs.
1203
- * - ZSTD_compress_generic() : Do compression using the CCtx.
1204
- * - ZSTD_freeCCtxParams() : Free the memory.
1205
- *
1206
- * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
1207
- * for static allocation for single-threaded compression.
1208
- */
1209
- ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
1210
-
1211
- /*! ZSTD_resetCCtxParams() :
1212
- * Reset params to default, with the default compression level.
1213
- */
1214
- ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params);
1215
-
1216
- /*! ZSTD_initCCtxParams() :
1217
- * Initializes the compression parameters of cctxParams according to
1218
- * compression level. All other parameters are reset to their default values.
1219
- */
1220
- ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, int compressionLevel);
1221
-
1222
- /*! ZSTD_initCCtxParams_advanced() :
1223
- * Initializes the compression and frame parameters of cctxParams according to
1224
- * params. All other parameters are reset to their default values.
1225
- */
1226
- ZSTDLIB_API size_t ZSTD_initCCtxParams_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
1227
-
1228
- ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
1229
-
1230
- /*! ZSTD_CCtxParam_setParameter() :
1231
- * Similar to ZSTD_CCtx_setParameter.
1232
- * Set one compression parameter, selected by enum ZSTD_cParameter.
1233
- * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
1234
- * Note : when `value` is an enum, cast it to unsigned for proper type checking.
1235
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1236
- */
1237
- ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
1238
-
1239
- /*! ZSTD_CCtx_setParametersUsingCCtxParams() :
1240
- * Apply a set of ZSTD_CCtx_params to the compression context.
1241
- * This must be done before the dictionary is loaded.
1242
- * The pledgedSrcSize is treated as unknown.
1243
- * Multithreading parameters are applied only if nbThreads > 1.
1244
- */
1245
- ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
1246
- ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
1247
-
1248
-
1249
- /*=== Advanced parameters for decompression API ===*/
1250
-
1251
- /* The following parameters must be set after creating a ZSTD_DCtx* (or ZSTD_DStream*) object,
1252
- * but before starting decompression of a frame.
1253
- */
1254
-
1255
- /*! ZSTD_DCtx_loadDictionary() :
1256
- * Create an internal DDict from dict buffer,
1257
- * to be used to decompress next frames.
1258
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1259
- * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
1260
- * meaning "return to no-dictionary mode".
1261
- * Note 1 : `dict` content will be copied internally.
1262
- * Use ZSTD_DCtx_loadDictionary_byReference()
1263
- * to reference dictionary content instead.
1264
- * In which case, the dictionary buffer must outlive its users.
1265
- * Note 2 : Loading a dictionary involves building tables,
1266
- * which has a non-negligible impact on CPU usage and latency.
1267
- * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
1268
- * how dictionary content will be interpreted and loaded.
1269
- */
1270
- ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */
1271
- ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */
1272
- ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); /* not implemented */
1273
-
1274
-
1275
- /*! ZSTD_DCtx_refDDict() :
1276
- * Reference a prepared dictionary, to be used to decompress next frames.
1277
- * The dictionary remains active for decompression of future frames using same DCtx.
1278
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1279
- * Note 1 : Currently, only one dictionary can be managed.
1280
- * Referencing a new dictionary effectively "discards" any previous one.
1281
- * Special : adding a NULL DDict means "return to no-dictionary mode".
1282
- * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
1283
- */
1284
- ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); /* not implemented */
1285
-
1286
-
1287
- /*! ZSTD_DCtx_refPrefix() :
1288
- * Reference a prefix (single-usage dictionary) for next compression job.
1289
- * Prefix is **only used once**. It must be explicitly referenced before each frame.
1290
- * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead.
1291
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1292
- * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
1293
- * Note 2 : Prefix buffer is referenced. It must outlive compression job.
1294
- * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
1295
- * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
1296
- * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
1297
- */
1298
- ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); /* not implemented */
1299
- ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode); /* not implemented */
1300
-
1301
-
1302
- /*! ZSTD_DCtx_setMaxWindowSize() :
1303
- * Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
1304
- * This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario).
1305
- * This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode.
1306
- * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX)
1307
- * @return : 0, or an error code (which can be tested using ZSTD_isError()).
1308
- */
1309
- ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
1310
-
1311
-
1312
- /*! ZSTD_DCtx_setFormat() :
1313
- * Instruct the decoder context about what kind of data to decode next.
1314
- * This instruction is mandatory to decode data without a fully-formed header,
1315
- * such ZSTD_f_zstd1_magicless for example.
1316
- * @return : 0, or an error code (which can be tested using ZSTD_isError()).
1317
- */
1318
- ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
1319
-
1320
-
1321
- /*! ZSTD_decompress_generic() :
1322
- * Behave the same as ZSTD_decompressStream.
1323
- * Decompression parameters cannot be changed once decompression is started.
1324
- * @return : an error code, which can be tested using ZSTD_isError()
1325
- * if >0, a hint, nb of expected input bytes for next invocation.
1326
- * `0` means : a frame has just been fully decoded and flushed.
1327
- */
1328
- ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx,
1329
- ZSTD_outBuffer* output,
1330
- ZSTD_inBuffer* input);
1331
-
1332
-
1333
- /*! ZSTD_decompress_generic_simpleArgs() :
1334
- * Same as ZSTD_decompress_generic(),
1335
- * but using only integral types as arguments.
1336
- * Argument list is larger than ZSTD_{in,out}Buffer,
1337
- * but can be helpful for binders from dynamic languages
1338
- * which have troubles handling structures containing memory pointers.
1339
- */
1340
- ZSTDLIB_API size_t ZSTD_decompress_generic_simpleArgs (
1341
- ZSTD_DCtx* dctx,
1342
- void* dst, size_t dstCapacity, size_t* dstPos,
1343
- const void* src, size_t srcSize, size_t* srcPos);
1344
-
1345
-
1346
- /*! ZSTD_DCtx_reset() :
1347
- * Return a DCtx to clean state.
1348
- * If a decompression was ongoing, any internal data not yet flushed is cancelled.
1349
- * All parameters are back to default values, including sticky ones.
1350
- * Dictionary (if any) is dropped.
1351
- * Parameters can be modified again after a reset.
1352
- */
1353
- ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
1354
-
1355
-
1356
1872
 
1357
1873
  /* ============================ */
1358
1874
  /** Block level API */
@@ -1372,10 +1888,10 @@ ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
1372
1888
  + copyCCtx() and copyDCtx() can be used too
1373
1889
  - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
1374
1890
  + If input is larger than a block size, it's necessary to split input data into multiple blocks
1375
- + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
1891
+ + For inputs larger than a single block, really consider using regular ZSTD_compress() instead.
1376
1892
  Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
1377
1893
  - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
1378
- In which case, nothing is produced into `dst`.
1894
+ In which case, nothing is produced into `dst` !
1379
1895
  + User must test for such outcome and deal directly with uncompressed data
1380
1896
  + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
1381
1897
  + In case of multiple successive blocks, should some of them be uncompressed,
@@ -1383,13 +1899,11 @@ ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx);
1383
1899
  Use ZSTD_insertBlock() for such a case.
1384
1900
  */
1385
1901
 
1386
- #define ZSTD_BLOCKSIZELOG_MAX 17
1387
- #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */
1388
1902
  /*===== Raw zstd block functions =====*/
1389
1903
  ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
1390
1904
  ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
1391
1905
  ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
1392
- ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression */
1906
+ ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
1393
1907
 
1394
1908
 
1395
1909
  #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */