zstd-ruby 1.2.0.0 → 1.3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/ext/zstdruby/libzstd/Makefile +7 -5
  4. data/ext/zstdruby/libzstd/common/bitstream.h +23 -9
  5. data/ext/zstdruby/libzstd/common/error_private.c +4 -1
  6. data/ext/zstdruby/libzstd/common/huf.h +20 -0
  7. data/ext/zstdruby/libzstd/common/mem.h +0 -14
  8. data/ext/zstdruby/libzstd/common/pool.c +12 -0
  9. data/ext/zstdruby/libzstd/common/pool.h +5 -0
  10. data/ext/zstdruby/libzstd/common/threading.c +0 -1
  11. data/ext/zstdruby/libzstd/common/zstd_common.c +25 -18
  12. data/ext/zstdruby/libzstd/common/zstd_errors.h +15 -7
  13. data/ext/zstdruby/libzstd/common/zstd_internal.h +59 -9
  14. data/ext/zstdruby/libzstd/compress/huf_compress.c +7 -3
  15. data/ext/zstdruby/libzstd/compress/zstd_compress.c +1082 -487
  16. data/ext/zstdruby/libzstd/compress/zstd_opt.h +30 -15
  17. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +362 -158
  18. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +49 -13
  19. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +150 -26
  20. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +380 -258
  21. data/ext/zstdruby/libzstd/dictBuilder/cover.c +23 -37
  22. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +30 -40
  23. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +104 -95
  24. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +11 -10
  25. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +14 -19
  26. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +13 -12
  27. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +13 -14
  28. data/ext/zstdruby/libzstd/zstd.h +507 -166
  29. data/lib/zstd-ruby/version.rb +1 -1
  30. metadata +2 -2
@@ -816,13 +816,13 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
816
816
  bitD->bitContainer = *(const BYTE*)(bitD->start);
817
817
  switch(srcSize)
818
818
  {
819
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
820
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
821
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
822
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
823
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
824
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
825
- default:;
819
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);/* fall-through */
820
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);/* fall-through */
821
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);/* fall-through */
822
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; /* fall-through */
823
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; /* fall-through */
824
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; /* fall-through */
825
+ default: break;
826
826
  }
827
827
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
828
828
  if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
@@ -3665,7 +3665,7 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
3665
3665
  break;
3666
3666
  }
3667
3667
  zbc->stage = ZBUFFds_read;
3668
-
3668
+ /* fall-through */
3669
3669
  case ZBUFFds_read:
3670
3670
  {
3671
3671
  size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
@@ -3691,7 +3691,7 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
3691
3691
  if (ip==iend) { notDone = 0; break; } /* no more input */
3692
3692
  zbc->stage = ZBUFFds_load;
3693
3693
  }
3694
-
3694
+ /* fall-through */
3695
3695
  case ZBUFFds_load:
3696
3696
  {
3697
3697
  size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
@@ -3711,9 +3711,10 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
3711
3711
  if (!decodedSize) { zbc->stage = ZBUFFds_read; break; } /* this was just a header */
3712
3712
  zbc->outEnd = zbc->outStart + decodedSize;
3713
3713
  zbc->stage = ZBUFFds_flush;
3714
- // break; /* ZBUFFds_flush follows */
3714
+ /* ZBUFFds_flush follows */
3715
3715
  }
3716
3716
  }
3717
+ /* fall-through */
3717
3718
  case ZBUFFds_flush:
3718
3719
  {
3719
3720
  size_t toFlushSize = zbc->outEnd - zbc->outStart;
@@ -326,13 +326,6 @@ size_t ZSTDv05_decompress_usingPreparedDCtx(
326
326
  * Streaming functions (direct mode)
327
327
  ****************************************/
328
328
  size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx);
329
- size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize);
330
- void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dctx, const ZSTDv05_DCtx* preparedDCtx);
331
-
332
- size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize);
333
-
334
- size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx);
335
- size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
336
329
 
337
330
  /*
338
331
  Streaming decompression, direct mode (bufferless)
@@ -818,13 +811,13 @@ MEM_STATIC size_t BITv05_initDStream(BITv05_DStream_t* bitD, const void* srcBuff
818
811
  bitD->bitContainer = *(const BYTE*)(bitD->start);
819
812
  switch(srcSize)
820
813
  {
821
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
822
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
823
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
824
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
825
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
826
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
827
- default:;
814
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);/* fall-through */
815
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);/* fall-through */
816
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);/* fall-through */
817
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; /* fall-through */
818
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; /* fall-through */
819
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; /* fall-through */
820
+ default: break;
828
821
  }
829
822
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
830
823
  if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
@@ -3955,7 +3948,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3955
3948
  zbc->stage = ZBUFFv05ds_decodeHeader;
3956
3949
  break;
3957
3950
  }
3958
-
3951
+ /* fall-through */
3959
3952
  case ZBUFFv05ds_loadHeader:
3960
3953
  /* complete header from src */
3961
3954
  {
@@ -3973,7 +3966,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3973
3966
  }
3974
3967
  // zbc->stage = ZBUFFv05ds_decodeHeader; break; /* useless : stage follows */
3975
3968
  }
3976
-
3969
+ /* fall-through */
3977
3970
  case ZBUFFv05ds_decodeHeader:
3978
3971
  /* apply header to create / resize buffers */
3979
3972
  {
@@ -4000,7 +3993,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
4000
3993
  break;
4001
3994
  }
4002
3995
  zbc->stage = ZBUFFv05ds_read;
4003
-
3996
+ /* fall-through */
4004
3997
  case ZBUFFv05ds_read:
4005
3998
  {
4006
3999
  size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
@@ -4024,7 +4017,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
4024
4017
  if (ip==iend) { notDone = 0; break; } /* no more input */
4025
4018
  zbc->stage = ZBUFFv05ds_load;
4026
4019
  }
4027
-
4020
+ /* fall-through */
4028
4021
  case ZBUFFv05ds_load:
4029
4022
  {
4030
4023
  size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
@@ -4045,7 +4038,9 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
4045
4038
  zbc->outEnd = zbc->outStart + decodedSize;
4046
4039
  zbc->stage = ZBUFFv05ds_flush;
4047
4040
  // break; /* ZBUFFv05ds_flush follows */
4048
- } }
4041
+ }
4042
+ }
4043
+ /* fall-through */
4049
4044
  case ZBUFFv05ds_flush:
4050
4045
  {
4051
4046
  size_t toFlushSize = zbc->outEnd - zbc->outStart;
@@ -910,13 +910,13 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
910
910
  bitD->bitContainer = *(const BYTE*)(bitD->start);
911
911
  switch(srcSize)
912
912
  {
913
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
914
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
915
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
916
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
917
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
918
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
919
- default:;
913
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);/* fall-through */
914
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);/* fall-through */
915
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);/* fall-through */
916
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; /* fall-through */
917
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; /* fall-through */
918
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; /* fall-through */
919
+ default: break;
920
920
  }
921
921
  { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
922
922
  if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
@@ -3789,7 +3789,7 @@ size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapac
3789
3789
  return 0;
3790
3790
  }
3791
3791
  dctx->expected = 0; /* not necessary to copy more */
3792
-
3792
+ /* fall-through */
3793
3793
  case ZSTDds_decodeFrameHeader:
3794
3794
  { size_t result;
3795
3795
  memcpy(dctx->headerBuffer + ZSTDv06_frameHeaderSize_min, src, dctx->expected);
@@ -4116,7 +4116,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4116
4116
  if (zbd->outBuff == NULL) return ERROR(memory_allocation);
4117
4117
  } } }
4118
4118
  zbd->stage = ZBUFFds_read;
4119
-
4119
+ /* fall-through */
4120
4120
  case ZBUFFds_read:
4121
4121
  { size_t const neededInSize = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
4122
4122
  if (neededInSize==0) { /* end of frame */
@@ -4138,7 +4138,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4138
4138
  if (ip==iend) { notDone = 0; break; } /* no more input */
4139
4139
  zbd->stage = ZBUFFds_load;
4140
4140
  }
4141
-
4141
+ /* fall-through */
4142
4142
  case ZBUFFds_load:
4143
4143
  { size_t const neededInSize = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
4144
4144
  size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
@@ -4159,8 +4159,9 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4159
4159
  zbd->outEnd = zbd->outStart + decodedSize;
4160
4160
  zbd->stage = ZBUFFds_flush;
4161
4161
  // break; /* ZBUFFds_flush follows */
4162
- } }
4163
-
4162
+ }
4163
+ }
4164
+ /* fall-through */
4164
4165
  case ZBUFFds_flush:
4165
4166
  { size_t const toFlushSize = zbd->outEnd - zbd->outStart;
4166
4167
  size_t const flushedSize = ZBUFFv06_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
@@ -580,13 +580,13 @@ MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuff
580
580
  bitD->bitContainer = *(const BYTE*)(bitD->start);
581
581
  switch(srcSize)
582
582
  {
583
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
584
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
585
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
586
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
587
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
588
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
589
- default:;
583
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);/* fall-through */
584
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);/* fall-through */
585
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);/* fall-through */
586
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; /* fall-through */
587
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; /* fall-through */
588
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; /* fall-through */
589
+ default: break;
590
590
  }
591
591
  { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
592
592
  bitD->bitsConsumed = lastByte ? 8 - BITv07_highbit32(lastByte) : 0;
@@ -2920,8 +2920,6 @@ typedef struct {
2920
2920
  void ZSTDv07_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
2921
2921
 
2922
2922
  /* custom memory allocation functions */
2923
- void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size);
2924
- void ZSTDv07_defaultFreeFunction(void* opaque, void* address);
2925
2923
  static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction, ZSTDv07_defaultFreeFunction, NULL };
2926
2924
 
2927
2925
  #endif /* ZSTDv07_CCOMMON_H_MODULE */
@@ -4047,7 +4045,7 @@ size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapac
4047
4045
  return 0;
4048
4046
  }
4049
4047
  dctx->expected = 0; /* not necessary to copy more */
4050
-
4048
+ /* fall-through */
4051
4049
  case ZSTDds_decodeFrameHeader:
4052
4050
  { size_t result;
4053
4051
  memcpy(dctx->headerBuffer + ZSTDv07_frameHeaderSize_min, src, dctx->expected);
@@ -4494,7 +4492,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
4494
4492
  } } }
4495
4493
  zbd->stage = ZBUFFds_read;
4496
4494
  /* pass-through */
4497
-
4495
+ /* fall-through */
4498
4496
  case ZBUFFds_read:
4499
4497
  { size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);
4500
4498
  if (neededInSize==0) { /* end of frame */
@@ -4517,7 +4515,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
4517
4515
  if (ip==iend) { notDone = 0; break; } /* no more input */
4518
4516
  zbd->stage = ZBUFFds_load;
4519
4517
  }
4520
-
4518
+ /* fall-through */
4521
4519
  case ZBUFFds_load:
4522
4520
  { size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);
4523
4521
  size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
@@ -4540,8 +4538,9 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
4540
4538
  zbd->stage = ZBUFFds_flush;
4541
4539
  /* break; */
4542
4540
  /* pass-through */
4543
- } }
4544
-
4541
+ }
4542
+ }
4543
+ /* fall-through */
4545
4544
  case ZBUFFds_flush:
4546
4545
  { size_t const toFlushSize = zbd->outEnd - zbd->outStart;
4547
4546
  size_t const flushedSize = ZBUFFv07_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
@@ -19,10 +19,12 @@ extern "C" {
19
19
 
20
20
 
21
21
  /* ===== ZSTDLIB_API : control library symbols visibility ===== */
22
- #if defined(__GNUC__) && (__GNUC__ >= 4)
23
- # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
24
- #else
25
- # define ZSTDLIB_VISIBILITY
22
+ #ifndef ZSTDLIB_VISIBILITY
23
+ # if defined(__GNUC__) && (__GNUC__ >= 4)
24
+ # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
25
+ # else
26
+ # define ZSTDLIB_VISIBILITY
27
+ # endif
26
28
  #endif
27
29
  #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
28
30
  # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
@@ -36,35 +38,37 @@ extern "C" {
36
38
  /*******************************************************************************************************
37
39
  Introduction
38
40
 
39
- zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
40
- at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
41
- decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
41
+ zstd, short for Zstandard, is a fast lossless compression algorithm,
42
+ targeting real-time compression scenarios at zlib-level and better compression ratios.
43
+ The zstd compression library provides in-memory compression and decompression functions.
44
+ The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
42
45
  Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
43
46
  Compression can be done in:
44
47
  - a single step (described as Simple API)
45
48
  - a single step, reusing a context (described as Explicit memory management)
46
49
  - unbounded multiple steps (described as Streaming compression)
47
- The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
50
+ The compression ratio achievable on small data can be highly improved using a dictionary in:
48
51
  - a single step (described as Simple dictionary API)
49
52
  - a single step, reusing a dictionary (described as Fast dictionary API)
50
53
 
51
54
  Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
52
- These APIs shall never be used with a dynamic library.
55
+ Advanced experimental APIs shall never be used with a dynamic library.
53
56
  They are not "stable", their definition may change in the future. Only static linking is allowed.
54
57
  *********************************************************************************************************/
55
58
 
56
59
  /*------ Version ------*/
57
60
  #define ZSTD_VERSION_MAJOR 1
58
- #define ZSTD_VERSION_MINOR 2
61
+ #define ZSTD_VERSION_MINOR 3
59
62
  #define ZSTD_VERSION_RELEASE 0
60
63
 
64
+ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
65
+ ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */
66
+
61
67
  #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
62
68
  #define ZSTD_QUOTE(str) #str
63
69
  #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
64
70
  #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
65
-
66
- #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
67
- ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< library version number; to be used when checking dll version */
71
+ ZSTDLIB_API const char* ZSTD_versionString(void); /* v1.3.0 */
68
72
 
69
73
 
70
74
  /***************************************
@@ -81,38 +85,48 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
81
85
 
82
86
  /*! ZSTD_decompress() :
83
87
  * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
84
- * `dstCapacity` is an upper bound of originalSize.
88
+ * `dstCapacity` is an upper bound of originalSize to regenerate.
85
89
  * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
86
90
  * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
87
91
  * or an errorCode if it fails (which can be tested using ZSTD_isError()). */
88
92
  ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
89
93
  const void* src, size_t compressedSize);
90
94
 
91
- /*! ZSTD_getDecompressedSize() :
92
- * NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
93
- * ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
94
- * frame, but distinguishes empty frames from frames with an unknown size, or errors.
95
- *
96
- * Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple
97
- * concatenated frames in one buffer, and so is more general.
98
- * As a result however, it requires more computation and entire frames to be passed to it,
99
- * as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
100
- *
101
- * 'src' is the start of a zstd compressed frame.
102
- * @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
103
- * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
104
- * When `return==0`, data to decompress could be any size.
95
+ /*! ZSTD_getFrameContentSize() : v1.3.0
96
+ * `src` should point to the start of a ZSTD encoded frame.
97
+ * `srcSize` must be at least as large as the frame header.
98
+ * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
99
+ * @return : - decompressed size of the frame in `src`, if known
100
+ * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
101
+ * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
102
+ * note 1 : a 0 return value means the frame is valid but "empty".
103
+ * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
104
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
105
105
  * In which case, it's necessary to use streaming mode to decompress data.
106
- * Optionally, application can still use ZSTD_decompress() while relying on implied limits.
107
- * (For example, data may be necessarily cut into blocks <= 16 KB).
108
- * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
109
- * note 3 : decompressed size can be very large (64-bits value),
106
+ * Optionally, application can rely on some implicit limit,
107
+ * as ZSTD_decompress() only needs an upper bound of decompressed size.
108
+ * (For example, data could be necessarily cut into blocks <= 16 KB).
109
+ * note 3 : decompressed size is always present when compression is done with ZSTD_compress()
110
+ * note 4 : decompressed size can be very large (64-bits value),
110
111
  * potentially larger than what local system can handle as a single memory segment.
111
112
  * In which case, it's necessary to use streaming mode to decompress data.
112
- * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
113
- * Always ensure result fits within application's authorized limits.
113
+ * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
114
+ * Always ensure return value fits within application's authorized limits.
114
115
  * Each application can set its own limits.
115
- * note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
116
+ * note 6 : This function replaces ZSTD_getDecompressedSize() */
117
+ #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
118
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
119
+ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
120
+
121
+ /*! ZSTD_getDecompressedSize() :
122
+ * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
123
+ * Both functions work the same way,
124
+ * but ZSTD_getDecompressedSize() blends
125
+ * "empty", "unknown" and "error" results in the same return value (0),
126
+ * while ZSTD_getFrameContentSize() distinguishes them.
127
+ *
128
+ * 'src' is the start of a zstd compressed frame.
129
+ * @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. */
116
130
  ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
117
131
 
118
132
 
@@ -137,29 +151,35 @@ ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
137
151
 
138
152
  /*! ZSTD_compressCCtx() :
139
153
  * Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
140
- ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
154
+ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx,
155
+ void* dst, size_t dstCapacity,
156
+ const void* src, size_t srcSize,
157
+ int compressionLevel);
141
158
 
142
159
  /*= Decompression context
143
160
  * When decompressing many times,
144
- * it is recommended to allocate a context just once, and re-use it for each successive compression operation.
161
+ * it is recommended to allocate a context only once,
162
+ * and re-use it for each successive compression operation.
145
163
  * This will make workload friendlier for system's memory.
146
- * Use one context per thread for parallel execution in multi-threaded environments. */
164
+ * Use one context per thread for parallel execution. */
147
165
  typedef struct ZSTD_DCtx_s ZSTD_DCtx;
148
166
  ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
149
167
  ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
150
168
 
151
169
  /*! ZSTD_decompressDCtx() :
152
- * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */
153
- ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
170
+ * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
171
+ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
172
+ void* dst, size_t dstCapacity,
173
+ const void* src, size_t srcSize);
154
174
 
155
175
 
156
176
  /**************************
157
177
  * Simple dictionary API
158
178
  ***************************/
159
179
  /*! ZSTD_compress_usingDict() :
160
- * Compression using a predefined Dictionary (see dictBuilder/zdict.h).
161
- * Note : This function loads the dictionary, resulting in significant startup delay.
162
- * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
180
+ * Compression using a predefined Dictionary (see dictBuilder/zdict.h).
181
+ * Note : This function loads the dictionary, resulting in significant startup delay.
182
+ * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
163
183
  ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
164
184
  void* dst, size_t dstCapacity,
165
185
  const void* src, size_t srcSize,
@@ -167,30 +187,31 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
167
187
  int compressionLevel);
168
188
 
169
189
  /*! ZSTD_decompress_usingDict() :
170
- * Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
171
- * Dictionary must be identical to the one used during compression.
172
- * Note : This function loads the dictionary, resulting in significant startup delay.
173
- * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
190
+ * Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
191
+ * Dictionary must be identical to the one used during compression.
192
+ * Note : This function loads the dictionary, resulting in significant startup delay.
193
+ * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
174
194
  ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
175
195
  void* dst, size_t dstCapacity,
176
196
  const void* src, size_t srcSize,
177
197
  const void* dict,size_t dictSize);
178
198
 
179
199
 
180
- /****************************
181
- * Fast dictionary API
182
- ****************************/
200
+ /**********************************
201
+ * Bulk processing dictionary API
202
+ *********************************/
183
203
  typedef struct ZSTD_CDict_s ZSTD_CDict;
184
204
 
185
205
  /*! ZSTD_createCDict() :
186
- * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
187
- * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
188
- * ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
189
- * `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
190
- ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
206
+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
207
+ * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
208
+ * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
209
+ * `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict */
210
+ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
211
+ int compressionLevel);
191
212
 
192
213
  /*! ZSTD_freeCDict() :
193
- * Function frees memory allocated by ZSTD_createCDict(). */
214
+ * Function frees memory allocated by ZSTD_createCDict(). */
194
215
  ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
195
216
 
196
217
  /*! ZSTD_compress_usingCDict() :
@@ -207,17 +228,17 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
207
228
  typedef struct ZSTD_DDict_s ZSTD_DDict;
208
229
 
209
230
  /*! ZSTD_createDDict() :
210
- * Create a digested dictionary, ready to start decompression operation without startup delay.
211
- * dictBuffer can be released after DDict creation, as its content is copied inside DDict */
231
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
232
+ * dictBuffer can be released after DDict creation, as its content is copied inside DDict */
212
233
  ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
213
234
 
214
235
  /*! ZSTD_freeDDict() :
215
- * Function frees memory allocated with ZSTD_createDDict() */
236
+ * Function frees memory allocated with ZSTD_createDDict() */
216
237
  ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
217
238
 
218
239
  /*! ZSTD_decompress_usingDDict() :
219
- * Decompression using a digested Dictionary.
220
- * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
240
+ * Decompression using a digested Dictionary.
241
+ * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
221
242
  ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
222
243
  void* dst, size_t dstCapacity,
223
244
  const void* src, size_t srcSize,
@@ -274,14 +295,17 @@ typedef struct ZSTD_outBuffer_s {
274
295
  * ZSTD_endStream() instructs to finish a frame.
275
296
  * It will perform a flush and write frame epilogue.
276
297
  * The epilogue is required for decoders to consider a frame completed.
277
- * Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
298
+ * ZSTD_endStream() may not be able to flush full data if `output->size` is too small.
278
299
  * In which case, call again ZSTD_endStream() to complete the flush.
279
- * @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
300
+ * @return : 0 if frame fully completed and fully flushed,
301
+ or >0 if some data is still present within internal buffer
302
+ (value is minimum size estimation for remaining data to flush, but it could be more)
280
303
  * or an error code, which can be tested using ZSTD_isError().
281
304
  *
282
305
  * *******************************************************************/
283
306
 
284
- typedef struct ZSTD_CStream_s ZSTD_CStream;
307
+ typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
308
+ /* Continue to distinguish them for compatibility with versions <= v1.2.0 */
285
309
  /*===== ZSTD_CStream management functions =====*/
286
310
  ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
287
311
  ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
@@ -319,7 +343,8 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
319
343
  * The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
320
344
  * *******************************************************************************/
321
345
 
322
- typedef struct ZSTD_DStream_s ZSTD_DStream;
346
+ typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
347
+ /* Continue to distinguish them for compatibility with versions <= v1.2.0 */
323
348
  /*===== ZSTD_DStream management functions =====*/
324
349
  ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
325
350
  ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
@@ -334,23 +359,22 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
334
359
  #endif /* ZSTD_H_235446 */
335
360
 
336
361
 
337
- #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
338
- #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
339
362
 
340
363
  /****************************************************************************************
341
364
  * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
342
365
  * The definitions in this section are considered experimental.
343
- * They should never be used with a dynamic library, as they may change in the future.
344
- * They are provided for advanced usages.
366
+ * They should never be used with a dynamic library, as prototypes may change in the future.
367
+ * They are provided for advanced scenarios.
345
368
  * Use them only in association with static linking.
346
369
  * ***************************************************************************************/
347
370
 
371
+ #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
372
+ #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
373
+
348
374
  /* --- Constants ---*/
349
375
  #define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
350
376
  #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
351
-
352
- #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
353
- #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
377
+ #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
354
378
 
355
379
  #define ZSTD_WINDOWLOG_MAX_32 27
356
380
  #define ZSTD_WINDOWLOG_MAX_64 27
@@ -370,14 +394,15 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
370
394
 
371
395
  #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
372
396
  #define ZSTD_FRAMEHEADERSIZE_MIN 6
373
- static const size_t ZSTD_frameHeaderSize_prefix = 5;
374
- static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
397
+ static const size_t ZSTD_frameHeaderSize_prefix = 5; /* minimum input size to know frame header size */
375
398
  static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
399
+ static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
376
400
  static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
377
401
 
378
402
 
379
403
  /*--- Advanced types ---*/
380
- typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */
404
+ typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
405
+ ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */
381
406
 
382
407
  typedef struct {
383
408
  unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -400,76 +425,141 @@ typedef struct {
400
425
  ZSTD_frameParameters fParams;
401
426
  } ZSTD_parameters;
402
427
 
428
+ typedef struct {
429
+ unsigned long long frameContentSize;
430
+ size_t windowSize;
431
+ unsigned dictID;
432
+ unsigned checksumFlag;
433
+ } ZSTD_frameHeader;
434
+
403
435
  /*= Custom memory allocation functions */
404
436
  typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
405
437
  typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
406
438
  typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
439
+ /* use this constant to defer to stdlib's functions */
440
+ static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
441
+
407
442
 
408
443
  /***************************************
409
- * Compressed size functions
444
+ * Frame size functions
410
445
  ***************************************/
411
446
 
412
447
  /*! ZSTD_findFrameCompressedSize() :
413
448
  * `src` should point to the start of a ZSTD encoded frame or skippable frame
414
449
  * `srcSize` must be at least as large as the frame
415
- * @return : the compressed size of the frame pointed to by `src`, suitable to pass to
416
- * `ZSTD_decompress` or similar, or an error code if given invalid input. */
450
+ * @return : the compressed size of the first frame starting at `src`,
451
+ * suitable to pass to `ZSTD_decompress` or similar,
452
+ * or an error code if input is invalid */
417
453
  ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
418
454
 
455
+ /*! ZSTD_findDecompressedSize() :
456
+ * `src` should point the start of a series of ZSTD encoded and/or skippable frames
457
+ * `srcSize` must be the _exact_ size of this series
458
+ * (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`)
459
+ * @return : - decompressed size of all data in all successive frames
460
+ * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
461
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
462
+ *
463
+ * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
464
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
465
+ * In which case, it's necessary to use streaming mode to decompress data.
466
+ * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
467
+ * note 3 : decompressed size can be very large (64-bits value),
468
+ * potentially larger than what local system can handle as a single memory segment.
469
+ * In which case, it's necessary to use streaming mode to decompress data.
470
+ * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
471
+ * Always ensure result fits within application's authorized limits.
472
+ * Each application can set its own limits.
473
+ * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
474
+ * read each contained frame header. This is fast as most of the data is skipped,
475
+ * however it does mean that all frame data must be present and valid. */
476
+ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
477
+
478
+ /*! ZSTD_frameHeaderSize() :
479
+ * `src` should point to the start of a ZSTD frame
480
+ * `srcSize` must be >= ZSTD_frameHeaderSize_prefix.
481
+ * @return : size of the Frame Header */
482
+ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
483
+
484
+
419
485
  /***************************************
420
- * Decompressed size functions
486
+ * Context memory usage
421
487
  ***************************************/
422
- /*! ZSTD_getFrameContentSize() :
423
- * `src` should point to the start of a ZSTD encoded frame
424
- * `srcSize` must be at least as large as the frame header. A value greater than or equal
425
- * to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
426
- * @return : decompressed size of the frame pointed to be `src` if known, otherwise
427
- * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
428
- * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
429
- ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
430
488
 
431
- /*! ZSTD_findDecompressedSize() :
432
- * `src` should point the start of a series of ZSTD encoded and/or skippable frames
433
- * `srcSize` must be the _exact_ size of this series
434
- * (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
435
- * @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
436
- * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
437
- * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
438
- *
439
- * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
440
- * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
441
- * In which case, it's necessary to use streaming mode to decompress data.
442
- * Optionally, application can still use ZSTD_decompress() while relying on implied limits.
443
- * (For example, data may be necessarily cut into blocks <= 16 KB).
444
- * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
445
- * note 3 : decompressed size can be very large (64-bits value),
446
- * potentially larger than what local system can handle as a single memory segment.
447
- * In which case, it's necessary to use streaming mode to decompress data.
448
- * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
449
- * Always ensure result fits within application's authorized limits.
450
- * Each application can set its own limits.
451
- * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
452
- * read each contained frame header. This is efficient as most of the data is skipped,
453
- * however it does mean that all frame data must be present and valid. */
454
- ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
489
+ /*! ZSTD_sizeof_*() :
490
+ * These functions give the current memory usage of selected object.
491
+ * Object memory usage can evolve if it's re-used multiple times. */
492
+ ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
493
+ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
494
+ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
495
+ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
496
+ ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
497
+ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
498
+
499
+ /*! ZSTD_estimate*() :
500
+ * These functions make it possible to estimate memory usage
501
+ * of a future {D,C}Ctx, before its creation.
502
+ * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
503
+ * It will also consider src size to be arbitrarily "large", which is worst case.
504
+ * If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
505
+ * ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
506
+ * Note : CCtx estimation is only correct for single-threaded compression */
507
+ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
508
+ ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
509
+ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
510
+
511
+ /*! ZSTD_estimate?StreamSize() :
512
+ * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
513
+ * It will also consider src size to be arbitrarily "large", which is worst case.
514
+ * If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
515
+ * ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
516
+ * Note : CStream estimation is only correct for single-threaded compression.
517
+ * ZSTD_DStream memory budget depends on window Size.
518
+ * This information can be passed manually, using ZSTD_estimateDStreamSize,
519
+ * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
520
+ * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
521
+ * an internal ?Dict will be created, which additional size is not estimated here.
522
+ * In this case, get total size by adding ZSTD_estimate?DictSize */
523
+ ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
524
+ ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
525
+ ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
526
+ ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
527
+
528
+ /*! ZSTD_estimate?DictSize() :
529
+ * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
530
+ * ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
531
+ * Note : dictionary created "byReference" are smaller */
532
+ ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
533
+ ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
534
+ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
455
535
 
456
536
 
457
537
  /***************************************
458
538
  * Advanced compression functions
459
539
  ***************************************/
460
- /*! ZSTD_estimateCCtxSize() :
461
- * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
462
- * `frameContentSize` is an optional parameter, provide `0` if unknown */
463
- ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
464
-
465
540
  /*! ZSTD_createCCtx_advanced() :
466
541
  * Create a ZSTD compression context using external alloc and free functions */
467
542
  ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
468
543
 
469
- /*! ZSTD_sizeofCCtx() :
470
- * Gives the amount of memory used by a given ZSTD_CCtx */
471
- ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
544
+ /*! ZSTD_initStaticCCtx() : initialize a fixed-size zstd compression context
545
+ * workspace: The memory area to emplace the context into.
546
+ * Provided pointer must 8-bytes aligned.
547
+ * It must outlive context usage.
548
+ * workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
549
+ * to determine how large workspace must be to support scenario.
550
+ * @return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
551
+ * Note : zstd will never resize nor malloc() when using a static cctx.
552
+ * If it needs more memory than available, it will simply error out.
553
+ * Note 2 : there is no corresponding "free" function.
554
+ * Since workspace was allocated externally, it must be freed externally too.
555
+ * Limitation 1 : currently not compatible with internal CDict creation, such as
556
+ * ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
557
+ * Limitation 2 : currently not compatible with multi-threading
558
+ */
559
+ ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
560
+
472
561
 
562
+ /* !!! To be deprecated !!! */
473
563
  typedef enum {
474
564
  ZSTD_p_forceWindow, /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
475
565
  ZSTD_p_forceRawDict /* Force loading dictionary in "content-only" mode (no header analysis) */
@@ -479,20 +569,43 @@ typedef enum {
479
569
  * @result : 0, or an error code (which can be tested with ZSTD_isError()) */
480
570
  ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
481
571
 
572
+
482
573
  /*! ZSTD_createCDict_byReference() :
483
574
  * Create a digested dictionary for compression
484
575
  * Dictionary content is simply referenced, and therefore stays in dictBuffer.
485
576
  * It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
486
577
  ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
487
578
 
579
+
580
+ typedef enum { ZSTD_dm_auto=0, /* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
581
+ ZSTD_dm_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
582
+ ZSTD_dm_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */
583
+ } ZSTD_dictMode_e;
488
584
  /*! ZSTD_createCDict_advanced() :
489
585
  * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
490
- ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
491
- ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
492
-
493
- /*! ZSTD_sizeof_CDict() :
494
- * Gives the amount of memory used by a given ZSTD_sizeof_CDict */
495
- ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
586
+ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
587
+ unsigned byReference, ZSTD_dictMode_e dictMode,
588
+ ZSTD_compressionParameters cParams,
589
+ ZSTD_customMem customMem);
590
+
591
+ /*! ZSTD_initStaticCDict_advanced() :
592
+ * Generate a digested dictionary in provided memory area.
593
+ * workspace: The memory area to emplace the dictionary into.
594
+ * Provided pointer must 8-bytes aligned.
595
+ * It must outlive dictionary usage.
596
+ * workspaceSize: Use ZSTD_estimateCDictSize()
597
+ * to determine how large workspace must be.
598
+ * cParams : use ZSTD_getCParams() to transform a compression level
599
+ * into its relevants cParams.
600
+ * @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
601
+ * Note : there is no corresponding "free" function.
602
+ * Since workspace was allocated externally, it must be freed externally.
603
+ */
604
+ ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict(
605
+ void* workspace, size_t workspaceSize,
606
+ const void* dict, size_t dictSize,
607
+ unsigned byReference, ZSTD_dictMode_e dictMode,
608
+ ZSTD_compressionParameters cParams);
496
609
 
497
610
  /*! ZSTD_getCParams() :
498
611
  * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
@@ -509,8 +622,8 @@ ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long l
509
622
  ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
510
623
 
511
624
  /*! ZSTD_adjustCParams() :
512
- * optimize params for a given `srcSize` and `dictSize`.
513
- * both values are optional, select `0` if unknown. */
625
+ * optimize params for a given `srcSize` and `dictSize`.
626
+ * both values are optional, select `0` if unknown. */
514
627
  ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
515
628
 
516
629
  /*! ZSTD_compress_advanced() :
@@ -538,22 +651,32 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
538
651
  * Note 3 : Skippable Frame Identifiers are considered valid. */
539
652
  ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
540
653
 
541
- /*! ZSTD_estimateDCtxSize() :
542
- * Gives the potential amount of memory allocated to create a ZSTD_DCtx */
543
- ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
544
-
545
654
  /*! ZSTD_createDCtx_advanced() :
546
655
  * Create a ZSTD decompression context using external alloc and free functions */
547
656
  ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
548
657
 
549
- /*! ZSTD_sizeof_DCtx() :
550
- * Gives the amount of memory used by a given ZSTD_DCtx */
551
- ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
658
+ /*! ZSTD_initStaticDCtx() : initialize a fixed-size zstd decompression context
659
+ * workspace: The memory area to emplace the context into.
660
+ * Provided pointer must 8-bytes aligned.
661
+ * It must outlive context usage.
662
+ * workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
663
+ * to determine how large workspace must be to support scenario.
664
+ * @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
665
+ * Note : zstd will never resize nor malloc() when using a static dctx.
666
+ * If it needs more memory than available, it will simply error out.
667
+ * Note 2 : static dctx is incompatible with legacy support
668
+ * Note 3 : there is no corresponding "free" function.
669
+ * Since workspace was allocated externally, it must be freed externally.
670
+ * Limitation : currently not compatible with internal DDict creation,
671
+ * such as ZSTD_initDStream_usingDict().
672
+ */
673
+ ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
552
674
 
553
675
  /*! ZSTD_createDDict_byReference() :
554
676
  * Create a digested dictionary, ready to start decompression operation without startup delay.
555
- * Dictionary content is simply referenced, and therefore stays in dictBuffer.
556
- * It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
677
+ * Dictionary content is referenced, and therefore stays in dictBuffer.
678
+ * It is important that dictBuffer outlives DDict,
679
+ * it must remain read accessible throughout the lifetime of DDict */
557
680
  ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
558
681
 
559
682
  /*! ZSTD_createDDict_advanced() :
@@ -561,9 +684,20 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
561
684
  ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
562
685
  unsigned byReference, ZSTD_customMem customMem);
563
686
 
564
- /*! ZSTD_sizeof_DDict() :
565
- * Gives the amount of memory used by a given ZSTD_DDict */
566
- ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
687
+ /*! ZSTD_initStaticDDict() :
688
+ * Generate a digested dictionary in provided memory area.
689
+ * workspace: The memory area to emplace the dictionary into.
690
+ * Provided pointer must 8-bytes aligned.
691
+ * It must outlive dictionary usage.
692
+ * workspaceSize: Use ZSTD_estimateDDictSize()
693
+ * to determine how large workspace must be.
694
+ * @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
695
+ * Note : there is no corresponding "free" function.
696
+ * Since workspace was allocated externally, it must be freed externally.
697
+ */
698
+ ZSTDLIB_API ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
699
+ const void* dict, size_t dictSize,
700
+ unsigned byReference);
567
701
 
568
702
  /*! ZSTD_getDictID_fromDict() :
569
703
  * Provides the dictID stored within dictionary.
@@ -586,7 +720,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
586
720
  * Note : this use case also happens when using a non-conformant dictionary.
587
721
  * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
588
722
  * - This is not a Zstandard frame.
589
- * When identifying the exact failure cause, it's possible to use ZSTD_getFrameParams(), which will provide a more precise error code. */
723
+ * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
590
724
  ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
591
725
 
592
726
 
@@ -596,13 +730,13 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
596
730
 
597
731
  /*===== Advanced Streaming compression functions =====*/
598
732
  ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
599
- ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /**< size of CStream is variable, depending primarily on compression level */
733
+ ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
600
734
  ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
601
- ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
735
+ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
602
736
  ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
603
737
  ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
604
738
  ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
605
- ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
739
+ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
606
740
 
607
741
  /*! ZSTD_resetCStream() :
608
742
  * start a new compression job, using same parameters from previous job.
@@ -617,11 +751,11 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg
617
751
  /*===== Advanced Streaming decompression functions =====*/
618
752
  typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
619
753
  ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
620
- ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
754
+ ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
621
755
  ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
756
+ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
622
757
  ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
623
758
  ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
624
- ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
625
759
 
626
760
 
627
761
  /*********************************************************************
@@ -683,21 +817,24 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
683
817
  Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
684
818
  A ZSTD_DCtx object can be re-used multiple times.
685
819
 
686
- First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
687
- It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
688
- such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
689
- and the dictionary ID used.
820
+ First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
821
+ It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
822
+ such as minimum rolling buffer size to allocate to decompress data (`windowSize`),
823
+ and the dictionary ID in use.
690
824
  (Note : content size is optional, it may not be present. 0 means : content size unknown).
691
825
  Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
692
826
  As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
693
- Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
694
- Frame parameters are extracted from the beginning of the compressed frame.
695
- Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
696
- @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
827
+ Each application can set its own limit, depending on local restrictions.
828
+ For extended interoperability, it is recommended to support windowSize of at least 8 MB.
829
+ Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
830
+ Data fragment must be large enough to ensure successful decoding.
831
+ `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
832
+ @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
697
833
  >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
698
834
  errorCode, which can be tested using ZSTD_isError().
699
835
 
700
- Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
836
+ Start decompression, with ZSTD_decompressBegin().
837
+ If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
701
838
  Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
702
839
 
703
840
  Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
@@ -729,30 +866,233 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
729
866
  b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
730
867
  c) Frame Content - any content (User Data) of length equal to Frame Size
731
868
  For skippable frames ZSTD_decompressContinue() always returns 0.
732
- For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
869
+ For skippable frames ZSTD_getFrameHeader() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
733
870
  Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
734
871
  For purposes of decompression, it is valid in both cases to skip the frame using
735
872
  ZSTD_findFrameCompressedSize to find its size in bytes.
736
873
  It also returns Frame Size as fparamsPtr->frameContentSize.
737
874
  */
738
875
 
739
- typedef struct {
740
- unsigned long long frameContentSize;
741
- unsigned windowSize;
742
- unsigned dictID;
743
- unsigned checksumFlag;
744
- } ZSTD_frameParams;
745
-
746
876
  /*===== Buffer-less streaming decompression functions =====*/
747
- ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
877
+ ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
748
878
  ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
749
879
  ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
880
+ ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
750
881
  ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
882
+
751
883
  ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
752
884
  ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
753
885
  typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
754
886
  ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
755
887
 
888
+
889
+
890
+ /*=== New advanced API (experimental, and compression only) ===*/
891
+
892
+ /* notes on API design :
893
+ * In this proposal, parameters are pushed one by one into an existing CCtx,
894
+ * and then applied on all subsequent compression jobs.
895
+ * When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT.
896
+ *
897
+ * This API is intended to replace all others experimental API.
898
+ * It can basically do all other use cases, and even new ones.
899
+ * It stands a good chance to become "stable",
900
+ * after a reasonable testing period.
901
+ */
902
+
903
+ /* note on naming convention :
904
+ * Initially, the API favored names like ZSTD_setCCtxParameter() .
905
+ * In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
906
+ * The main driver is that it identifies more clearly the target object type.
907
+ * It feels clearer in light of potential variants :
908
+ * ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
909
+ * ZSTD_DCtx_setParameter() (rather than ZSTD_setDCtxParameter() )
910
+ * Left variant feels easier to distinguish.
911
+ */
912
+
913
+ /* note on enum design :
914
+ * All enum will be manually set to explicit values before reaching "stable API" status */
915
+
916
+ typedef enum {
917
+ /* compression parameters */
918
+ ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
919
+ * Default level is ZSTD_CLEVEL_DEFAULT==3.
920
+ * Special: value 0 means "do not change cLevel". */
921
+ ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
922
+ * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
923
+ * Special: value 0 means "do not change windowLog". */
924
+ ZSTD_p_hashLog, /* Size of the probe table, as a power of 2.
925
+ * Resulting table size is (1 << (hashLog+2)).
926
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
927
+ * Larger tables improve compression ratio of strategies <= dFast,
928
+ * and improve speed of strategies > dFast.
929
+ * Special: value 0 means "do not change hashLog". */
930
+ ZSTD_p_chainLog, /* Size of the full-search table, as a power of 2.
931
+ * Resulting table size is (1 << (chainLog+2)).
932
+ * Larger tables result in better and slower compression.
933
+ * This parameter is useless when using "fast" strategy.
934
+ * Special: value 0 means "do not change chainLog". */
935
+ ZSTD_p_searchLog, /* Number of search attempts, as a power of 2.
936
+ * More attempts result in better and slower compression.
937
+ * This parameter is useless when using "fast" and "dFast" strategies.
938
+ * Special: value 0 means "do not change searchLog". */
939
+ ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller).
940
+ * Larger values make faster compression and decompression, but decrease ratio.
941
+ * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
942
+ * Note that currently, for all strategies < btopt, effective minimum is 4.
943
+ * Note that currently, for all strategies > fast, effective maximum is 6.
944
+ * Special: value 0 means "do not change minMatchLength". */
945
+ ZSTD_p_targetLength, /* Only useful for strategies >= btopt.
946
+ * Length of Match considered "good enough" to stop search.
947
+ * Larger values make compression stronger and slower.
948
+ * Special: value 0 means "do not change targetLength". */
949
+ ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
950
+ * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
951
+ * The higher the value of selected strategy, the more complex it is,
952
+ * resulting in stronger and slower compression.
953
+ * Special: value 0 means "do not change strategy". */
954
+
955
+ /* frame parameters */
956
+ ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
957
+ ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
958
+ ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */
959
+
960
+ /* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */
961
+ ZSTD_p_dictMode=300, /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
962
+ * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
963
+ ZSTD_p_refDictContent, /* Dictionary content will be referenced, instead of copied (default:0==byCopy).
964
+ * It requires that dictionary buffer outlives its users */
965
+
966
+ /* multi-threading parameters */
967
+ ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
968
+ * More threads improve speed, but also increase memory usage.
969
+ * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
970
+ * Special: value 0 means "do not change nbThreads" */
971
+ ZSTD_p_jobSize, /* Size of a compression job. Each compression job is completed in parallel.
972
+ * 0 means default, which is dynamically determined based on compression parameters.
973
+ * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
974
+ * The minimum size is automatically and transparently enforced */
975
+ ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
976
+ * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
977
+
978
+ /* advanced parameters - may not remain available after API update */
979
+ ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
980
+ * even when referencing into Dictionary content (default:0) */
981
+
982
+ } ZSTD_cParameter;
983
+
984
+
985
+ /*! ZSTD_CCtx_setParameter() :
986
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
987
+ * Note : when `value` is an enum, cast it to unsigned for proper type checking.
988
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()). */
989
+ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
990
+
991
+ /*! ZSTD_CCtx_setPledgedSrcSize() :
992
+ * Total input data size to be compressed as a single frame.
993
+ * This value will be controlled at the end, and result in error if not respected.
994
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
995
+ * Note 1 : 0 means zero, empty.
996
+ * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
997
+ * Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
998
+ * Note 2 : If all data is provided and consumed in a single round,
999
+ * this value is overriden by srcSize instead. */
1000
+ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
1001
+
1002
+ /*! ZSTD_CCtx_loadDictionary() :
1003
+ * Create an internal CDict from dict buffer.
1004
+ * Decompression will have to use same buffer.
1005
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1006
+ * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
1007
+ * meaning "return to no-dictionary mode".
1008
+ * Note 1 : `dict` content will be copied internally,
1009
+ * except if ZSTD_p_refDictContent is set before loading.
1010
+ * Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
1011
+ * For this reason, compression parameters cannot be changed anymore after loading a dictionary.
1012
+ * It's also a CPU-heavy operation, with non-negligible impact on latency.
1013
+ * Note 3 : Dictionary will be used for all future compression jobs.
1014
+ * To return to "no-dictionary" situation, load a NULL dictionary */
1015
+ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
1016
+
1017
+ /*! ZSTD_CCtx_refCDict() :
1018
+ * Reference a prepared dictionary, to be used for all next compression jobs.
1019
+ * Note that compression parameters are enforced from within CDict,
1020
+ * and supercede any compression parameter previously set within CCtx.
1021
+ * The dictionary will remain valid for future compression jobs using same CCtx.
1022
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1023
+ * Special : adding a NULL CDict means "return to no-dictionary mode".
1024
+ * Note 1 : Currently, only one dictionary can be managed.
1025
+ * Adding a new dictionary effectively "discards" any previous one.
1026
+ * Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
1027
+ */
1028
+ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
1029
+
1030
+ /*! ZSTD_CCtx_refPrefix() :
1031
+ * Reference a prefix (single-usage dictionary) for next compression job.
1032
+ * Decompression need same prefix to properly regenerate data.
1033
+ * Prefix is **only used once**. Tables are discarded at end of compression job.
1034
+ * Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
1035
+ * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
1036
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1037
+ * Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
1038
+ * Note 1 : Prefix buffer is referenced. It must outlive compression job.
1039
+ * Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
1040
+ * It's a CPU-heavy operation, with non-negligible impact on latency.
1041
+ * Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter() */
1042
+ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
1043
+
1044
+
1045
+
1046
+ typedef enum {
1047
+ ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
1048
+ ZSTD_e_flush, /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
1049
+ ZSTD_e_end /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
1050
+ } ZSTD_EndDirective;
1051
+
1052
+ /*! ZSTD_compress_generic() :
1053
+ * Behave about the same as ZSTD_compressStream. To note :
1054
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
1055
+ * - Compression parameters cannot be changed once compression is started.
1056
+ * - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
1057
+ * - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
1058
+ * - @return provides the minimum amount of data still to flush from internal buffers
1059
+ * or an error code, which can be tested using ZSTD_isError().
1060
+ * if @return != 0, flush is not fully completed, there is some data left within internal buffers.
1061
+ * - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
1062
+ * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
1063
+ * It is necessary to fully flush internal buffers
1064
+ * before starting a new compression job, or changing compression parameters.
1065
+ */
1066
+ ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
1067
+ ZSTD_outBuffer* output,
1068
+ ZSTD_inBuffer* input,
1069
+ ZSTD_EndDirective endOp);
1070
+
1071
+ /*! ZSTD_CCtx_reset() :
1072
+ * Return a CCtx to clean state.
1073
+ * Useful after an error, or to interrupt an ongoing compression job and start a new one.
1074
+ * Any internal data not yet flushed is cancelled.
1075
+ * Dictionary (if any) is dropped.
1076
+ * It's possible to modify compression parameters after a reset.
1077
+ */
1078
+ ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); /* Not ready yet ! */
1079
+
1080
+
1081
+ /*! ZSTD_compress_generic_simpleArgs() :
1082
+ * Same as ZSTD_compress_generic(),
1083
+ * but using only integral types as arguments.
1084
+ * Argument list is larger and less expressive than ZSTD_{in,out}Buffer,
1085
+ * but can be helpful for binders from dynamic languages
1086
+ * which have troubles handling structures containing memory pointers.
1087
+ */
1088
+ size_t ZSTD_compress_generic_simpleArgs (
1089
+ ZSTD_CCtx* cctx,
1090
+ void* dst, size_t dstCapacity, size_t* dstPos,
1091
+ const void* src, size_t srcSize, size_t* srcPos,
1092
+ ZSTD_EndDirective endOp);
1093
+
1094
+
1095
+
756
1096
  /**
757
1097
  Block functions
758
1098
 
@@ -767,7 +1107,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
767
1107
  + compression : any ZSTD_compressBegin*() variant, including with dictionary
768
1108
  + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
769
1109
  + copyCCtx() and copyDCtx() can be used too
770
- - Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX
1110
+ - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
771
1111
  + If input is larger than a block size, it's necessary to split input data into multiple blocks
772
1112
  + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
773
1113
  Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
@@ -780,9 +1120,10 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
780
1120
  Use ZSTD_insertBlock() for such a case.
781
1121
  */
782
1122
 
783
- #define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
1123
+ #define ZSTD_BLOCKSIZELOG_MAX 17
1124
+ #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */
784
1125
  /*===== Raw zstd block functions =====*/
785
- ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
1126
+ ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
786
1127
  ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
787
1128
  ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
788
1129
  ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */