zstd-ruby 1.2.0.0 → 1.3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/ext/zstdruby/libzstd/Makefile +7 -5
  4. data/ext/zstdruby/libzstd/common/bitstream.h +23 -9
  5. data/ext/zstdruby/libzstd/common/error_private.c +4 -1
  6. data/ext/zstdruby/libzstd/common/huf.h +20 -0
  7. data/ext/zstdruby/libzstd/common/mem.h +0 -14
  8. data/ext/zstdruby/libzstd/common/pool.c +12 -0
  9. data/ext/zstdruby/libzstd/common/pool.h +5 -0
  10. data/ext/zstdruby/libzstd/common/threading.c +0 -1
  11. data/ext/zstdruby/libzstd/common/zstd_common.c +25 -18
  12. data/ext/zstdruby/libzstd/common/zstd_errors.h +15 -7
  13. data/ext/zstdruby/libzstd/common/zstd_internal.h +59 -9
  14. data/ext/zstdruby/libzstd/compress/huf_compress.c +7 -3
  15. data/ext/zstdruby/libzstd/compress/zstd_compress.c +1082 -487
  16. data/ext/zstdruby/libzstd/compress/zstd_opt.h +30 -15
  17. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +362 -158
  18. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +49 -13
  19. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +150 -26
  20. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +380 -258
  21. data/ext/zstdruby/libzstd/dictBuilder/cover.c +23 -37
  22. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +30 -40
  23. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +104 -95
  24. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +11 -10
  25. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +14 -19
  26. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +13 -12
  27. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +13 -14
  28. data/ext/zstdruby/libzstd/zstd.h +507 -166
  29. data/lib/zstd-ruby/version.rb +1 -1
  30. metadata +2 -2
@@ -816,13 +816,13 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
816
816
  bitD->bitContainer = *(const BYTE*)(bitD->start);
817
817
  switch(srcSize)
818
818
  {
819
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
820
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
821
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
822
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
823
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
824
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
825
- default:;
819
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);/* fall-through */
820
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);/* fall-through */
821
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);/* fall-through */
822
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; /* fall-through */
823
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; /* fall-through */
824
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; /* fall-through */
825
+ default: break;
826
826
  }
827
827
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
828
828
  if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
@@ -3665,7 +3665,7 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
3665
3665
  break;
3666
3666
  }
3667
3667
  zbc->stage = ZBUFFds_read;
3668
-
3668
+ /* fall-through */
3669
3669
  case ZBUFFds_read:
3670
3670
  {
3671
3671
  size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
@@ -3691,7 +3691,7 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
3691
3691
  if (ip==iend) { notDone = 0; break; } /* no more input */
3692
3692
  zbc->stage = ZBUFFds_load;
3693
3693
  }
3694
-
3694
+ /* fall-through */
3695
3695
  case ZBUFFds_load:
3696
3696
  {
3697
3697
  size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
@@ -3711,9 +3711,10 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
3711
3711
  if (!decodedSize) { zbc->stage = ZBUFFds_read; break; } /* this was just a header */
3712
3712
  zbc->outEnd = zbc->outStart + decodedSize;
3713
3713
  zbc->stage = ZBUFFds_flush;
3714
- // break; /* ZBUFFds_flush follows */
3714
+ /* ZBUFFds_flush follows */
3715
3715
  }
3716
3716
  }
3717
+ /* fall-through */
3717
3718
  case ZBUFFds_flush:
3718
3719
  {
3719
3720
  size_t toFlushSize = zbc->outEnd - zbc->outStart;
@@ -326,13 +326,6 @@ size_t ZSTDv05_decompress_usingPreparedDCtx(
326
326
  * Streaming functions (direct mode)
327
327
  ****************************************/
328
328
  size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx);
329
- size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize);
330
- void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dctx, const ZSTDv05_DCtx* preparedDCtx);
331
-
332
- size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize);
333
-
334
- size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx);
335
- size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
336
329
 
337
330
  /*
338
331
  Streaming decompression, direct mode (bufferless)
@@ -818,13 +811,13 @@ MEM_STATIC size_t BITv05_initDStream(BITv05_DStream_t* bitD, const void* srcBuff
818
811
  bitD->bitContainer = *(const BYTE*)(bitD->start);
819
812
  switch(srcSize)
820
813
  {
821
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
822
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
823
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
824
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
825
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
826
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
827
- default:;
814
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);/* fall-through */
815
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);/* fall-through */
816
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);/* fall-through */
817
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; /* fall-through */
818
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; /* fall-through */
819
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; /* fall-through */
820
+ default: break;
828
821
  }
829
822
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
830
823
  if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
@@ -3955,7 +3948,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3955
3948
  zbc->stage = ZBUFFv05ds_decodeHeader;
3956
3949
  break;
3957
3950
  }
3958
-
3951
+ /* fall-through */
3959
3952
  case ZBUFFv05ds_loadHeader:
3960
3953
  /* complete header from src */
3961
3954
  {
@@ -3973,7 +3966,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3973
3966
  }
3974
3967
  // zbc->stage = ZBUFFv05ds_decodeHeader; break; /* useless : stage follows */
3975
3968
  }
3976
-
3969
+ /* fall-through */
3977
3970
  case ZBUFFv05ds_decodeHeader:
3978
3971
  /* apply header to create / resize buffers */
3979
3972
  {
@@ -4000,7 +3993,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
4000
3993
  break;
4001
3994
  }
4002
3995
  zbc->stage = ZBUFFv05ds_read;
4003
-
3996
+ /* fall-through */
4004
3997
  case ZBUFFv05ds_read:
4005
3998
  {
4006
3999
  size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
@@ -4024,7 +4017,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
4024
4017
  if (ip==iend) { notDone = 0; break; } /* no more input */
4025
4018
  zbc->stage = ZBUFFv05ds_load;
4026
4019
  }
4027
-
4020
+ /* fall-through */
4028
4021
  case ZBUFFv05ds_load:
4029
4022
  {
4030
4023
  size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
@@ -4045,7 +4038,9 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
4045
4038
  zbc->outEnd = zbc->outStart + decodedSize;
4046
4039
  zbc->stage = ZBUFFv05ds_flush;
4047
4040
  // break; /* ZBUFFv05ds_flush follows */
4048
- } }
4041
+ }
4042
+ }
4043
+ /* fall-through */
4049
4044
  case ZBUFFv05ds_flush:
4050
4045
  {
4051
4046
  size_t toFlushSize = zbc->outEnd - zbc->outStart;
@@ -910,13 +910,13 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
910
910
  bitD->bitContainer = *(const BYTE*)(bitD->start);
911
911
  switch(srcSize)
912
912
  {
913
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
914
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
915
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
916
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
917
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
918
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
919
- default:;
913
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);/* fall-through */
914
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);/* fall-through */
915
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);/* fall-through */
916
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; /* fall-through */
917
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; /* fall-through */
918
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; /* fall-through */
919
+ default: break;
920
920
  }
921
921
  { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
922
922
  if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
@@ -3789,7 +3789,7 @@ size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapac
3789
3789
  return 0;
3790
3790
  }
3791
3791
  dctx->expected = 0; /* not necessary to copy more */
3792
-
3792
+ /* fall-through */
3793
3793
  case ZSTDds_decodeFrameHeader:
3794
3794
  { size_t result;
3795
3795
  memcpy(dctx->headerBuffer + ZSTDv06_frameHeaderSize_min, src, dctx->expected);
@@ -4116,7 +4116,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4116
4116
  if (zbd->outBuff == NULL) return ERROR(memory_allocation);
4117
4117
  } } }
4118
4118
  zbd->stage = ZBUFFds_read;
4119
-
4119
+ /* fall-through */
4120
4120
  case ZBUFFds_read:
4121
4121
  { size_t const neededInSize = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
4122
4122
  if (neededInSize==0) { /* end of frame */
@@ -4138,7 +4138,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4138
4138
  if (ip==iend) { notDone = 0; break; } /* no more input */
4139
4139
  zbd->stage = ZBUFFds_load;
4140
4140
  }
4141
-
4141
+ /* fall-through */
4142
4142
  case ZBUFFds_load:
4143
4143
  { size_t const neededInSize = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
4144
4144
  size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
@@ -4159,8 +4159,9 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4159
4159
  zbd->outEnd = zbd->outStart + decodedSize;
4160
4160
  zbd->stage = ZBUFFds_flush;
4161
4161
  // break; /* ZBUFFds_flush follows */
4162
- } }
4163
-
4162
+ }
4163
+ }
4164
+ /* fall-through */
4164
4165
  case ZBUFFds_flush:
4165
4166
  { size_t const toFlushSize = zbd->outEnd - zbd->outStart;
4166
4167
  size_t const flushedSize = ZBUFFv06_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
@@ -580,13 +580,13 @@ MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuff
580
580
  bitD->bitContainer = *(const BYTE*)(bitD->start);
581
581
  switch(srcSize)
582
582
  {
583
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
584
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
585
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
586
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
587
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
588
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
589
- default:;
583
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);/* fall-through */
584
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);/* fall-through */
585
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);/* fall-through */
586
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; /* fall-through */
587
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; /* fall-through */
588
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; /* fall-through */
589
+ default: break;
590
590
  }
591
591
  { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
592
592
  bitD->bitsConsumed = lastByte ? 8 - BITv07_highbit32(lastByte) : 0;
@@ -2920,8 +2920,6 @@ typedef struct {
2920
2920
  void ZSTDv07_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
2921
2921
 
2922
2922
  /* custom memory allocation functions */
2923
- void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size);
2924
- void ZSTDv07_defaultFreeFunction(void* opaque, void* address);
2925
2923
  static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction, ZSTDv07_defaultFreeFunction, NULL };
2926
2924
 
2927
2925
  #endif /* ZSTDv07_CCOMMON_H_MODULE */
@@ -4047,7 +4045,7 @@ size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapac
4047
4045
  return 0;
4048
4046
  }
4049
4047
  dctx->expected = 0; /* not necessary to copy more */
4050
-
4048
+ /* fall-through */
4051
4049
  case ZSTDds_decodeFrameHeader:
4052
4050
  { size_t result;
4053
4051
  memcpy(dctx->headerBuffer + ZSTDv07_frameHeaderSize_min, src, dctx->expected);
@@ -4494,7 +4492,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
4494
4492
  } } }
4495
4493
  zbd->stage = ZBUFFds_read;
4496
4494
  /* pass-through */
4497
-
4495
+ /* fall-through */
4498
4496
  case ZBUFFds_read:
4499
4497
  { size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);
4500
4498
  if (neededInSize==0) { /* end of frame */
@@ -4517,7 +4515,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
4517
4515
  if (ip==iend) { notDone = 0; break; } /* no more input */
4518
4516
  zbd->stage = ZBUFFds_load;
4519
4517
  }
4520
-
4518
+ /* fall-through */
4521
4519
  case ZBUFFds_load:
4522
4520
  { size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);
4523
4521
  size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
@@ -4540,8 +4538,9 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
4540
4538
  zbd->stage = ZBUFFds_flush;
4541
4539
  /* break; */
4542
4540
  /* pass-through */
4543
- } }
4544
-
4541
+ }
4542
+ }
4543
+ /* fall-through */
4545
4544
  case ZBUFFds_flush:
4546
4545
  { size_t const toFlushSize = zbd->outEnd - zbd->outStart;
4547
4546
  size_t const flushedSize = ZBUFFv07_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
@@ -19,10 +19,12 @@ extern "C" {
19
19
 
20
20
 
21
21
  /* ===== ZSTDLIB_API : control library symbols visibility ===== */
22
- #if defined(__GNUC__) && (__GNUC__ >= 4)
23
- # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
24
- #else
25
- # define ZSTDLIB_VISIBILITY
22
+ #ifndef ZSTDLIB_VISIBILITY
23
+ # if defined(__GNUC__) && (__GNUC__ >= 4)
24
+ # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
25
+ # else
26
+ # define ZSTDLIB_VISIBILITY
27
+ # endif
26
28
  #endif
27
29
  #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
28
30
  # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
@@ -36,35 +38,37 @@ extern "C" {
36
38
  /*******************************************************************************************************
37
39
  Introduction
38
40
 
39
- zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
40
- at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
41
- decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
41
+ zstd, short for Zstandard, is a fast lossless compression algorithm,
42
+ targeting real-time compression scenarios at zlib-level and better compression ratios.
43
+ The zstd compression library provides in-memory compression and decompression functions.
44
+ The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
42
45
  Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
43
46
  Compression can be done in:
44
47
  - a single step (described as Simple API)
45
48
  - a single step, reusing a context (described as Explicit memory management)
46
49
  - unbounded multiple steps (described as Streaming compression)
47
- The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
50
+ The compression ratio achievable on small data can be highly improved using a dictionary in:
48
51
  - a single step (described as Simple dictionary API)
49
52
  - a single step, reusing a dictionary (described as Fast dictionary API)
50
53
 
51
54
  Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
52
- These APIs shall never be used with a dynamic library.
55
+ Advanced experimental APIs shall never be used with a dynamic library.
53
56
  They are not "stable", their definition may change in the future. Only static linking is allowed.
54
57
  *********************************************************************************************************/
55
58
 
56
59
  /*------ Version ------*/
57
60
  #define ZSTD_VERSION_MAJOR 1
58
- #define ZSTD_VERSION_MINOR 2
61
+ #define ZSTD_VERSION_MINOR 3
59
62
  #define ZSTD_VERSION_RELEASE 0
60
63
 
64
+ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
65
+ ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */
66
+
61
67
  #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
62
68
  #define ZSTD_QUOTE(str) #str
63
69
  #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
64
70
  #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
65
-
66
- #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
67
- ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< library version number; to be used when checking dll version */
71
+ ZSTDLIB_API const char* ZSTD_versionString(void); /* v1.3.0 */
68
72
 
69
73
 
70
74
  /***************************************
@@ -81,38 +85,48 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
81
85
 
82
86
  /*! ZSTD_decompress() :
83
87
  * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
84
- * `dstCapacity` is an upper bound of originalSize.
88
+ * `dstCapacity` is an upper bound of originalSize to regenerate.
85
89
  * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
86
90
  * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
87
91
  * or an errorCode if it fails (which can be tested using ZSTD_isError()). */
88
92
  ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
89
93
  const void* src, size_t compressedSize);
90
94
 
91
- /*! ZSTD_getDecompressedSize() :
92
- * NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
93
- * ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
94
- * frame, but distinguishes empty frames from frames with an unknown size, or errors.
95
- *
96
- * Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple
97
- * concatenated frames in one buffer, and so is more general.
98
- * As a result however, it requires more computation and entire frames to be passed to it,
99
- * as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
100
- *
101
- * 'src' is the start of a zstd compressed frame.
102
- * @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
103
- * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
104
- * When `return==0`, data to decompress could be any size.
95
+ /*! ZSTD_getFrameContentSize() : v1.3.0
96
+ * `src` should point to the start of a ZSTD encoded frame.
97
+ * `srcSize` must be at least as large as the frame header.
98
+ * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
99
+ * @return : - decompressed size of the frame in `src`, if known
100
+ * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
101
+ * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
102
+ * note 1 : a 0 return value means the frame is valid but "empty".
103
+ * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
104
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
105
105
  * In which case, it's necessary to use streaming mode to decompress data.
106
- * Optionally, application can still use ZSTD_decompress() while relying on implied limits.
107
- * (For example, data may be necessarily cut into blocks <= 16 KB).
108
- * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
109
- * note 3 : decompressed size can be very large (64-bits value),
106
+ * Optionally, application can rely on some implicit limit,
107
+ * as ZSTD_decompress() only needs an upper bound of decompressed size.
108
+ * (For example, data could be necessarily cut into blocks <= 16 KB).
109
+ * note 3 : decompressed size is always present when compression is done with ZSTD_compress()
110
+ * note 4 : decompressed size can be very large (64-bits value),
110
111
  * potentially larger than what local system can handle as a single memory segment.
111
112
  * In which case, it's necessary to use streaming mode to decompress data.
112
- * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
113
- * Always ensure result fits within application's authorized limits.
113
+ * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
114
+ * Always ensure return value fits within application's authorized limits.
114
115
  * Each application can set its own limits.
115
- * note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
116
+ * note 6 : This function replaces ZSTD_getDecompressedSize() */
117
+ #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
118
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
119
+ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
120
+
121
+ /*! ZSTD_getDecompressedSize() :
122
+ * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
123
+ * Both functions work the same way,
124
+ * but ZSTD_getDecompressedSize() blends
125
+ * "empty", "unknown" and "error" results in the same return value (0),
126
+ * while ZSTD_getFrameContentSize() distinguishes them.
127
+ *
128
+ * 'src' is the start of a zstd compressed frame.
129
+ * @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. */
116
130
  ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
117
131
 
118
132
 
@@ -137,29 +151,35 @@ ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
137
151
 
138
152
  /*! ZSTD_compressCCtx() :
139
153
  * Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
140
- ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
154
+ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx,
155
+ void* dst, size_t dstCapacity,
156
+ const void* src, size_t srcSize,
157
+ int compressionLevel);
141
158
 
142
159
  /*= Decompression context
143
160
  * When decompressing many times,
144
- * it is recommended to allocate a context just once, and re-use it for each successive compression operation.
161
+ * it is recommended to allocate a context only once,
162
+ * and re-use it for each successive compression operation.
145
163
  * This will make workload friendlier for system's memory.
146
- * Use one context per thread for parallel execution in multi-threaded environments. */
164
+ * Use one context per thread for parallel execution. */
147
165
  typedef struct ZSTD_DCtx_s ZSTD_DCtx;
148
166
  ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
149
167
  ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
150
168
 
151
169
  /*! ZSTD_decompressDCtx() :
152
- * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */
153
- ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
170
+ * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
171
+ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
172
+ void* dst, size_t dstCapacity,
173
+ const void* src, size_t srcSize);
154
174
 
155
175
 
156
176
  /**************************
157
177
  * Simple dictionary API
158
178
  ***************************/
159
179
  /*! ZSTD_compress_usingDict() :
160
- * Compression using a predefined Dictionary (see dictBuilder/zdict.h).
161
- * Note : This function loads the dictionary, resulting in significant startup delay.
162
- * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
180
+ * Compression using a predefined Dictionary (see dictBuilder/zdict.h).
181
+ * Note : This function loads the dictionary, resulting in significant startup delay.
182
+ * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
163
183
  ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
164
184
  void* dst, size_t dstCapacity,
165
185
  const void* src, size_t srcSize,
@@ -167,30 +187,31 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
167
187
  int compressionLevel);
168
188
 
169
189
  /*! ZSTD_decompress_usingDict() :
170
- * Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
171
- * Dictionary must be identical to the one used during compression.
172
- * Note : This function loads the dictionary, resulting in significant startup delay.
173
- * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
190
+ * Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
191
+ * Dictionary must be identical to the one used during compression.
192
+ * Note : This function loads the dictionary, resulting in significant startup delay.
193
+ * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
174
194
  ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
175
195
  void* dst, size_t dstCapacity,
176
196
  const void* src, size_t srcSize,
177
197
  const void* dict,size_t dictSize);
178
198
 
179
199
 
180
- /****************************
181
- * Fast dictionary API
182
- ****************************/
200
+ /**********************************
201
+ * Bulk processing dictionary API
202
+ *********************************/
183
203
  typedef struct ZSTD_CDict_s ZSTD_CDict;
184
204
 
185
205
  /*! ZSTD_createCDict() :
186
- * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
187
- * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
188
- * ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
189
- * `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
190
- ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
206
+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
207
+ * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
208
+ * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
209
+ * `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict */
210
+ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
211
+ int compressionLevel);
191
212
 
192
213
  /*! ZSTD_freeCDict() :
193
- * Function frees memory allocated by ZSTD_createCDict(). */
214
+ * Function frees memory allocated by ZSTD_createCDict(). */
194
215
  ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
195
216
 
196
217
  /*! ZSTD_compress_usingCDict() :
@@ -207,17 +228,17 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
207
228
  typedef struct ZSTD_DDict_s ZSTD_DDict;
208
229
 
209
230
  /*! ZSTD_createDDict() :
210
- * Create a digested dictionary, ready to start decompression operation without startup delay.
211
- * dictBuffer can be released after DDict creation, as its content is copied inside DDict */
231
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
232
+ * dictBuffer can be released after DDict creation, as its content is copied inside DDict */
212
233
  ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
213
234
 
214
235
  /*! ZSTD_freeDDict() :
215
- * Function frees memory allocated with ZSTD_createDDict() */
236
+ * Function frees memory allocated with ZSTD_createDDict() */
216
237
  ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
217
238
 
218
239
  /*! ZSTD_decompress_usingDDict() :
219
- * Decompression using a digested Dictionary.
220
- * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
240
+ * Decompression using a digested Dictionary.
241
+ * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
221
242
  ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
222
243
  void* dst, size_t dstCapacity,
223
244
  const void* src, size_t srcSize,
@@ -274,14 +295,17 @@ typedef struct ZSTD_outBuffer_s {
274
295
  * ZSTD_endStream() instructs to finish a frame.
275
296
  * It will perform a flush and write frame epilogue.
276
297
  * The epilogue is required for decoders to consider a frame completed.
277
- * Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
298
+ * ZSTD_endStream() may not be able to flush full data if `output->size` is too small.
278
299
  * In which case, call again ZSTD_endStream() to complete the flush.
279
- * @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
300
+ * @return : 0 if frame fully completed and fully flushed,
301
+ or >0 if some data is still present within internal buffer
302
+ (value is minimum size estimation for remaining data to flush, but it could be more)
280
303
  * or an error code, which can be tested using ZSTD_isError().
281
304
  *
282
305
  * *******************************************************************/
283
306
 
284
- typedef struct ZSTD_CStream_s ZSTD_CStream;
307
+ typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
308
+ /* Continue to distinguish them for compatibility with versions <= v1.2.0 */
285
309
  /*===== ZSTD_CStream management functions =====*/
286
310
  ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
287
311
  ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
@@ -319,7 +343,8 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
319
343
  * The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
320
344
  * *******************************************************************************/
321
345
 
322
- typedef struct ZSTD_DStream_s ZSTD_DStream;
346
+ typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
347
+ /* Continue to distinguish them for compatibility with versions <= v1.2.0 */
323
348
  /*===== ZSTD_DStream management functions =====*/
324
349
  ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
325
350
  ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
@@ -334,23 +359,22 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
334
359
  #endif /* ZSTD_H_235446 */
335
360
 
336
361
 
337
- #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
338
- #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
339
362
 
340
363
  /****************************************************************************************
341
364
  * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
342
365
  * The definitions in this section are considered experimental.
343
- * They should never be used with a dynamic library, as they may change in the future.
344
- * They are provided for advanced usages.
366
+ * They should never be used with a dynamic library, as prototypes may change in the future.
367
+ * They are provided for advanced scenarios.
345
368
  * Use them only in association with static linking.
346
369
  * ***************************************************************************************/
347
370
 
371
+ #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
372
+ #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
373
+
348
374
  /* --- Constants ---*/
349
375
  #define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
350
376
  #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
351
-
352
- #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
353
- #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
377
+ #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
354
378
 
355
379
  #define ZSTD_WINDOWLOG_MAX_32 27
356
380
  #define ZSTD_WINDOWLOG_MAX_64 27
@@ -370,14 +394,15 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
370
394
 
371
395
  #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
372
396
  #define ZSTD_FRAMEHEADERSIZE_MIN 6
373
- static const size_t ZSTD_frameHeaderSize_prefix = 5;
374
- static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
397
+ static const size_t ZSTD_frameHeaderSize_prefix = 5; /* minimum input size to know frame header size */
375
398
  static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
399
+ static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
376
400
  static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
377
401
 
378
402
 
379
403
  /*--- Advanced types ---*/
380
- typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */
404
+ typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
405
+ ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */
381
406
 
382
407
  typedef struct {
383
408
  unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -400,76 +425,141 @@ typedef struct {
400
425
  ZSTD_frameParameters fParams;
401
426
  } ZSTD_parameters;
402
427
 
428
+ typedef struct {
429
+ unsigned long long frameContentSize;
430
+ size_t windowSize;
431
+ unsigned dictID;
432
+ unsigned checksumFlag;
433
+ } ZSTD_frameHeader;
434
+
403
435
  /*= Custom memory allocation functions */
404
436
  typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
405
437
  typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
406
438
  typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
439
+ /* use this constant to defer to stdlib's functions */
440
+ static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
441
+
407
442
 
408
443
  /***************************************
409
- * Compressed size functions
444
+ * Frame size functions
410
445
  ***************************************/
411
446
 
412
447
  /*! ZSTD_findFrameCompressedSize() :
413
448
  * `src` should point to the start of a ZSTD encoded frame or skippable frame
414
449
  * `srcSize` must be at least as large as the frame
415
- * @return : the compressed size of the frame pointed to by `src`, suitable to pass to
416
- * `ZSTD_decompress` or similar, or an error code if given invalid input. */
450
+ * @return : the compressed size of the first frame starting at `src`,
451
+ * suitable to pass to `ZSTD_decompress` or similar,
452
+ * or an error code if input is invalid */
417
453
  ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
418
454
 
455
+ /*! ZSTD_findDecompressedSize() :
456
+ * `src` should point the start of a series of ZSTD encoded and/or skippable frames
457
+ * `srcSize` must be the _exact_ size of this series
458
+ * (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`)
459
+ * @return : - decompressed size of all data in all successive frames
460
+ * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
461
+ * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
462
+ *
463
+ * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
464
+ * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
465
+ * In which case, it's necessary to use streaming mode to decompress data.
466
+ * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
467
+ * note 3 : decompressed size can be very large (64-bits value),
468
+ * potentially larger than what local system can handle as a single memory segment.
469
+ * In which case, it's necessary to use streaming mode to decompress data.
470
+ * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
471
+ * Always ensure result fits within application's authorized limits.
472
+ * Each application can set its own limits.
473
+ * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
474
+ * read each contained frame header. This is fast as most of the data is skipped,
475
+ * however it does mean that all frame data must be present and valid. */
476
+ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
477
+
478
+ /*! ZSTD_frameHeaderSize() :
479
+ * `src` should point to the start of a ZSTD frame
480
+ * `srcSize` must be >= ZSTD_frameHeaderSize_prefix.
481
+ * @return : size of the Frame Header */
482
+ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
483
+
484
+
419
485
  /***************************************
420
- * Decompressed size functions
486
+ * Context memory usage
421
487
  ***************************************/
422
- /*! ZSTD_getFrameContentSize() :
423
- * `src` should point to the start of a ZSTD encoded frame
424
- * `srcSize` must be at least as large as the frame header. A value greater than or equal
425
- * to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
426
- * @return : decompressed size of the frame pointed to be `src` if known, otherwise
427
- * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
428
- * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
429
- ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
430
488
 
431
- /*! ZSTD_findDecompressedSize() :
432
- * `src` should point the start of a series of ZSTD encoded and/or skippable frames
433
- * `srcSize` must be the _exact_ size of this series
434
- * (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
435
- * @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
436
- * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
437
- * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
438
- *
439
- * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
440
- * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
441
- * In which case, it's necessary to use streaming mode to decompress data.
442
- * Optionally, application can still use ZSTD_decompress() while relying on implied limits.
443
- * (For example, data may be necessarily cut into blocks <= 16 KB).
444
- * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
445
- * note 3 : decompressed size can be very large (64-bits value),
446
- * potentially larger than what local system can handle as a single memory segment.
447
- * In which case, it's necessary to use streaming mode to decompress data.
448
- * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
449
- * Always ensure result fits within application's authorized limits.
450
- * Each application can set its own limits.
451
- * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
452
- * read each contained frame header. This is efficient as most of the data is skipped,
453
- * however it does mean that all frame data must be present and valid. */
454
- ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
489
+ /*! ZSTD_sizeof_*() :
490
+ * These functions give the current memory usage of selected object.
491
+ * Object memory usage can evolve if it's re-used multiple times. */
492
+ ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
493
+ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
494
+ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
495
+ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
496
+ ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
497
+ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
498
+
499
+ /*! ZSTD_estimate*() :
500
+ * These functions make it possible to estimate memory usage
501
+ * of a future {D,C}Ctx, before its creation.
502
+ * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
503
+ * It will also consider src size to be arbitrarily "large", which is worst case.
504
+ * If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
505
+ * ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
506
+ * Note : CCtx estimation is only correct for single-threaded compression */
507
+ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
508
+ ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
509
+ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
510
+
511
+ /*! ZSTD_estimate?StreamSize() :
512
+ * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
513
+ * It will also consider src size to be arbitrarily "large", which is worst case.
514
+ * If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
515
+ * ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
516
+ * Note : CStream estimation is only correct for single-threaded compression.
517
+ * ZSTD_DStream memory budget depends on window Size.
518
+ * This information can be passed manually, using ZSTD_estimateDStreamSize,
519
+ * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
520
+ * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
521
+ * an internal ?Dict will be created, which additional size is not estimated here.
522
+ * In this case, get total size by adding ZSTD_estimate?DictSize */
523
+ ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
524
+ ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
525
+ ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
526
+ ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
527
+
528
+ /*! ZSTD_estimate?DictSize() :
529
+ * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
530
+ * ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
531
+ * Note : dictionary created "byReference" are smaller */
532
+ ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
533
+ ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
534
+ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
455
535
 
456
536
 
457
537
  /***************************************
458
538
  * Advanced compression functions
459
539
  ***************************************/
460
- /*! ZSTD_estimateCCtxSize() :
461
- * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
462
- * `frameContentSize` is an optional parameter, provide `0` if unknown */
463
- ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
464
-
465
540
  /*! ZSTD_createCCtx_advanced() :
466
541
  * Create a ZSTD compression context using external alloc and free functions */
467
542
  ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
468
543
 
469
- /*! ZSTD_sizeofCCtx() :
470
- * Gives the amount of memory used by a given ZSTD_CCtx */
471
- ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
544
+ /*! ZSTD_initStaticCCtx() : initialize a fixed-size zstd compression context
545
+ * workspace: The memory area to emplace the context into.
546
+ * Provided pointer must 8-bytes aligned.
547
+ * It must outlive context usage.
548
+ * workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
549
+ * to determine how large workspace must be to support scenario.
550
+ * @return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
551
+ * Note : zstd will never resize nor malloc() when using a static cctx.
552
+ * If it needs more memory than available, it will simply error out.
553
+ * Note 2 : there is no corresponding "free" function.
554
+ * Since workspace was allocated externally, it must be freed externally too.
555
+ * Limitation 1 : currently not compatible with internal CDict creation, such as
556
+ * ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
557
+ * Limitation 2 : currently not compatible with multi-threading
558
+ */
559
+ ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
560
+
472
561
 
562
+ /* !!! To be deprecated !!! */
473
563
  typedef enum {
474
564
  ZSTD_p_forceWindow, /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
475
565
  ZSTD_p_forceRawDict /* Force loading dictionary in "content-only" mode (no header analysis) */
@@ -479,20 +569,43 @@ typedef enum {
479
569
  * @result : 0, or an error code (which can be tested with ZSTD_isError()) */
480
570
  ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
481
571
 
572
+
482
573
  /*! ZSTD_createCDict_byReference() :
483
574
  * Create a digested dictionary for compression
484
575
  * Dictionary content is simply referenced, and therefore stays in dictBuffer.
485
576
  * It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
486
577
  ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
487
578
 
579
+
580
+ typedef enum { ZSTD_dm_auto=0, /* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
581
+ ZSTD_dm_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
582
+ ZSTD_dm_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */
583
+ } ZSTD_dictMode_e;
488
584
  /*! ZSTD_createCDict_advanced() :
489
585
  * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
490
- ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
491
- ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
492
-
493
- /*! ZSTD_sizeof_CDict() :
494
- * Gives the amount of memory used by a given ZSTD_sizeof_CDict */
495
- ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
586
+ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
587
+ unsigned byReference, ZSTD_dictMode_e dictMode,
588
+ ZSTD_compressionParameters cParams,
589
+ ZSTD_customMem customMem);
590
+
591
+ /*! ZSTD_initStaticCDict_advanced() :
592
+ * Generate a digested dictionary in provided memory area.
593
+ * workspace: The memory area to emplace the dictionary into.
594
+ * Provided pointer must 8-bytes aligned.
595
+ * It must outlive dictionary usage.
596
+ * workspaceSize: Use ZSTD_estimateCDictSize()
597
+ * to determine how large workspace must be.
598
+ * cParams : use ZSTD_getCParams() to transform a compression level
599
+ * into its relevants cParams.
600
+ * @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
601
+ * Note : there is no corresponding "free" function.
602
+ * Since workspace was allocated externally, it must be freed externally.
603
+ */
604
+ ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict(
605
+ void* workspace, size_t workspaceSize,
606
+ const void* dict, size_t dictSize,
607
+ unsigned byReference, ZSTD_dictMode_e dictMode,
608
+ ZSTD_compressionParameters cParams);
496
609
 
497
610
  /*! ZSTD_getCParams() :
498
611
  * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
@@ -509,8 +622,8 @@ ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long l
509
622
  ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
510
623
 
511
624
  /*! ZSTD_adjustCParams() :
512
- * optimize params for a given `srcSize` and `dictSize`.
513
- * both values are optional, select `0` if unknown. */
625
+ * optimize params for a given `srcSize` and `dictSize`.
626
+ * both values are optional, select `0` if unknown. */
514
627
  ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
515
628
 
516
629
  /*! ZSTD_compress_advanced() :
@@ -538,22 +651,32 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
538
651
  * Note 3 : Skippable Frame Identifiers are considered valid. */
539
652
  ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
540
653
 
541
- /*! ZSTD_estimateDCtxSize() :
542
- * Gives the potential amount of memory allocated to create a ZSTD_DCtx */
543
- ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
544
-
545
654
  /*! ZSTD_createDCtx_advanced() :
546
655
  * Create a ZSTD decompression context using external alloc and free functions */
547
656
  ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
548
657
 
549
- /*! ZSTD_sizeof_DCtx() :
550
- * Gives the amount of memory used by a given ZSTD_DCtx */
551
- ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
658
+ /*! ZSTD_initStaticDCtx() : initialize a fixed-size zstd decompression context
659
+ * workspace: The memory area to emplace the context into.
660
+ * Provided pointer must 8-bytes aligned.
661
+ * It must outlive context usage.
662
+ * workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
663
+ * to determine how large workspace must be to support scenario.
664
+ * @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
665
+ * Note : zstd will never resize nor malloc() when using a static dctx.
666
+ * If it needs more memory than available, it will simply error out.
667
+ * Note 2 : static dctx is incompatible with legacy support
668
+ * Note 3 : there is no corresponding "free" function.
669
+ * Since workspace was allocated externally, it must be freed externally.
670
+ * Limitation : currently not compatible with internal DDict creation,
671
+ * such as ZSTD_initDStream_usingDict().
672
+ */
673
+ ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
552
674
 
553
675
  /*! ZSTD_createDDict_byReference() :
554
676
  * Create a digested dictionary, ready to start decompression operation without startup delay.
555
- * Dictionary content is simply referenced, and therefore stays in dictBuffer.
556
- * It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
677
+ * Dictionary content is referenced, and therefore stays in dictBuffer.
678
+ * It is important that dictBuffer outlives DDict,
679
+ * it must remain read accessible throughout the lifetime of DDict */
557
680
  ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
558
681
 
559
682
  /*! ZSTD_createDDict_advanced() :
@@ -561,9 +684,20 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
561
684
  ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
562
685
  unsigned byReference, ZSTD_customMem customMem);
563
686
 
564
- /*! ZSTD_sizeof_DDict() :
565
- * Gives the amount of memory used by a given ZSTD_DDict */
566
- ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
687
+ /*! ZSTD_initStaticDDict() :
688
+ * Generate a digested dictionary in provided memory area.
689
+ * workspace: The memory area to emplace the dictionary into.
690
+ * Provided pointer must 8-bytes aligned.
691
+ * It must outlive dictionary usage.
692
+ * workspaceSize: Use ZSTD_estimateDDictSize()
693
+ * to determine how large workspace must be.
694
+ * @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
695
+ * Note : there is no corresponding "free" function.
696
+ * Since workspace was allocated externally, it must be freed externally.
697
+ */
698
+ ZSTDLIB_API ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
699
+ const void* dict, size_t dictSize,
700
+ unsigned byReference);
567
701
 
568
702
  /*! ZSTD_getDictID_fromDict() :
569
703
  * Provides the dictID stored within dictionary.
@@ -586,7 +720,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
586
720
  * Note : this use case also happens when using a non-conformant dictionary.
587
721
  * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
588
722
  * - This is not a Zstandard frame.
589
- * When identifying the exact failure cause, it's possible to use ZSTD_getFrameParams(), which will provide a more precise error code. */
723
+ * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
590
724
  ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
591
725
 
592
726
 
@@ -596,13 +730,13 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
596
730
 
597
731
  /*===== Advanced Streaming compression functions =====*/
598
732
  ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
599
- ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /**< size of CStream is variable, depending primarily on compression level */
733
+ ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
600
734
  ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
601
- ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
735
+ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
602
736
  ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
603
737
  ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
604
738
  ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
605
- ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
739
+ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
606
740
 
607
741
  /*! ZSTD_resetCStream() :
608
742
  * start a new compression job, using same parameters from previous job.
@@ -617,11 +751,11 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg
617
751
  /*===== Advanced Streaming decompression functions =====*/
618
752
  typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
619
753
  ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
620
- ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
754
+ ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
621
755
  ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
756
+ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
622
757
  ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
623
758
  ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
624
- ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
625
759
 
626
760
 
627
761
  /*********************************************************************
@@ -683,21 +817,24 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
683
817
  Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
684
818
  A ZSTD_DCtx object can be re-used multiple times.
685
819
 
686
- First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
687
- It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
688
- such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
689
- and the dictionary ID used.
820
+ First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
821
+ It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
822
+ such as minimum rolling buffer size to allocate to decompress data (`windowSize`),
823
+ and the dictionary ID in use.
690
824
  (Note : content size is optional, it may not be present. 0 means : content size unknown).
691
825
  Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
692
826
  As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
693
- Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
694
- Frame parameters are extracted from the beginning of the compressed frame.
695
- Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
696
- @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
827
+ Each application can set its own limit, depending on local restrictions.
828
+ For extended interoperability, it is recommended to support windowSize of at least 8 MB.
829
+ Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
830
+ Data fragment must be large enough to ensure successful decoding.
831
+ `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
832
+ @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
697
833
  >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
698
834
  errorCode, which can be tested using ZSTD_isError().
699
835
 
700
- Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
836
+ Start decompression, with ZSTD_decompressBegin().
837
+ If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
701
838
  Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
702
839
 
703
840
  Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
@@ -729,30 +866,233 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
729
866
  b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
730
867
  c) Frame Content - any content (User Data) of length equal to Frame Size
731
868
  For skippable frames ZSTD_decompressContinue() always returns 0.
732
- For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
869
+ For skippable frames ZSTD_getFrameHeader() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
733
870
  Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
734
871
  For purposes of decompression, it is valid in both cases to skip the frame using
735
872
  ZSTD_findFrameCompressedSize to find its size in bytes.
736
873
  It also returns Frame Size as fparamsPtr->frameContentSize.
737
874
  */
738
875
 
739
- typedef struct {
740
- unsigned long long frameContentSize;
741
- unsigned windowSize;
742
- unsigned dictID;
743
- unsigned checksumFlag;
744
- } ZSTD_frameParams;
745
-
746
876
  /*===== Buffer-less streaming decompression functions =====*/
747
- ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
877
+ ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
748
878
  ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
749
879
  ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
880
+ ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
750
881
  ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
882
+
751
883
  ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
752
884
  ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
753
885
  typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
754
886
  ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
755
887
 
888
+
889
+
890
+ /*=== New advanced API (experimental, and compression only) ===*/
891
+
892
+ /* notes on API design :
893
+ * In this proposal, parameters are pushed one by one into an existing CCtx,
894
+ * and then applied on all subsequent compression jobs.
895
+ * When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT.
896
+ *
897
+ * This API is intended to replace all others experimental API.
898
+ * It can basically do all other use cases, and even new ones.
899
+ * It stands a good chance to become "stable",
900
+ * after a reasonable testing period.
901
+ */
902
+
903
+ /* note on naming convention :
904
+ * Initially, the API favored names like ZSTD_setCCtxParameter() .
905
+ * In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
906
+ * The main driver is that it identifies more clearly the target object type.
907
+ * It feels clearer in light of potential variants :
908
+ * ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
909
+ * ZSTD_DCtx_setParameter() (rather than ZSTD_setDCtxParameter() )
910
+ * Left variant feels easier to distinguish.
911
+ */
912
+
913
+ /* note on enum design :
914
+ * All enum will be manually set to explicit values before reaching "stable API" status */
915
+
916
+ typedef enum {
917
+ /* compression parameters */
918
+ ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
919
+ * Default level is ZSTD_CLEVEL_DEFAULT==3.
920
+ * Special: value 0 means "do not change cLevel". */
921
+ ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
922
+ * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
923
+ * Special: value 0 means "do not change windowLog". */
924
+ ZSTD_p_hashLog, /* Size of the probe table, as a power of 2.
925
+ * Resulting table size is (1 << (hashLog+2)).
926
+ * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
927
+ * Larger tables improve compression ratio of strategies <= dFast,
928
+ * and improve speed of strategies > dFast.
929
+ * Special: value 0 means "do not change hashLog". */
930
+ ZSTD_p_chainLog, /* Size of the full-search table, as a power of 2.
931
+ * Resulting table size is (1 << (chainLog+2)).
932
+ * Larger tables result in better and slower compression.
933
+ * This parameter is useless when using "fast" strategy.
934
+ * Special: value 0 means "do not change chainLog". */
935
+ ZSTD_p_searchLog, /* Number of search attempts, as a power of 2.
936
+ * More attempts result in better and slower compression.
937
+ * This parameter is useless when using "fast" and "dFast" strategies.
938
+ * Special: value 0 means "do not change searchLog". */
939
+ ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller).
940
+ * Larger values make faster compression and decompression, but decrease ratio.
941
+ * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
942
+ * Note that currently, for all strategies < btopt, effective minimum is 4.
943
+ * Note that currently, for all strategies > fast, effective maximum is 6.
944
+ * Special: value 0 means "do not change minMatchLength". */
945
+ ZSTD_p_targetLength, /* Only useful for strategies >= btopt.
946
+ * Length of Match considered "good enough" to stop search.
947
+ * Larger values make compression stronger and slower.
948
+ * Special: value 0 means "do not change targetLength". */
949
+ ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
950
+ * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
951
+ * The higher the value of selected strategy, the more complex it is,
952
+ * resulting in stronger and slower compression.
953
+ * Special: value 0 means "do not change strategy". */
954
+
955
+ /* frame parameters */
956
+ ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
957
+ ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
958
+ ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */
959
+
960
+ /* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */
961
+ ZSTD_p_dictMode=300, /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
962
+ * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
963
+ ZSTD_p_refDictContent, /* Dictionary content will be referenced, instead of copied (default:0==byCopy).
964
+ * It requires that dictionary buffer outlives its users */
965
+
966
+ /* multi-threading parameters */
967
+ ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
968
+ * More threads improve speed, but also increase memory usage.
969
+ * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
970
+ * Special: value 0 means "do not change nbThreads" */
971
+ ZSTD_p_jobSize, /* Size of a compression job. Each compression job is completed in parallel.
972
+ * 0 means default, which is dynamically determined based on compression parameters.
973
+ * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
974
+ * The minimum size is automatically and transparently enforced */
975
+ ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
976
+ * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
977
+
978
+ /* advanced parameters - may not remain available after API update */
979
+ ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
980
+ * even when referencing into Dictionary content (default:0) */
981
+
982
+ } ZSTD_cParameter;
983
+
984
+
985
+ /*! ZSTD_CCtx_setParameter() :
986
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
987
+ * Note : when `value` is an enum, cast it to unsigned for proper type checking.
988
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()). */
989
+ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
990
+
991
+ /*! ZSTD_CCtx_setPledgedSrcSize() :
992
+ * Total input data size to be compressed as a single frame.
993
+ * This value will be controlled at the end, and result in error if not respected.
994
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
995
+ * Note 1 : 0 means zero, empty.
996
+ * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
997
+ * Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
998
+ * Note 2 : If all data is provided and consumed in a single round,
999
+ * this value is overriden by srcSize instead. */
1000
+ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
1001
+
1002
+ /*! ZSTD_CCtx_loadDictionary() :
1003
+ * Create an internal CDict from dict buffer.
1004
+ * Decompression will have to use same buffer.
1005
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1006
+ * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
1007
+ * meaning "return to no-dictionary mode".
1008
+ * Note 1 : `dict` content will be copied internally,
1009
+ * except if ZSTD_p_refDictContent is set before loading.
1010
+ * Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
1011
+ * For this reason, compression parameters cannot be changed anymore after loading a dictionary.
1012
+ * It's also a CPU-heavy operation, with non-negligible impact on latency.
1013
+ * Note 3 : Dictionary will be used for all future compression jobs.
1014
+ * To return to "no-dictionary" situation, load a NULL dictionary */
1015
+ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
1016
+
1017
+ /*! ZSTD_CCtx_refCDict() :
1018
+ * Reference a prepared dictionary, to be used for all next compression jobs.
1019
+ * Note that compression parameters are enforced from within CDict,
1020
+ * and supercede any compression parameter previously set within CCtx.
1021
+ * The dictionary will remain valid for future compression jobs using same CCtx.
1022
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1023
+ * Special : adding a NULL CDict means "return to no-dictionary mode".
1024
+ * Note 1 : Currently, only one dictionary can be managed.
1025
+ * Adding a new dictionary effectively "discards" any previous one.
1026
+ * Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
1027
+ */
1028
+ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
1029
+
1030
+ /*! ZSTD_CCtx_refPrefix() :
1031
+ * Reference a prefix (single-usage dictionary) for next compression job.
1032
+ * Decompression need same prefix to properly regenerate data.
1033
+ * Prefix is **only used once**. Tables are discarded at end of compression job.
1034
+ * Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
1035
+ * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
1036
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
1037
+ * Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
1038
+ * Note 1 : Prefix buffer is referenced. It must outlive compression job.
1039
+ * Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
1040
+ * It's a CPU-heavy operation, with non-negligible impact on latency.
1041
+ * Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter() */
1042
+ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
1043
+
1044
+
1045
+
1046
+ typedef enum {
1047
+ ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
1048
+ ZSTD_e_flush, /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
1049
+ ZSTD_e_end /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
1050
+ } ZSTD_EndDirective;
1051
+
1052
+ /*! ZSTD_compress_generic() :
1053
+ * Behave about the same as ZSTD_compressStream. To note :
1054
+ * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
1055
+ * - Compression parameters cannot be changed once compression is started.
1056
+ * - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
1057
+ * - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
1058
+ * - @return provides the minimum amount of data still to flush from internal buffers
1059
+ * or an error code, which can be tested using ZSTD_isError().
1060
+ * if @return != 0, flush is not fully completed, there is some data left within internal buffers.
1061
+ * - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
1062
+ * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
1063
+ * It is necessary to fully flush internal buffers
1064
+ * before starting a new compression job, or changing compression parameters.
1065
+ */
1066
+ ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
1067
+ ZSTD_outBuffer* output,
1068
+ ZSTD_inBuffer* input,
1069
+ ZSTD_EndDirective endOp);
1070
+
1071
+ /*! ZSTD_CCtx_reset() :
1072
+ * Return a CCtx to clean state.
1073
+ * Useful after an error, or to interrupt an ongoing compression job and start a new one.
1074
+ * Any internal data not yet flushed is cancelled.
1075
+ * Dictionary (if any) is dropped.
1076
+ * It's possible to modify compression parameters after a reset.
1077
+ */
1078
+ ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); /* Not ready yet ! */
1079
+
1080
+
1081
+ /*! ZSTD_compress_generic_simpleArgs() :
1082
+ * Same as ZSTD_compress_generic(),
1083
+ * but using only integral types as arguments.
1084
+ * Argument list is larger and less expressive than ZSTD_{in,out}Buffer,
1085
+ * but can be helpful for binders from dynamic languages
1086
+ * which have troubles handling structures containing memory pointers.
1087
+ */
1088
+ size_t ZSTD_compress_generic_simpleArgs (
1089
+ ZSTD_CCtx* cctx,
1090
+ void* dst, size_t dstCapacity, size_t* dstPos,
1091
+ const void* src, size_t srcSize, size_t* srcPos,
1092
+ ZSTD_EndDirective endOp);
1093
+
1094
+
1095
+
756
1096
  /**
757
1097
  Block functions
758
1098
 
@@ -767,7 +1107,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
767
1107
  + compression : any ZSTD_compressBegin*() variant, including with dictionary
768
1108
  + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
769
1109
  + copyCCtx() and copyDCtx() can be used too
770
- - Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX
1110
+ - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
771
1111
  + If input is larger than a block size, it's necessary to split input data into multiple blocks
772
1112
  + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
773
1113
  Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
@@ -780,9 +1120,10 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
780
1120
  Use ZSTD_insertBlock() for such a case.
781
1121
  */
782
1122
 
783
- #define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
1123
+ #define ZSTD_BLOCKSIZELOG_MAX 17
1124
+ #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */
784
1125
  /*===== Raw zstd block functions =====*/
785
- ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
1126
+ ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
786
1127
  ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
787
1128
  ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
788
1129
  ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */