zstd-ruby 1.3.5.0 → 1.3.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +4 -2
  3. data/README.md +2 -1
  4. data/ext/zstdruby/libzstd/BUCK +1 -0
  5. data/ext/zstdruby/libzstd/Makefile +25 -13
  6. data/ext/zstdruby/libzstd/README.md +11 -10
  7. data/ext/zstdruby/libzstd/common/bitstream.h +8 -11
  8. data/ext/zstdruby/libzstd/common/compiler.h +30 -8
  9. data/ext/zstdruby/libzstd/common/cpu.h +1 -1
  10. data/ext/zstdruby/libzstd/common/mem.h +20 -2
  11. data/ext/zstdruby/libzstd/common/xxhash.c +1 -0
  12. data/ext/zstdruby/libzstd/common/zstd_internal.h +3 -2
  13. data/ext/zstdruby/libzstd/compress/fse_compress.c +55 -48
  14. data/ext/zstdruby/libzstd/compress/hist.h +1 -1
  15. data/ext/zstdruby/libzstd/compress/huf_compress.c +1 -1
  16. data/ext/zstdruby/libzstd/compress/zstd_compress.c +290 -147
  17. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +5 -2
  18. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +63 -51
  19. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +3 -4
  20. data/ext/zstdruby/libzstd/compress/zstd_fast.c +44 -33
  21. data/ext/zstdruby/libzstd/compress/zstd_fast.h +3 -4
  22. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +125 -116
  23. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +13 -15
  24. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +9 -11
  25. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +0 -1
  26. data/ext/zstdruby/libzstd/compress/zstd_opt.c +42 -36
  27. data/ext/zstdruby/libzstd/compress/zstd_opt.h +8 -9
  28. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +96 -51
  29. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +16 -6
  30. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +3 -3
  31. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +169 -101
  32. data/ext/zstdruby/libzstd/dictBuilder/cover.c +111 -87
  33. data/ext/zstdruby/libzstd/dictBuilder/cover.h +83 -0
  34. data/ext/zstdruby/libzstd/dictBuilder/divsufsort.c +3 -3
  35. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +728 -0
  36. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +34 -31
  37. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +60 -5
  38. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +9 -3
  39. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +6 -0
  40. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +6 -0
  41. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +1 -5
  42. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +12 -9
  43. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +10 -10
  44. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +20 -18
  45. data/ext/zstdruby/libzstd/zstd.h +109 -50
  46. data/lib/zstd-ruby/version.rb +1 -1
  47. metadata +4 -2
@@ -293,7 +293,7 @@ static dictItem ZDICT_analyzePos(
293
293
  refinedEnd = refinedStart + selectedCount;
294
294
  }
295
295
 
296
- /* evaluate gain based on new ref */
296
+ /* evaluate gain based on new dict */
297
297
  start = refinedStart;
298
298
  pos = suffix[refinedStart];
299
299
  end = start;
@@ -341,7 +341,7 @@ static dictItem ZDICT_analyzePos(
341
341
  for (i=MINMATCHLENGTH; i<=(int)maxLength; i++)
342
342
  savings[i] = savings[i-1] + (lengthList[i] * (i-3));
343
343
 
344
- DISPLAYLEVEL(4, "Selected ref at position %u, of length %u : saves %u (ratio: %.2f) \n",
344
+ DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n",
345
345
  (U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength);
346
346
 
347
347
  solution.pos = (U32)pos;
@@ -581,7 +581,7 @@ static void ZDICT_fillNoise(void* buffer, size_t length)
581
581
 
582
582
  typedef struct
583
583
  {
584
- ZSTD_CCtx* ref; /* contains reference to dictionary */
584
+ ZSTD_CDict* dict; /* dictionary */
585
585
  ZSTD_CCtx* zc; /* working context */
586
586
  void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */
587
587
  } EStats_ress_t;
@@ -597,8 +597,9 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
597
597
  size_t cSize;
598
598
 
599
599
  if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
600
- { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0);
601
- if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
600
+ { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict);
601
+ if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; }
602
+
602
603
  }
603
604
  cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
604
605
  if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
@@ -697,7 +698,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
697
698
  short litLengthNCount[MaxLL+1];
698
699
  U32 repOffset[MAXREPOFFSET];
699
700
  offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
700
- EStats_ress_t esr;
701
+ EStats_ress_t esr = { NULL, NULL, NULL };
701
702
  ZSTD_parameters params;
702
703
  U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total;
703
704
  size_t pos = 0, errorCode;
@@ -708,14 +709,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
708
709
 
709
710
  /* init */
710
711
  DEBUGLOG(4, "ZDICT_analyzeEntropy");
711
- esr.ref = ZSTD_createCCtx();
712
- esr.zc = ZSTD_createCCtx();
713
- esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX);
714
- if (!esr.ref || !esr.zc || !esr.workPlace) {
715
- eSize = ERROR(memory_allocation);
716
- DISPLAYLEVEL(1, "Not enough memory \n");
717
- goto _cleanup;
718
- }
719
712
  if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionaryCreation_failed); goto _cleanup; } /* too large dictionary */
720
713
  for (u=0; u<256; u++) countLit[u] = 1; /* any character must be described */
721
714
  for (u=0; u<=offcodeMax; u++) offcodeCount[u] = 1;
@@ -726,12 +719,15 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
726
719
  memset(bestRepOffset, 0, sizeof(bestRepOffset));
727
720
  if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
728
721
  params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
729
- { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
730
- if (ZSTD_isError(beginResult)) {
731
- DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced() failed : %s \n", ZSTD_getErrorName(beginResult));
732
- eSize = ERROR(GENERIC);
733
- goto _cleanup;
734
- } }
722
+
723
+ esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem);
724
+ esr.zc = ZSTD_createCCtx();
725
+ esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX);
726
+ if (!esr.dict || !esr.zc || !esr.workPlace) {
727
+ eSize = ERROR(memory_allocation);
728
+ DISPLAYLEVEL(1, "Not enough memory \n");
729
+ goto _cleanup;
730
+ }
735
731
 
736
732
  /* collect stats on all samples */
737
733
  for (u=0; u<nbFiles; u++) {
@@ -856,7 +852,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
856
852
  eSize += 12;
857
853
 
858
854
  _cleanup:
859
- ZSTD_freeCCtx(esr.ref);
855
+ ZSTD_freeCDict(esr.dict);
860
856
  ZSTD_freeCCtx(esr.zc);
861
857
  free(esr.workPlace);
862
858
 
@@ -867,8 +863,8 @@ _cleanup:
867
863
 
868
864
  size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
869
865
  const void* customDictContent, size_t dictContentSize,
870
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
871
- ZDICT_params_t params)
866
+ const void* samplesBuffer, const size_t* samplesSizes,
867
+ unsigned nbSamples, ZDICT_params_t params)
872
868
  {
873
869
  size_t hSize;
874
870
  #define HBUFFSIZE 256 /* should prove large enough for all entropy headers */
@@ -914,9 +910,10 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
914
910
  }
915
911
 
916
912
 
917
- size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
918
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
919
- ZDICT_params_t params)
913
+ static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
914
+ void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
915
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
916
+ ZDICT_params_t params)
920
917
  {
921
918
  int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
922
919
  U32 const notificationLevel = params.notificationLevel;
@@ -947,7 +944,11 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
947
944
  return MIN(dictBufferCapacity, hSize+dictContentSize);
948
945
  }
949
946
 
950
-
947
+ /* Hidden declaration for dbio.c */
948
+ size_t ZDICT_trainFromBuffer_unsafe_legacy(
949
+ void* dictBuffer, size_t maxDictSize,
950
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
951
+ ZDICT_legacy_params_t params);
951
952
  /*! ZDICT_trainFromBuffer_unsafe_legacy() :
952
953
  * Warning : `samplesBuffer` must be followed by noisy guard band.
953
954
  * @return : size of dictionary, or an error code which can be tested with ZDICT_isError()
@@ -991,8 +992,10 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(
991
992
  U32 const pos = dictList[u].pos;
992
993
  U32 const length = dictList[u].length;
993
994
  U32 const printedLength = MIN(40, length);
994
- if ((pos > samplesBuffSize) || ((pos + length) > samplesBuffSize))
995
+ if ((pos > samplesBuffSize) || ((pos + length) > samplesBuffSize)) {
996
+ free(dictList);
995
997
  return ERROR(GENERIC); /* should never happen */
998
+ }
996
999
  DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
997
1000
  u, length, pos, dictList[u].savings);
998
1001
  ZDICT_printHex((const char*)samplesBuffer+pos, printedLength);
@@ -1082,17 +1085,17 @@ size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity,
1082
1085
  size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
1083
1086
  const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)
1084
1087
  {
1085
- ZDICT_cover_params_t params;
1088
+ ZDICT_fastCover_params_t params;
1086
1089
  DEBUGLOG(3, "ZDICT_trainFromBuffer");
1087
1090
  memset(&params, 0, sizeof(params));
1088
1091
  params.d = 8;
1089
1092
  params.steps = 4;
1090
1093
  /* Default to level 6 since no compression level information is available */
1091
- params.zParams.compressionLevel = 6;
1094
+ params.zParams.compressionLevel = 3;
1092
1095
  #if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1)
1093
1096
  params.zParams.notificationLevel = DEBUGLEVEL;
1094
1097
  #endif
1095
- return ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, dictBufferCapacity,
1098
+ return ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, dictBufferCapacity,
1096
1099
  samplesBuffer, samplesSizes, nbSamples,
1097
1100
  &params);
1098
1101
  }
@@ -39,7 +39,8 @@ extern "C" {
39
39
 
40
40
  /*! ZDICT_trainFromBuffer():
41
41
  * Train a dictionary from an array of samples.
42
- * Redirect towards ZDICT_optimizeTrainFromBuffer_cover() single-threaded, with d=8 and steps=4.
42
+ * Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4,
43
+ * f=20, and accel=1.
43
44
  * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
44
45
  * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
45
46
  * The resulting dictionary will be saved into `dictBuffer`.
@@ -52,7 +53,8 @@ extern "C" {
52
53
  * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
53
54
  */
54
55
  ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
55
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
56
+ const void* samplesBuffer,
57
+ const size_t* samplesSizes, unsigned nbSamples);
56
58
 
57
59
 
58
60
  /*====== Helper functions ======*/
@@ -84,11 +86,22 @@ typedef struct {
84
86
  typedef struct {
85
87
  unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
86
88
  unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
87
- unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
89
+ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
88
90
  unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
91
+ double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
89
92
  ZDICT_params_t zParams;
90
93
  } ZDICT_cover_params_t;
91
94
 
95
+ typedef struct {
96
+ unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
97
+ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
98
+ unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/
99
+ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
100
+ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
101
+ double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
102
+ unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
103
+ ZDICT_params_t zParams;
104
+ } ZDICT_fastCover_params_t;
92
105
 
93
106
  /*! ZDICT_trainFromBuffer_cover():
94
107
  * Train a dictionary from an array of samples using the COVER algorithm.
@@ -115,9 +128,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
115
128
  * dictionary constructed with those parameters is stored in `dictBuffer`.
116
129
  *
117
130
  * All of the parameters d, k, steps are optional.
118
- * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}.
131
+ * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
119
132
  * if steps is zero it defaults to its default value.
120
- * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048].
133
+ * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
121
134
  *
122
135
  * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
123
136
  * or an error code, which can be tested with ZDICT_isError().
@@ -129,6 +142,48 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
129
142
  const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
130
143
  ZDICT_cover_params_t* parameters);
131
144
 
145
+ /*! ZDICT_trainFromBuffer_fastCover():
146
+ * Train a dictionary from an array of samples using a modified version of COVER algorithm.
147
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
148
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
149
+ * d and k are required.
150
+ * All other parameters are optional, will use default values if not provided
151
+ * The resulting dictionary will be saved into `dictBuffer`.
152
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
153
+ * or an error code, which can be tested with ZDICT_isError().
154
+ * Note: ZDICT_trainFromBuffer_fastCover() requires about 1 bytes of memory for each input byte and additionally another 6 * 2^f bytes of memory .
155
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
156
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
157
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
158
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
159
+ */
160
+ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
161
+ size_t dictBufferCapacity, const void *samplesBuffer,
162
+ const size_t *samplesSizes, unsigned nbSamples,
163
+ ZDICT_fastCover_params_t parameters);
164
+
165
+ /*! ZDICT_optimizeTrainFromBuffer_fastCover():
166
+ * The same requirements as above hold for all the parameters except `parameters`.
167
+ * This function tries many parameter combinations (specifically, k and d combinations)
168
+ * and picks the best parameters. `*parameters` is filled with the best parameters found,
169
+ * dictionary constructed with those parameters is stored in `dictBuffer`.
170
+ * All of the parameters d, k, steps, f, and accel are optional.
171
+ * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
172
+ * if steps is zero it defaults to its default value.
173
+ * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
174
+ * If f is zero, default value of 20 is used.
175
+ * If accel is zero, default value of 1 is used.
176
+ *
177
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
178
+ * or an error code, which can be tested with ZDICT_isError().
179
+ * On success `*parameters` contains the parameters selected.
180
+ * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 1 byte of memory for each input byte and additionally another 6 * 2^f bytes of memory for each thread.
181
+ */
182
+ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
183
+ size_t dictBufferCapacity, const void* samplesBuffer,
184
+ const size_t* samplesSizes, unsigned nbSamples,
185
+ ZDICT_fastCover_params_t* parameters);
186
+
132
187
  /*! ZDICT_finalizeDictionary():
133
188
  * Given a custom content as a basis for dictionary, and a set of samples,
134
189
  * finalize dictionary by adding headers and statistics.
@@ -668,11 +668,17 @@ static size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t
668
668
  switch(srcSize)
669
669
  {
670
670
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
671
+ /* fallthrough */
671
672
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
673
+ /* fallthrough */
672
674
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
675
+ /* fallthrough */
673
676
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
677
+ /* fallthrough */
674
678
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
679
+ /* fallthrough */
675
680
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
681
+ /* fallthrough */
676
682
  default:;
677
683
  }
678
684
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -1458,7 +1464,7 @@ unsigned ZSTDv01_isError(size_t code) { return ERR_isError(code); }
1458
1464
  * Decompression code
1459
1465
  **************************************************************/
1460
1466
 
1461
- size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
1467
+ static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
1462
1468
  {
1463
1469
  const BYTE* const in = (const BYTE* const)src;
1464
1470
  BYTE headerFlags;
@@ -1511,7 +1517,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
1511
1517
  }
1512
1518
 
1513
1519
 
1514
- size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
1520
+ static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
1515
1521
  void* dst, size_t maxDstSize,
1516
1522
  const BYTE** litStart, size_t* litSize,
1517
1523
  const void* src, size_t srcSize)
@@ -1563,7 +1569,7 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
1563
1569
  }
1564
1570
 
1565
1571
 
1566
- size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
1572
+ static size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
1567
1573
  FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
1568
1574
  const void* src, size_t srcSize)
1569
1575
  {
@@ -399,11 +399,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
399
399
  switch(srcSize)
400
400
  {
401
401
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
402
+ /* fallthrough */
402
403
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
404
+ /* fallthrough */
403
405
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
406
+ /* fallthrough */
404
407
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
408
+ /* fallthrough */
405
409
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
410
+ /* fallthrough */
406
411
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
412
+ /* fallthrough */
407
413
  default:;
408
414
  }
409
415
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -402,11 +402,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
402
402
  switch(srcSize)
403
403
  {
404
404
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
405
+ /* fallthrough */
405
406
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
407
+ /* fallthrough */
406
408
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
409
+ /* fallthrough */
407
410
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
411
+ /* fallthrough */
408
412
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
413
+ /* fallthrough */
409
414
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
415
+ /* fallthrough */
410
416
  default:;
411
417
  }
412
418
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -1093,6 +1093,7 @@ static size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, un
1093
1093
  if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
1094
1094
 
1095
1095
  /* Init, lay down lowprob symbols */
1096
+ memset(tableDecode, 0, sizeof(FSE_DECODE_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
1096
1097
  DTableH.tableLog = (U16)tableLog;
1097
1098
  for (s=0; s<=maxSymbolValue; s++)
1098
1099
  {
@@ -3621,8 +3622,3 @@ size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDs
3621
3622
 
3622
3623
  ZSTD_DCtx* ZSTDv04_createDCtx(void) { return ZSTD_createDCtx(); }
3623
3624
  size_t ZSTDv04_freeDCtx(ZSTD_DCtx* dctx) { return ZSTD_freeDCtx(dctx); }
3624
-
3625
- size_t ZSTDv04_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
3626
- {
3627
- return ZSTD_getFrameParams(params, src, srcSize);
3628
- }
@@ -1224,6 +1224,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns
1224
1224
  if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
1225
1225
 
1226
1226
  /* Init, lay down lowprob symbols */
1227
+ memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
1227
1228
  DTableH.tableLog = (U16)tableLog;
1228
1229
  for (s=0; s<=maxSymbolValue; s++) {
1229
1230
  if (normalizedCounter[s]==-1) {
@@ -2658,6 +2659,7 @@ struct ZSTDv05_DCtx_s
2658
2659
  BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
2659
2660
  }; /* typedef'd to ZSTDv05_DCtx within "zstd_static.h" */
2660
2661
 
2662
+ size_t ZSTDv05_sizeofDCtx (void); /* Hidden declaration */
2661
2663
  size_t ZSTDv05_sizeofDCtx (void) { return sizeof(ZSTDv05_DCtx); }
2662
2664
 
2663
2665
  size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx)
@@ -2822,7 +2824,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
2822
2824
  }
2823
2825
 
2824
2826
 
2825
- size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
2827
+ static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
2826
2828
  {
2827
2829
  const BYTE* const in = (const BYTE* const)src;
2828
2830
  BYTE headerFlags;
@@ -2845,6 +2847,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
2845
2847
 
2846
2848
  static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
2847
2849
  {
2850
+ if (dst==NULL) return ERROR(dstSize_tooSmall);
2848
2851
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
2849
2852
  memcpy(dst, src, srcSize);
2850
2853
  return srcSize;
@@ -2853,8 +2856,8 @@ static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src
2853
2856
 
2854
2857
  /*! ZSTDv05_decodeLiteralsBlock() :
2855
2858
  @return : nb of bytes read from src (< srcSize ) */
2856
- size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2857
- const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
2859
+ static size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2860
+ const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
2858
2861
  {
2859
2862
  const BYTE* const istart = (const BYTE*) src;
2860
2863
 
@@ -2988,7 +2991,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2988
2991
  }
2989
2992
 
2990
2993
 
2991
- size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
2994
+ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
2992
2995
  FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
2993
2996
  const void* src, size_t srcSize, U32 flagStaticTable)
2994
2997
  {
@@ -3297,11 +3300,11 @@ static size_t ZSTDv05_decompressSequences(
3297
3300
  BYTE* const ostart = (BYTE* const)dst;
3298
3301
  BYTE* op = ostart;
3299
3302
  BYTE* const oend = ostart + maxDstSize;
3300
- size_t errorCode, dumpsLength;
3303
+ size_t errorCode, dumpsLength=0;
3301
3304
  const BYTE* litPtr = dctx->litPtr;
3302
3305
  const BYTE* const litEnd = litPtr + dctx->litSize;
3303
- int nbSeq;
3304
- const BYTE* dumps;
3306
+ int nbSeq=0;
3307
+ const BYTE* dumps = NULL;
3305
3308
  U32* DTableLL = dctx->LLTable;
3306
3309
  U32* DTableML = dctx->MLTable;
3307
3310
  U32* DTableOffb = dctx->OffTable;
@@ -3410,10 +3413,10 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
3410
3413
  BYTE* const oend = ostart + maxDstSize;
3411
3414
  size_t remainingSize = srcSize;
3412
3415
  blockProperties_t blockProperties;
3416
+ memset(&blockProperties, 0, sizeof(blockProperties));
3413
3417
 
3414
3418
  /* Frame Header */
3415
- {
3416
- size_t frameHeaderSize;
3419
+ { size_t frameHeaderSize;
3417
3420
  if (srcSize < ZSTDv05_frameHeaderSize_min+ZSTDv05_blockHeaderSize) return ERROR(srcSize_wrong);
3418
3421
  frameHeaderSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
3419
3422
  if (ZSTDv05_isError(frameHeaderSize)) return frameHeaderSize;
@@ -1250,9 +1250,7 @@ const char* FSEv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
1250
1250
  /* **************************************************************
1251
1251
  * HUF Error Management
1252
1252
  ****************************************************************/
1253
- unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
1254
-
1255
- const char* HUFv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
1253
+ static unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
1256
1254
 
1257
1255
 
1258
1256
  /*-**************************************************************
@@ -2823,7 +2821,8 @@ struct ZSTDv06_DCtx_s
2823
2821
  BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
2824
2822
  }; /* typedef'd to ZSTDv06_DCtx within "zstd_static.h" */
2825
2823
 
2826
- size_t ZSTDv06_sizeofDCtx (void) { return sizeof(ZSTDv06_DCtx); } /* non published interface */
2824
+ size_t ZSTDv06_sizeofDCtx (void); /* Hidden declaration */
2825
+ size_t ZSTDv06_sizeofDCtx (void) { return sizeof(ZSTDv06_DCtx); }
2827
2826
 
2828
2827
  size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx)
2829
2828
  {
@@ -3022,7 +3021,7 @@ typedef struct
3022
3021
 
3023
3022
  /*! ZSTDv06_getcBlockSize() :
3024
3023
  * Provides the size of compressed block from block header `src` */
3025
- size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
3024
+ static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
3026
3025
  {
3027
3026
  const BYTE* const in = (const BYTE* const)src;
3028
3027
  U32 cSize;
@@ -3041,6 +3040,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
3041
3040
 
3042
3041
  static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
3043
3042
  {
3043
+ if (dst==NULL) return ERROR(dstSize_tooSmall);
3044
3044
  if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
3045
3045
  memcpy(dst, src, srcSize);
3046
3046
  return srcSize;
@@ -3049,7 +3049,7 @@ static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* sr
3049
3049
 
3050
3050
  /*! ZSTDv06_decodeLiteralsBlock() :
3051
3051
  @return : nb of bytes read from src (< srcSize ) */
3052
- size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
3052
+ static size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
3053
3053
  const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
3054
3054
  {
3055
3055
  const BYTE* const istart = (const BYTE*) src;
@@ -3183,7 +3183,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
3183
3183
  @return : nb bytes read from src,
3184
3184
  or an error code if it fails, testable with ZSTDv06_isError()
3185
3185
  */
3186
- size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
3186
+ static size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
3187
3187
  const void* src, size_t srcSize,
3188
3188
  const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
3189
3189
  {
@@ -3213,7 +3213,7 @@ size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLo
3213
3213
  }
3214
3214
 
3215
3215
 
3216
- size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3216
+ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3217
3217
  FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
3218
3218
  const void* src, size_t srcSize)
3219
3219
  {
@@ -3358,7 +3358,7 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
3358
3358
  }
3359
3359
 
3360
3360
 
3361
- size_t ZSTDv06_execSequence(BYTE* op,
3361
+ static size_t ZSTDv06_execSequence(BYTE* op,
3362
3362
  BYTE* const oend, seq_t sequence,
3363
3363
  const BYTE** litPtr, const BYTE* const litLimit,
3364
3364
  const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
@@ -4006,7 +4006,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4006
4006
  if (ZSTDv06_isError(hSize)) return hSize;
4007
4007
  if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
4008
4008
  memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
4009
- zbd->lhSize += iend-ip; ip = iend; notDone = 0;
4009
+ zbd->lhSize += iend-ip;
4010
4010
  *dstCapacityPtr = 0;
4011
4011
  return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
4012
4012
  }