zstd-ruby 1.3.5.0 → 1.3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +4 -2
- data/README.md +2 -1
- data/ext/zstdruby/libzstd/BUCK +1 -0
- data/ext/zstdruby/libzstd/Makefile +25 -13
- data/ext/zstdruby/libzstd/README.md +11 -10
- data/ext/zstdruby/libzstd/common/bitstream.h +8 -11
- data/ext/zstdruby/libzstd/common/compiler.h +30 -8
- data/ext/zstdruby/libzstd/common/cpu.h +1 -1
- data/ext/zstdruby/libzstd/common/mem.h +20 -2
- data/ext/zstdruby/libzstd/common/xxhash.c +1 -0
- data/ext/zstdruby/libzstd/common/zstd_internal.h +3 -2
- data/ext/zstdruby/libzstd/compress/fse_compress.c +55 -48
- data/ext/zstdruby/libzstd/compress/hist.h +1 -1
- data/ext/zstdruby/libzstd/compress/huf_compress.c +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +290 -147
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +5 -2
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +63 -51
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +3 -4
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +44 -33
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +3 -4
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +125 -116
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +13 -15
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +9 -11
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +0 -1
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +42 -36
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +8 -9
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +96 -51
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +16 -6
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +3 -3
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +169 -101
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +111 -87
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +83 -0
- data/ext/zstdruby/libzstd/dictBuilder/divsufsort.c +3 -3
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +728 -0
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +34 -31
- data/ext/zstdruby/libzstd/dictBuilder/zdict.h +60 -5
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +9 -3
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +6 -0
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +6 -0
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +1 -5
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +12 -9
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +10 -10
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +20 -18
- data/ext/zstdruby/libzstd/zstd.h +109 -50
- data/lib/zstd-ruby/version.rb +1 -1
- metadata +4 -2
@@ -293,7 +293,7 @@ static dictItem ZDICT_analyzePos(
|
|
293
293
|
refinedEnd = refinedStart + selectedCount;
|
294
294
|
}
|
295
295
|
|
296
|
-
/* evaluate gain based on new
|
296
|
+
/* evaluate gain based on new dict */
|
297
297
|
start = refinedStart;
|
298
298
|
pos = suffix[refinedStart];
|
299
299
|
end = start;
|
@@ -341,7 +341,7 @@ static dictItem ZDICT_analyzePos(
|
|
341
341
|
for (i=MINMATCHLENGTH; i<=(int)maxLength; i++)
|
342
342
|
savings[i] = savings[i-1] + (lengthList[i] * (i-3));
|
343
343
|
|
344
|
-
DISPLAYLEVEL(4, "Selected
|
344
|
+
DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n",
|
345
345
|
(U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength);
|
346
346
|
|
347
347
|
solution.pos = (U32)pos;
|
@@ -581,7 +581,7 @@ static void ZDICT_fillNoise(void* buffer, size_t length)
|
|
581
581
|
|
582
582
|
typedef struct
|
583
583
|
{
|
584
|
-
|
584
|
+
ZSTD_CDict* dict; /* dictionary */
|
585
585
|
ZSTD_CCtx* zc; /* working context */
|
586
586
|
void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */
|
587
587
|
} EStats_ress_t;
|
@@ -597,8 +597,9 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
|
|
597
597
|
size_t cSize;
|
598
598
|
|
599
599
|
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
|
600
|
-
{ size_t const errorCode =
|
601
|
-
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning :
|
600
|
+
{ size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict);
|
601
|
+
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; }
|
602
|
+
|
602
603
|
}
|
603
604
|
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
|
604
605
|
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
|
@@ -697,7 +698,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
697
698
|
short litLengthNCount[MaxLL+1];
|
698
699
|
U32 repOffset[MAXREPOFFSET];
|
699
700
|
offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
|
700
|
-
EStats_ress_t esr;
|
701
|
+
EStats_ress_t esr = { NULL, NULL, NULL };
|
701
702
|
ZSTD_parameters params;
|
702
703
|
U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total;
|
703
704
|
size_t pos = 0, errorCode;
|
@@ -708,14 +709,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
708
709
|
|
709
710
|
/* init */
|
710
711
|
DEBUGLOG(4, "ZDICT_analyzeEntropy");
|
711
|
-
esr.ref = ZSTD_createCCtx();
|
712
|
-
esr.zc = ZSTD_createCCtx();
|
713
|
-
esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX);
|
714
|
-
if (!esr.ref || !esr.zc || !esr.workPlace) {
|
715
|
-
eSize = ERROR(memory_allocation);
|
716
|
-
DISPLAYLEVEL(1, "Not enough memory \n");
|
717
|
-
goto _cleanup;
|
718
|
-
}
|
719
712
|
if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionaryCreation_failed); goto _cleanup; } /* too large dictionary */
|
720
713
|
for (u=0; u<256; u++) countLit[u] = 1; /* any character must be described */
|
721
714
|
for (u=0; u<=offcodeMax; u++) offcodeCount[u] = 1;
|
@@ -726,12 +719,15 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
726
719
|
memset(bestRepOffset, 0, sizeof(bestRepOffset));
|
727
720
|
if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
|
728
721
|
params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
|
729
|
-
|
730
|
-
|
731
|
-
|
732
|
-
|
733
|
-
|
734
|
-
|
722
|
+
|
723
|
+
esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem);
|
724
|
+
esr.zc = ZSTD_createCCtx();
|
725
|
+
esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX);
|
726
|
+
if (!esr.dict || !esr.zc || !esr.workPlace) {
|
727
|
+
eSize = ERROR(memory_allocation);
|
728
|
+
DISPLAYLEVEL(1, "Not enough memory \n");
|
729
|
+
goto _cleanup;
|
730
|
+
}
|
735
731
|
|
736
732
|
/* collect stats on all samples */
|
737
733
|
for (u=0; u<nbFiles; u++) {
|
@@ -856,7 +852,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
856
852
|
eSize += 12;
|
857
853
|
|
858
854
|
_cleanup:
|
859
|
-
|
855
|
+
ZSTD_freeCDict(esr.dict);
|
860
856
|
ZSTD_freeCCtx(esr.zc);
|
861
857
|
free(esr.workPlace);
|
862
858
|
|
@@ -867,8 +863,8 @@ _cleanup:
|
|
867
863
|
|
868
864
|
size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
869
865
|
const void* customDictContent, size_t dictContentSize,
|
870
|
-
const void* samplesBuffer, const size_t* samplesSizes,
|
871
|
-
ZDICT_params_t params)
|
866
|
+
const void* samplesBuffer, const size_t* samplesSizes,
|
867
|
+
unsigned nbSamples, ZDICT_params_t params)
|
872
868
|
{
|
873
869
|
size_t hSize;
|
874
870
|
#define HBUFFSIZE 256 /* should prove large enough for all entropy headers */
|
@@ -914,9 +910,10 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
|
914
910
|
}
|
915
911
|
|
916
912
|
|
917
|
-
size_t ZDICT_addEntropyTablesFromBuffer_advanced(
|
918
|
-
|
919
|
-
|
913
|
+
static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
|
914
|
+
void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
915
|
+
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
916
|
+
ZDICT_params_t params)
|
920
917
|
{
|
921
918
|
int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
|
922
919
|
U32 const notificationLevel = params.notificationLevel;
|
@@ -947,7 +944,11 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
|
|
947
944
|
return MIN(dictBufferCapacity, hSize+dictContentSize);
|
948
945
|
}
|
949
946
|
|
950
|
-
|
947
|
+
/* Hidden declaration for dbio.c */
|
948
|
+
size_t ZDICT_trainFromBuffer_unsafe_legacy(
|
949
|
+
void* dictBuffer, size_t maxDictSize,
|
950
|
+
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
951
|
+
ZDICT_legacy_params_t params);
|
951
952
|
/*! ZDICT_trainFromBuffer_unsafe_legacy() :
|
952
953
|
* Warning : `samplesBuffer` must be followed by noisy guard band.
|
953
954
|
* @return : size of dictionary, or an error code which can be tested with ZDICT_isError()
|
@@ -991,8 +992,10 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(
|
|
991
992
|
U32 const pos = dictList[u].pos;
|
992
993
|
U32 const length = dictList[u].length;
|
993
994
|
U32 const printedLength = MIN(40, length);
|
994
|
-
if ((pos > samplesBuffSize) || ((pos + length) > samplesBuffSize))
|
995
|
+
if ((pos > samplesBuffSize) || ((pos + length) > samplesBuffSize)) {
|
996
|
+
free(dictList);
|
995
997
|
return ERROR(GENERIC); /* should never happen */
|
998
|
+
}
|
996
999
|
DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
|
997
1000
|
u, length, pos, dictList[u].savings);
|
998
1001
|
ZDICT_printHex((const char*)samplesBuffer+pos, printedLength);
|
@@ -1082,17 +1085,17 @@ size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity,
|
|
1082
1085
|
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
1083
1086
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)
|
1084
1087
|
{
|
1085
|
-
|
1088
|
+
ZDICT_fastCover_params_t params;
|
1086
1089
|
DEBUGLOG(3, "ZDICT_trainFromBuffer");
|
1087
1090
|
memset(¶ms, 0, sizeof(params));
|
1088
1091
|
params.d = 8;
|
1089
1092
|
params.steps = 4;
|
1090
1093
|
/* Default to level 6 since no compression level information is available */
|
1091
|
-
params.zParams.compressionLevel =
|
1094
|
+
params.zParams.compressionLevel = 3;
|
1092
1095
|
#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1)
|
1093
1096
|
params.zParams.notificationLevel = DEBUGLEVEL;
|
1094
1097
|
#endif
|
1095
|
-
return
|
1098
|
+
return ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, dictBufferCapacity,
|
1096
1099
|
samplesBuffer, samplesSizes, nbSamples,
|
1097
1100
|
¶ms);
|
1098
1101
|
}
|
@@ -39,7 +39,8 @@ extern "C" {
|
|
39
39
|
|
40
40
|
/*! ZDICT_trainFromBuffer():
|
41
41
|
* Train a dictionary from an array of samples.
|
42
|
-
* Redirect towards
|
42
|
+
* Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4,
|
43
|
+
* f=20, and accel=1.
|
43
44
|
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
44
45
|
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
45
46
|
* The resulting dictionary will be saved into `dictBuffer`.
|
@@ -52,7 +53,8 @@ extern "C" {
|
|
52
53
|
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
53
54
|
*/
|
54
55
|
ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
55
|
-
const void* samplesBuffer,
|
56
|
+
const void* samplesBuffer,
|
57
|
+
const size_t* samplesSizes, unsigned nbSamples);
|
56
58
|
|
57
59
|
|
58
60
|
/*====== Helper functions ======*/
|
@@ -84,11 +86,22 @@ typedef struct {
|
|
84
86
|
typedef struct {
|
85
87
|
unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
|
86
88
|
unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
|
87
|
-
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (
|
89
|
+
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
|
88
90
|
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
91
|
+
double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
|
89
92
|
ZDICT_params_t zParams;
|
90
93
|
} ZDICT_cover_params_t;
|
91
94
|
|
95
|
+
typedef struct {
|
96
|
+
unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
|
97
|
+
unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
|
98
|
+
unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/
|
99
|
+
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
|
100
|
+
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
101
|
+
double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
|
102
|
+
unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
|
103
|
+
ZDICT_params_t zParams;
|
104
|
+
} ZDICT_fastCover_params_t;
|
92
105
|
|
93
106
|
/*! ZDICT_trainFromBuffer_cover():
|
94
107
|
* Train a dictionary from an array of samples using the COVER algorithm.
|
@@ -115,9 +128,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
|
115
128
|
* dictionary constructed with those parameters is stored in `dictBuffer`.
|
116
129
|
*
|
117
130
|
* All of the parameters d, k, steps are optional.
|
118
|
-
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8
|
131
|
+
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
|
119
132
|
* if steps is zero it defaults to its default value.
|
120
|
-
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [
|
133
|
+
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
|
121
134
|
*
|
122
135
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
123
136
|
* or an error code, which can be tested with ZDICT_isError().
|
@@ -129,6 +142,48 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
|
129
142
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
130
143
|
ZDICT_cover_params_t* parameters);
|
131
144
|
|
145
|
+
/*! ZDICT_trainFromBuffer_fastCover():
|
146
|
+
* Train a dictionary from an array of samples using a modified version of COVER algorithm.
|
147
|
+
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
148
|
+
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
149
|
+
* d and k are required.
|
150
|
+
* All other parameters are optional, will use default values if not provided
|
151
|
+
* The resulting dictionary will be saved into `dictBuffer`.
|
152
|
+
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
153
|
+
* or an error code, which can be tested with ZDICT_isError().
|
154
|
+
* Note: ZDICT_trainFromBuffer_fastCover() requires about 1 bytes of memory for each input byte and additionally another 6 * 2^f bytes of memory .
|
155
|
+
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
|
156
|
+
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
|
157
|
+
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
158
|
+
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
159
|
+
*/
|
160
|
+
ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
|
161
|
+
size_t dictBufferCapacity, const void *samplesBuffer,
|
162
|
+
const size_t *samplesSizes, unsigned nbSamples,
|
163
|
+
ZDICT_fastCover_params_t parameters);
|
164
|
+
|
165
|
+
/*! ZDICT_optimizeTrainFromBuffer_fastCover():
|
166
|
+
* The same requirements as above hold for all the parameters except `parameters`.
|
167
|
+
* This function tries many parameter combinations (specifically, k and d combinations)
|
168
|
+
* and picks the best parameters. `*parameters` is filled with the best parameters found,
|
169
|
+
* dictionary constructed with those parameters is stored in `dictBuffer`.
|
170
|
+
* All of the parameters d, k, steps, f, and accel are optional.
|
171
|
+
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
|
172
|
+
* if steps is zero it defaults to its default value.
|
173
|
+
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
|
174
|
+
* If f is zero, default value of 20 is used.
|
175
|
+
* If accel is zero, default value of 1 is used.
|
176
|
+
*
|
177
|
+
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
178
|
+
* or an error code, which can be tested with ZDICT_isError().
|
179
|
+
* On success `*parameters` contains the parameters selected.
|
180
|
+
* Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 1 byte of memory for each input byte and additionally another 6 * 2^f bytes of memory for each thread.
|
181
|
+
*/
|
182
|
+
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
183
|
+
size_t dictBufferCapacity, const void* samplesBuffer,
|
184
|
+
const size_t* samplesSizes, unsigned nbSamples,
|
185
|
+
ZDICT_fastCover_params_t* parameters);
|
186
|
+
|
132
187
|
/*! ZDICT_finalizeDictionary():
|
133
188
|
* Given a custom content as a basis for dictionary, and a set of samples,
|
134
189
|
* finalize dictionary by adding headers and statistics.
|
@@ -668,11 +668,17 @@ static size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t
|
|
668
668
|
switch(srcSize)
|
669
669
|
{
|
670
670
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
671
|
+
/* fallthrough */
|
671
672
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
673
|
+
/* fallthrough */
|
672
674
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
675
|
+
/* fallthrough */
|
673
676
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
677
|
+
/* fallthrough */
|
674
678
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
679
|
+
/* fallthrough */
|
675
680
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
681
|
+
/* fallthrough */
|
676
682
|
default:;
|
677
683
|
}
|
678
684
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
@@ -1458,7 +1464,7 @@ unsigned ZSTDv01_isError(size_t code) { return ERR_isError(code); }
|
|
1458
1464
|
* Decompression code
|
1459
1465
|
**************************************************************/
|
1460
1466
|
|
1461
|
-
size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
1467
|
+
static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
1462
1468
|
{
|
1463
1469
|
const BYTE* const in = (const BYTE* const)src;
|
1464
1470
|
BYTE headerFlags;
|
@@ -1511,7 +1517,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
|
1511
1517
|
}
|
1512
1518
|
|
1513
1519
|
|
1514
|
-
size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
1520
|
+
static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
1515
1521
|
void* dst, size_t maxDstSize,
|
1516
1522
|
const BYTE** litStart, size_t* litSize,
|
1517
1523
|
const void* src, size_t srcSize)
|
@@ -1563,7 +1569,7 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
1563
1569
|
}
|
1564
1570
|
|
1565
1571
|
|
1566
|
-
size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
1572
|
+
static size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
1567
1573
|
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
|
1568
1574
|
const void* src, size_t srcSize)
|
1569
1575
|
{
|
@@ -399,11 +399,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
399
399
|
switch(srcSize)
|
400
400
|
{
|
401
401
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
402
|
+
/* fallthrough */
|
402
403
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
404
|
+
/* fallthrough */
|
403
405
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
406
|
+
/* fallthrough */
|
404
407
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
408
|
+
/* fallthrough */
|
405
409
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
410
|
+
/* fallthrough */
|
406
411
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
412
|
+
/* fallthrough */
|
407
413
|
default:;
|
408
414
|
}
|
409
415
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
@@ -402,11 +402,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
402
402
|
switch(srcSize)
|
403
403
|
{
|
404
404
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
405
|
+
/* fallthrough */
|
405
406
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
407
|
+
/* fallthrough */
|
406
408
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
409
|
+
/* fallthrough */
|
407
410
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
411
|
+
/* fallthrough */
|
408
412
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
413
|
+
/* fallthrough */
|
409
414
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
415
|
+
/* fallthrough */
|
410
416
|
default:;
|
411
417
|
}
|
412
418
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
@@ -1093,6 +1093,7 @@ static size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, un
|
|
1093
1093
|
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
1094
1094
|
|
1095
1095
|
/* Init, lay down lowprob symbols */
|
1096
|
+
memset(tableDecode, 0, sizeof(FSE_DECODE_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
|
1096
1097
|
DTableH.tableLog = (U16)tableLog;
|
1097
1098
|
for (s=0; s<=maxSymbolValue; s++)
|
1098
1099
|
{
|
@@ -3621,8 +3622,3 @@ size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDs
|
|
3621
3622
|
|
3622
3623
|
ZSTD_DCtx* ZSTDv04_createDCtx(void) { return ZSTD_createDCtx(); }
|
3623
3624
|
size_t ZSTDv04_freeDCtx(ZSTD_DCtx* dctx) { return ZSTD_freeDCtx(dctx); }
|
3624
|
-
|
3625
|
-
size_t ZSTDv04_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
|
3626
|
-
{
|
3627
|
-
return ZSTD_getFrameParams(params, src, srcSize);
|
3628
|
-
}
|
@@ -1224,6 +1224,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns
|
|
1224
1224
|
if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
1225
1225
|
|
1226
1226
|
/* Init, lay down lowprob symbols */
|
1227
|
+
memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
|
1227
1228
|
DTableH.tableLog = (U16)tableLog;
|
1228
1229
|
for (s=0; s<=maxSymbolValue; s++) {
|
1229
1230
|
if (normalizedCounter[s]==-1) {
|
@@ -2658,6 +2659,7 @@ struct ZSTDv05_DCtx_s
|
|
2658
2659
|
BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
|
2659
2660
|
}; /* typedef'd to ZSTDv05_DCtx within "zstd_static.h" */
|
2660
2661
|
|
2662
|
+
size_t ZSTDv05_sizeofDCtx (void); /* Hidden declaration */
|
2661
2663
|
size_t ZSTDv05_sizeofDCtx (void) { return sizeof(ZSTDv05_DCtx); }
|
2662
2664
|
|
2663
2665
|
size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx)
|
@@ -2822,7 +2824,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
|
|
2822
2824
|
}
|
2823
2825
|
|
2824
2826
|
|
2825
|
-
size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
2827
|
+
static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
2826
2828
|
{
|
2827
2829
|
const BYTE* const in = (const BYTE* const)src;
|
2828
2830
|
BYTE headerFlags;
|
@@ -2845,6 +2847,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|
2845
2847
|
|
2846
2848
|
static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
2847
2849
|
{
|
2850
|
+
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
2848
2851
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
2849
2852
|
memcpy(dst, src, srcSize);
|
2850
2853
|
return srcSize;
|
@@ -2853,8 +2856,8 @@ static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src
|
|
2853
2856
|
|
2854
2857
|
/*! ZSTDv05_decodeLiteralsBlock() :
|
2855
2858
|
@return : nb of bytes read from src (< srcSize ) */
|
2856
|
-
size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
|
2857
|
-
|
2859
|
+
static size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
|
2860
|
+
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
2858
2861
|
{
|
2859
2862
|
const BYTE* const istart = (const BYTE*) src;
|
2860
2863
|
|
@@ -2988,7 +2991,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
|
|
2988
2991
|
}
|
2989
2992
|
|
2990
2993
|
|
2991
|
-
size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
2994
|
+
static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
2992
2995
|
FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
|
2993
2996
|
const void* src, size_t srcSize, U32 flagStaticTable)
|
2994
2997
|
{
|
@@ -3297,11 +3300,11 @@ static size_t ZSTDv05_decompressSequences(
|
|
3297
3300
|
BYTE* const ostart = (BYTE* const)dst;
|
3298
3301
|
BYTE* op = ostart;
|
3299
3302
|
BYTE* const oend = ostart + maxDstSize;
|
3300
|
-
size_t errorCode, dumpsLength;
|
3303
|
+
size_t errorCode, dumpsLength=0;
|
3301
3304
|
const BYTE* litPtr = dctx->litPtr;
|
3302
3305
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
3303
|
-
int nbSeq;
|
3304
|
-
const BYTE* dumps;
|
3306
|
+
int nbSeq=0;
|
3307
|
+
const BYTE* dumps = NULL;
|
3305
3308
|
U32* DTableLL = dctx->LLTable;
|
3306
3309
|
U32* DTableML = dctx->MLTable;
|
3307
3310
|
U32* DTableOffb = dctx->OffTable;
|
@@ -3410,10 +3413,10 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
|
|
3410
3413
|
BYTE* const oend = ostart + maxDstSize;
|
3411
3414
|
size_t remainingSize = srcSize;
|
3412
3415
|
blockProperties_t blockProperties;
|
3416
|
+
memset(&blockProperties, 0, sizeof(blockProperties));
|
3413
3417
|
|
3414
3418
|
/* Frame Header */
|
3415
|
-
{
|
3416
|
-
size_t frameHeaderSize;
|
3419
|
+
{ size_t frameHeaderSize;
|
3417
3420
|
if (srcSize < ZSTDv05_frameHeaderSize_min+ZSTDv05_blockHeaderSize) return ERROR(srcSize_wrong);
|
3418
3421
|
frameHeaderSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
|
3419
3422
|
if (ZSTDv05_isError(frameHeaderSize)) return frameHeaderSize;
|
@@ -1250,9 +1250,7 @@ const char* FSEv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
|
1250
1250
|
/* **************************************************************
|
1251
1251
|
* HUF Error Management
|
1252
1252
|
****************************************************************/
|
1253
|
-
unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
|
1254
|
-
|
1255
|
-
const char* HUFv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
1253
|
+
static unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
|
1256
1254
|
|
1257
1255
|
|
1258
1256
|
/*-**************************************************************
|
@@ -2823,7 +2821,8 @@ struct ZSTDv06_DCtx_s
|
|
2823
2821
|
BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
|
2824
2822
|
}; /* typedef'd to ZSTDv06_DCtx within "zstd_static.h" */
|
2825
2823
|
|
2826
|
-
size_t ZSTDv06_sizeofDCtx (void)
|
2824
|
+
size_t ZSTDv06_sizeofDCtx (void); /* Hidden declaration */
|
2825
|
+
size_t ZSTDv06_sizeofDCtx (void) { return sizeof(ZSTDv06_DCtx); }
|
2827
2826
|
|
2828
2827
|
size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx)
|
2829
2828
|
{
|
@@ -3022,7 +3021,7 @@ typedef struct
|
|
3022
3021
|
|
3023
3022
|
/*! ZSTDv06_getcBlockSize() :
|
3024
3023
|
* Provides the size of compressed block from block header `src` */
|
3025
|
-
size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
3024
|
+
static size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
3026
3025
|
{
|
3027
3026
|
const BYTE* const in = (const BYTE* const)src;
|
3028
3027
|
U32 cSize;
|
@@ -3041,6 +3040,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|
3041
3040
|
|
3042
3041
|
static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
3043
3042
|
{
|
3043
|
+
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
3044
3044
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
3045
3045
|
memcpy(dst, src, srcSize);
|
3046
3046
|
return srcSize;
|
@@ -3049,7 +3049,7 @@ static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* sr
|
|
3049
3049
|
|
3050
3050
|
/*! ZSTDv06_decodeLiteralsBlock() :
|
3051
3051
|
@return : nb of bytes read from src (< srcSize ) */
|
3052
|
-
size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
3052
|
+
static size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
3053
3053
|
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
3054
3054
|
{
|
3055
3055
|
const BYTE* const istart = (const BYTE*) src;
|
@@ -3183,7 +3183,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
|
3183
3183
|
@return : nb bytes read from src,
|
3184
3184
|
or an error code if it fails, testable with ZSTDv06_isError()
|
3185
3185
|
*/
|
3186
|
-
size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
3186
|
+
static size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
3187
3187
|
const void* src, size_t srcSize,
|
3188
3188
|
const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
|
3189
3189
|
{
|
@@ -3213,7 +3213,7 @@ size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLo
|
|
3213
3213
|
}
|
3214
3214
|
|
3215
3215
|
|
3216
|
-
size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
3216
|
+
static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
3217
3217
|
FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
|
3218
3218
|
const void* src, size_t srcSize)
|
3219
3219
|
{
|
@@ -3358,7 +3358,7 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
3358
3358
|
}
|
3359
3359
|
|
3360
3360
|
|
3361
|
-
size_t ZSTDv06_execSequence(BYTE* op,
|
3361
|
+
static size_t ZSTDv06_execSequence(BYTE* op,
|
3362
3362
|
BYTE* const oend, seq_t sequence,
|
3363
3363
|
const BYTE** litPtr, const BYTE* const litLimit,
|
3364
3364
|
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
@@ -4006,7 +4006,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|
4006
4006
|
if (ZSTDv06_isError(hSize)) return hSize;
|
4007
4007
|
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
4008
4008
|
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
|
4009
|
-
zbd->lhSize += iend-ip;
|
4009
|
+
zbd->lhSize += iend-ip;
|
4010
4010
|
*dstCapacityPtr = 0;
|
4011
4011
|
return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
|
4012
4012
|
}
|