zstd-ruby 1.3.8.0 → 1.4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +6 -5
- data/README.md +1 -1
- data/ext/zstdruby/libzstd/Makefile +7 -3
- data/ext/zstdruby/libzstd/README.md +4 -2
- data/ext/zstdruby/libzstd/common/compiler.h +1 -1
- data/ext/zstdruby/libzstd/common/fse.h +1 -1
- data/ext/zstdruby/libzstd/common/threading.c +2 -2
- data/ext/zstdruby/libzstd/common/xxhash.c +2 -2
- data/ext/zstdruby/libzstd/common/zstd_internal.h +55 -2
- data/ext/zstdruby/libzstd/compress/fse_compress.c +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +423 -296
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +14 -11
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +203 -124
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +27 -11
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +41 -49
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +43 -26
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +4 -4
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +257 -164
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +51 -47
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +7 -0
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +58 -13
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +29 -0
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +25 -13
- data/ext/zstdruby/libzstd/dictBuilder/zdict.h +18 -8
- data/ext/zstdruby/libzstd/dll/example/build_package.bat +3 -2
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +42 -12
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +32 -7
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +12 -7
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +31 -12
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +12 -7
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +32 -12
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +12 -7
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +32 -12
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +12 -7
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +32 -7
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +12 -7
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +36 -8
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +10 -5
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +40 -9
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +10 -5
- data/ext/zstdruby/libzstd/zstd.h +689 -542
- data/lib/zstd-ruby/version.rb +1 -1
- data/zstd-ruby.gemspec +1 -1
- metadata +6 -7
- data/ext/zstdruby/libzstd/dll/libzstd.def +0 -87
@@ -38,6 +38,35 @@ typedef struct {
|
|
38
38
|
U32 score;
|
39
39
|
} COVER_segment_t;
|
40
40
|
|
41
|
+
/**
|
42
|
+
*Number of epochs and size of each epoch.
|
43
|
+
*/
|
44
|
+
typedef struct {
|
45
|
+
U32 num;
|
46
|
+
U32 size;
|
47
|
+
} COVER_epoch_info_t;
|
48
|
+
|
49
|
+
/**
|
50
|
+
* Computes the number of epochs and the size of each epoch.
|
51
|
+
* We will make sure that each epoch gets at least 10 * k bytes.
|
52
|
+
*
|
53
|
+
* The COVER algorithms divide the data up into epochs of equal size and
|
54
|
+
* select one segment from each epoch.
|
55
|
+
*
|
56
|
+
* @param maxDictSize The maximum allowed dictionary size.
|
57
|
+
* @param nbDmers The number of dmers we are training on.
|
58
|
+
* @param k The parameter k (segment size).
|
59
|
+
* @param passes The target number of passes over the dmer corpus.
|
60
|
+
* More passes means a better dictionary.
|
61
|
+
*/
|
62
|
+
COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize, U32 nbDmers,
|
63
|
+
U32 k, U32 passes);
|
64
|
+
|
65
|
+
/**
|
66
|
+
* Warns the user when their corpus is too small.
|
67
|
+
*/
|
68
|
+
void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel);
|
69
|
+
|
41
70
|
/**
|
42
71
|
* Checks total compressed size of a dictionary
|
43
72
|
*/
|
@@ -132,7 +132,7 @@ typedef struct {
|
|
132
132
|
*
|
133
133
|
* Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
|
134
134
|
*
|
135
|
-
* Once the dmer with hash value d is in the
|
135
|
+
* Once the dmer with hash value d is in the dictionary we set F(d) = 0.
|
136
136
|
*/
|
137
137
|
static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
|
138
138
|
U32 *freqs, U32 begin, U32 end,
|
@@ -161,7 +161,7 @@ static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
|
|
161
161
|
/* Get hash value of current dmer */
|
162
162
|
const size_t idx = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
|
163
163
|
|
164
|
-
/* Add frequency of this index to score if this is the first
|
164
|
+
/* Add frequency of this index to score if this is the first occurrence of index in active segment */
|
165
165
|
if (segmentFreqs[idx] == 0) {
|
166
166
|
activeSegment.score += freqs[idx];
|
167
167
|
}
|
@@ -386,29 +386,35 @@ FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
|
|
386
386
|
{
|
387
387
|
BYTE *const dict = (BYTE *)dictBuffer;
|
388
388
|
size_t tail = dictBufferCapacity;
|
389
|
-
/* Divide the data
|
390
|
-
|
391
|
-
|
392
|
-
const
|
393
|
-
|
389
|
+
/* Divide the data into epochs. We will select one segment from each epoch. */
|
390
|
+
const COVER_epoch_info_t epochs = COVER_computeEpochs(
|
391
|
+
(U32)dictBufferCapacity, (U32)ctx->nbDmers, parameters.k, 1);
|
392
|
+
const size_t maxZeroScoreRun = 10;
|
393
|
+
size_t zeroScoreRun = 0;
|
394
394
|
size_t epoch;
|
395
395
|
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
|
396
|
-
epochs,
|
396
|
+
(U32)epochs.num, (U32)epochs.size);
|
397
397
|
/* Loop through the epochs until there are no more segments or the dictionary
|
398
398
|
* is full.
|
399
399
|
*/
|
400
|
-
for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
|
401
|
-
const U32 epochBegin = (U32)(epoch *
|
402
|
-
const U32 epochEnd = epochBegin +
|
400
|
+
for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs.num) {
|
401
|
+
const U32 epochBegin = (U32)(epoch * epochs.size);
|
402
|
+
const U32 epochEnd = epochBegin + epochs.size;
|
403
403
|
size_t segmentSize;
|
404
404
|
/* Select a segment */
|
405
405
|
COVER_segment_t segment = FASTCOVER_selectSegment(
|
406
406
|
ctx, freqs, epochBegin, epochEnd, parameters, segmentFreqs);
|
407
407
|
|
408
|
-
/* If the segment covers no dmers, then we are out of content
|
408
|
+
/* If the segment covers no dmers, then we are out of content.
|
409
|
+
* There may be new content in other epochs, for continue for some time.
|
410
|
+
*/
|
409
411
|
if (segment.score == 0) {
|
410
|
-
|
412
|
+
if (++zeroScoreRun >= maxZeroScoreRun) {
|
413
|
+
break;
|
414
|
+
}
|
415
|
+
continue;
|
411
416
|
}
|
417
|
+
zeroScoreRun = 0;
|
412
418
|
|
413
419
|
/* Trim the segment if necessary and if it is too small then we are done */
|
414
420
|
segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
|
@@ -564,6 +570,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
|
|
564
570
|
DISPLAYLEVEL(1, "Failed to initialize context\n");
|
565
571
|
return ERROR(GENERIC);
|
566
572
|
}
|
573
|
+
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel);
|
567
574
|
/* Build the dictionary */
|
568
575
|
DISPLAYLEVEL(2, "Building dictionary\n");
|
569
576
|
{
|
@@ -616,6 +623,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
|
|
616
623
|
unsigned k;
|
617
624
|
COVER_best_t best;
|
618
625
|
POOL_ctx *pool = NULL;
|
626
|
+
int warned = 0;
|
619
627
|
/* Checks */
|
620
628
|
if (splitPoint <= 0 || splitPoint > 1) {
|
621
629
|
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n");
|
@@ -664,6 +672,10 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
|
|
664
672
|
POOL_free(pool);
|
665
673
|
return ERROR(GENERIC);
|
666
674
|
}
|
675
|
+
if (!warned) {
|
676
|
+
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel);
|
677
|
+
warned = 1;
|
678
|
+
}
|
667
679
|
/* Loop through k reusing the same context */
|
668
680
|
for (k = kMinK; k <= kMaxK; k += kStepSize) {
|
669
681
|
/* Prepare the arguments */
|
@@ -46,7 +46,12 @@ extern "C" {
|
|
46
46
|
* The resulting dictionary will be saved into `dictBuffer`.
|
47
47
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
48
48
|
* or an error code, which can be tested with ZDICT_isError().
|
49
|
-
* Note:
|
49
|
+
* Note: Dictionary training will fail if there are not enough samples to construct a
|
50
|
+
* dictionary, or if most of the samples are too small (< 8 bytes being the lower limit).
|
51
|
+
* If dictionary training fails, you should use zstd without a dictionary, as the dictionary
|
52
|
+
* would've been ineffective anyways. If you believe your samples would benefit from a dictionary
|
53
|
+
* please open an issue with details, and we can look into it.
|
54
|
+
* Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB.
|
50
55
|
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
|
51
56
|
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
|
52
57
|
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
@@ -110,6 +115,7 @@ typedef struct {
|
|
110
115
|
* The resulting dictionary will be saved into `dictBuffer`.
|
111
116
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
112
117
|
* or an error code, which can be tested with ZDICT_isError().
|
118
|
+
* See ZDICT_trainFromBuffer() for details on failure modes.
|
113
119
|
* Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte.
|
114
120
|
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
|
115
121
|
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
|
@@ -133,8 +139,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
|
133
139
|
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
|
134
140
|
*
|
135
141
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
136
|
-
*
|
137
|
-
*
|
142
|
+
* or an error code, which can be tested with ZDICT_isError().
|
143
|
+
* On success `*parameters` contains the parameters selected.
|
144
|
+
* See ZDICT_trainFromBuffer() for details on failure modes.
|
138
145
|
* Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
|
139
146
|
*/
|
140
147
|
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
@@ -151,7 +158,8 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
|
151
158
|
* The resulting dictionary will be saved into `dictBuffer`.
|
152
159
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
153
160
|
* or an error code, which can be tested with ZDICT_isError().
|
154
|
-
*
|
161
|
+
* See ZDICT_trainFromBuffer() for details on failure modes.
|
162
|
+
* Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory.
|
155
163
|
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
|
156
164
|
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
|
157
165
|
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
@@ -175,9 +183,10 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
|
|
175
183
|
* If accel is zero, default value of 1 is used.
|
176
184
|
*
|
177
185
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
178
|
-
*
|
179
|
-
*
|
180
|
-
*
|
186
|
+
* or an error code, which can be tested with ZDICT_isError().
|
187
|
+
* On success `*parameters` contains the parameters selected.
|
188
|
+
* See ZDICT_trainFromBuffer() for details on failure modes.
|
189
|
+
* Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
|
181
190
|
*/
|
182
191
|
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
183
192
|
size_t dictBufferCapacity, const void* samplesBuffer,
|
@@ -195,7 +204,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
|
195
204
|
* maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes.
|
196
205
|
*
|
197
206
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
|
198
|
-
*
|
207
|
+
* or an error code, which can be tested by ZDICT_isError().
|
199
208
|
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
|
200
209
|
* Note 2: dictBuffer and dictContent can overlap
|
201
210
|
*/
|
@@ -219,6 +228,7 @@ typedef struct {
|
|
219
228
|
* `parameters` is optional and can be provided with values set to 0 to mean "default".
|
220
229
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
221
230
|
* or an error code, which can be tested with ZDICT_isError().
|
231
|
+
* See ZDICT_trainFromBuffer() for details on failure modes.
|
222
232
|
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
|
223
233
|
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
|
224
234
|
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
@@ -6,14 +6,15 @@ COPY programs\datagen.h bin\example\
|
|
6
6
|
COPY programs\util.h bin\example\
|
7
7
|
COPY programs\platform.h bin\example\
|
8
8
|
COPY lib\common\mem.h bin\example\
|
9
|
-
COPY lib\common\zstd_errors.h bin\example\
|
10
9
|
COPY lib\common\zstd_internal.h bin\example\
|
11
10
|
COPY lib\common\error_private.h bin\example\
|
12
11
|
COPY lib\common\xxhash.h bin\example\
|
13
|
-
COPY lib\zstd.h bin\include\
|
14
12
|
COPY lib\libzstd.a bin\static\libzstd_static.lib
|
15
13
|
COPY lib\dll\libzstd.* bin\dll\
|
16
14
|
COPY lib\dll\example\Makefile bin\example\
|
17
15
|
COPY lib\dll\example\fullbench-dll.* bin\example\
|
18
16
|
COPY lib\dll\example\README.md bin\
|
17
|
+
COPY lib\zstd.h bin\include\
|
18
|
+
COPY lib\common\zstd_errors.h bin\include\
|
19
|
+
COPY lib\dictBuilder\zdict.h bin\include\
|
19
20
|
COPY programs\zstd.exe bin\zstd.exe
|
@@ -20,7 +20,7 @@ extern "C" {
|
|
20
20
|
***************************************/
|
21
21
|
#include "mem.h" /* MEM_STATIC */
|
22
22
|
#include "error_private.h" /* ERROR */
|
23
|
-
#include "
|
23
|
+
#include "zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
|
24
24
|
|
25
25
|
#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
|
26
26
|
# undef ZSTD_LEGACY_SUPPORT
|
@@ -178,43 +178,73 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
|
|
178
178
|
}
|
179
179
|
}
|
180
180
|
|
181
|
-
MEM_STATIC
|
182
|
-
size_t compressedSize)
|
181
|
+
MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize)
|
183
182
|
{
|
184
|
-
|
183
|
+
ZSTD_frameSizeInfo frameSizeInfo;
|
184
|
+
U32 const version = ZSTD_isLegacy(src, srcSize);
|
185
185
|
switch(version)
|
186
186
|
{
|
187
187
|
#if (ZSTD_LEGACY_SUPPORT <= 1)
|
188
188
|
case 1 :
|
189
|
-
|
189
|
+
ZSTDv01_findFrameSizeInfoLegacy(src, srcSize,
|
190
|
+
&frameSizeInfo.compressedSize,
|
191
|
+
&frameSizeInfo.decompressedBound);
|
192
|
+
break;
|
190
193
|
#endif
|
191
194
|
#if (ZSTD_LEGACY_SUPPORT <= 2)
|
192
195
|
case 2 :
|
193
|
-
|
196
|
+
ZSTDv02_findFrameSizeInfoLegacy(src, srcSize,
|
197
|
+
&frameSizeInfo.compressedSize,
|
198
|
+
&frameSizeInfo.decompressedBound);
|
199
|
+
break;
|
194
200
|
#endif
|
195
201
|
#if (ZSTD_LEGACY_SUPPORT <= 3)
|
196
202
|
case 3 :
|
197
|
-
|
203
|
+
ZSTDv03_findFrameSizeInfoLegacy(src, srcSize,
|
204
|
+
&frameSizeInfo.compressedSize,
|
205
|
+
&frameSizeInfo.decompressedBound);
|
206
|
+
break;
|
198
207
|
#endif
|
199
208
|
#if (ZSTD_LEGACY_SUPPORT <= 4)
|
200
209
|
case 4 :
|
201
|
-
|
210
|
+
ZSTDv04_findFrameSizeInfoLegacy(src, srcSize,
|
211
|
+
&frameSizeInfo.compressedSize,
|
212
|
+
&frameSizeInfo.decompressedBound);
|
213
|
+
break;
|
202
214
|
#endif
|
203
215
|
#if (ZSTD_LEGACY_SUPPORT <= 5)
|
204
216
|
case 5 :
|
205
|
-
|
217
|
+
ZSTDv05_findFrameSizeInfoLegacy(src, srcSize,
|
218
|
+
&frameSizeInfo.compressedSize,
|
219
|
+
&frameSizeInfo.decompressedBound);
|
220
|
+
break;
|
206
221
|
#endif
|
207
222
|
#if (ZSTD_LEGACY_SUPPORT <= 6)
|
208
223
|
case 6 :
|
209
|
-
|
224
|
+
ZSTDv06_findFrameSizeInfoLegacy(src, srcSize,
|
225
|
+
&frameSizeInfo.compressedSize,
|
226
|
+
&frameSizeInfo.decompressedBound);
|
227
|
+
break;
|
210
228
|
#endif
|
211
229
|
#if (ZSTD_LEGACY_SUPPORT <= 7)
|
212
230
|
case 7 :
|
213
|
-
|
231
|
+
ZSTDv07_findFrameSizeInfoLegacy(src, srcSize,
|
232
|
+
&frameSizeInfo.compressedSize,
|
233
|
+
&frameSizeInfo.decompressedBound);
|
234
|
+
break;
|
214
235
|
#endif
|
215
236
|
default :
|
216
|
-
|
237
|
+
frameSizeInfo.compressedSize = ERROR(prefix_unknown);
|
238
|
+
frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
|
239
|
+
break;
|
217
240
|
}
|
241
|
+
return frameSizeInfo;
|
242
|
+
}
|
243
|
+
|
244
|
+
MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src, size_t srcSize)
|
245
|
+
{
|
246
|
+
ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfoLegacy(src, srcSize);
|
247
|
+
return frameSizeInfo.compressedSize;
|
218
248
|
}
|
219
249
|
|
220
250
|
MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
|
@@ -1336,6 +1336,8 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51E; /* 3rd version : seqNb header
|
|
1336
1336
|
#define LITERAL_NOENTROPY 63
|
1337
1337
|
#define COMMAND_NOENTROPY 7 /* to remove */
|
1338
1338
|
|
1339
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
1340
|
+
|
1339
1341
|
static const size_t ZSTD_blockHeaderSize = 3;
|
1340
1342
|
static const size_t ZSTD_frameHeaderSize = 4;
|
1341
1343
|
|
@@ -1757,7 +1759,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
1757
1759
|
BYTE* const base, BYTE* const oend)
|
1758
1760
|
{
|
1759
1761
|
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
1760
|
-
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /*
|
1762
|
+
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
1761
1763
|
const BYTE* const ostart = op;
|
1762
1764
|
const size_t litLength = sequence.litLength;
|
1763
1765
|
BYTE* const endMatch = op + litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
@@ -1999,36 +2001,59 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|
1999
2001
|
return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
2000
2002
|
}
|
2001
2003
|
|
2002
|
-
|
2004
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
2005
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
2006
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
2007
|
+
{
|
2008
|
+
*cSize = ret;
|
2009
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
2010
|
+
}
|
2011
|
+
|
2012
|
+
void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
2003
2013
|
{
|
2004
2014
|
const BYTE* ip = (const BYTE*)src;
|
2005
2015
|
size_t remainingSize = srcSize;
|
2016
|
+
size_t nbBlocks = 0;
|
2006
2017
|
U32 magicNumber;
|
2007
2018
|
blockProperties_t blockProperties;
|
2008
2019
|
|
2009
2020
|
/* Frame Header */
|
2010
|
-
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize)
|
2021
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
|
2022
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
2023
|
+
return;
|
2024
|
+
}
|
2011
2025
|
magicNumber = ZSTD_readBE32(src);
|
2012
|
-
if (magicNumber != ZSTD_magicNumber)
|
2026
|
+
if (magicNumber != ZSTD_magicNumber) {
|
2027
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
2028
|
+
return;
|
2029
|
+
}
|
2013
2030
|
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
2014
2031
|
|
2015
2032
|
/* Loop on each block */
|
2016
2033
|
while (1)
|
2017
2034
|
{
|
2018
2035
|
size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties);
|
2019
|
-
if (ZSTDv01_isError(blockSize))
|
2036
|
+
if (ZSTDv01_isError(blockSize)) {
|
2037
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, blockSize);
|
2038
|
+
return;
|
2039
|
+
}
|
2020
2040
|
|
2021
2041
|
ip += ZSTD_blockHeaderSize;
|
2022
2042
|
remainingSize -= ZSTD_blockHeaderSize;
|
2023
|
-
if (blockSize > remainingSize)
|
2043
|
+
if (blockSize > remainingSize) {
|
2044
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
2045
|
+
return;
|
2046
|
+
}
|
2024
2047
|
|
2025
2048
|
if (blockSize == 0) break; /* bt_end */
|
2026
2049
|
|
2027
2050
|
ip += blockSize;
|
2028
2051
|
remainingSize -= blockSize;
|
2052
|
+
nbBlocks++;
|
2029
2053
|
}
|
2030
2054
|
|
2031
|
-
|
2055
|
+
*cSize = ip - (const BYTE*)src;
|
2056
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
2032
2057
|
}
|
2033
2058
|
|
2034
2059
|
/*******************************
|
@@ -35,13 +35,18 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
|
|
35
35
|
size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
|
36
36
|
const void* src, size_t compressedSize);
|
37
37
|
|
38
|
-
/**
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
38
|
+
/**
|
39
|
+
ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format
|
40
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
41
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
42
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
43
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
44
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
45
|
+
|
46
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
47
|
+
*/
|
48
|
+
void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
49
|
+
size_t* cSize, unsigned long long* dBound);
|
45
50
|
|
46
51
|
/**
|
47
52
|
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
|
@@ -2728,6 +2728,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
2728
2728
|
#define LITERAL_NOENTROPY 63
|
2729
2729
|
#define COMMAND_NOENTROPY 7 /* to remove */
|
2730
2730
|
|
2731
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
2732
|
+
|
2731
2733
|
static const size_t ZSTD_blockHeaderSize = 3;
|
2732
2734
|
static const size_t ZSTD_frameHeaderSize = 4;
|
2733
2735
|
|
@@ -3096,7 +3098,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3096
3098
|
BYTE* const base, BYTE* const oend)
|
3097
3099
|
{
|
3098
3100
|
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
3099
|
-
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /*
|
3101
|
+
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
3100
3102
|
const BYTE* const ostart = op;
|
3101
3103
|
BYTE* const oLitEnd = op + sequence.litLength;
|
3102
3104
|
BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
@@ -3312,37 +3314,59 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|
3312
3314
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
3313
3315
|
}
|
3314
3316
|
|
3315
|
-
|
3317
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
3318
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
3319
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
3316
3320
|
{
|
3321
|
+
*cSize = ret;
|
3322
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
3323
|
+
}
|
3317
3324
|
|
3325
|
+
void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
3326
|
+
{
|
3318
3327
|
const BYTE* ip = (const BYTE*)src;
|
3319
3328
|
size_t remainingSize = srcSize;
|
3329
|
+
size_t nbBlocks = 0;
|
3320
3330
|
U32 magicNumber;
|
3321
3331
|
blockProperties_t blockProperties;
|
3322
3332
|
|
3323
3333
|
/* Frame Header */
|
3324
|
-
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize)
|
3334
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
|
3335
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3336
|
+
return;
|
3337
|
+
}
|
3325
3338
|
magicNumber = MEM_readLE32(src);
|
3326
|
-
if (magicNumber != ZSTD_magicNumber)
|
3339
|
+
if (magicNumber != ZSTD_magicNumber) {
|
3340
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
3341
|
+
return;
|
3342
|
+
}
|
3327
3343
|
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
3328
3344
|
|
3329
3345
|
/* Loop on each block */
|
3330
3346
|
while (1)
|
3331
3347
|
{
|
3332
3348
|
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
3333
|
-
if (ZSTD_isError(cBlockSize))
|
3349
|
+
if (ZSTD_isError(cBlockSize)) {
|
3350
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
3351
|
+
return;
|
3352
|
+
}
|
3334
3353
|
|
3335
3354
|
ip += ZSTD_blockHeaderSize;
|
3336
3355
|
remainingSize -= ZSTD_blockHeaderSize;
|
3337
|
-
if (cBlockSize > remainingSize)
|
3356
|
+
if (cBlockSize > remainingSize) {
|
3357
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3358
|
+
return;
|
3359
|
+
}
|
3338
3360
|
|
3339
3361
|
if (cBlockSize == 0) break; /* bt_end */
|
3340
3362
|
|
3341
3363
|
ip += cBlockSize;
|
3342
3364
|
remainingSize -= cBlockSize;
|
3365
|
+
nbBlocks++;
|
3343
3366
|
}
|
3344
3367
|
|
3345
|
-
|
3368
|
+
*cSize = ip - (const BYTE*)src;
|
3369
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
3346
3370
|
}
|
3347
3371
|
|
3348
3372
|
/*******************************
|
@@ -3458,11 +3482,6 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
|
3458
3482
|
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
3459
3483
|
}
|
3460
3484
|
|
3461
|
-
size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
|
3462
|
-
{
|
3463
|
-
return ZSTD_findFrameCompressedSize(src, compressedSize);
|
3464
|
-
}
|
3465
|
-
|
3466
3485
|
ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
|
3467
3486
|
{
|
3468
3487
|
return (ZSTDv02_Dctx*)ZSTD_createDCtx();
|