zstd-ruby 1.3.0.0 → 1.3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/ext/zstdruby/libzstd/common/bitstream.h +40 -41
- data/ext/zstdruby/libzstd/common/compiler.h +85 -0
- data/ext/zstdruby/libzstd/common/error_private.c +8 -10
- data/ext/zstdruby/libzstd/common/error_private.h +4 -4
- data/ext/zstdruby/libzstd/common/fse.h +11 -5
- data/ext/zstdruby/libzstd/common/fse_decompress.c +3 -22
- data/ext/zstdruby/libzstd/common/huf.h +5 -6
- data/ext/zstdruby/libzstd/common/mem.h +6 -6
- data/ext/zstdruby/libzstd/common/pool.c +61 -27
- data/ext/zstdruby/libzstd/common/pool.h +10 -10
- data/ext/zstdruby/libzstd/common/threading.h +5 -6
- data/ext/zstdruby/libzstd/common/xxhash.c +28 -22
- data/ext/zstdruby/libzstd/common/zstd_common.c +4 -4
- data/ext/zstdruby/libzstd/common/zstd_errors.h +30 -32
- data/ext/zstdruby/libzstd/common/zstd_internal.h +57 -56
- data/ext/zstdruby/libzstd/compress/fse_compress.c +4 -22
- data/ext/zstdruby/libzstd/compress/huf_compress.c +4 -3
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +314 -304
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +118 -116
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +223 -156
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +10 -9
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +8 -24
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +95 -115
- data/ext/zstdruby/libzstd/deprecated/zbuff.h +4 -4
- data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +4 -5
- data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +4 -4
- data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +4 -4
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +7 -9
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +5 -5
- data/ext/zstdruby/libzstd/dictBuilder/zdict.h +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +8 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +5 -5
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +5 -5
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +5 -5
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +4 -4
- data/ext/zstdruby/libzstd/zstd.h +16 -14
- data/lib/zstd-ruby/version.rb +1 -1
- metadata +3 -2
@@ -1,10 +1,10 @@
|
|
1
|
-
|
1
|
+
/*
|
2
2
|
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
6
|
-
* LICENSE file in the root directory of this source tree
|
7
|
-
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
8
|
*/
|
9
9
|
|
10
10
|
#ifndef ZSTDMT_COMPRESS_H
|
@@ -15,10 +15,11 @@
|
|
15
15
|
#endif
|
16
16
|
|
17
17
|
|
18
|
-
/* Note :
|
19
|
-
*
|
20
|
-
*
|
21
|
-
*
|
18
|
+
/* Note : This is an internal API.
|
19
|
+
* Some methods are still exposed (ZSTDLIB_API),
|
20
|
+
* because it used to be the only way to invoke MT compression.
|
21
|
+
* Now, it's recommended to use ZSTD_compress_generic() instead.
|
22
|
+
* These methods will stop being exposed in a future version */
|
22
23
|
|
23
24
|
/* === Dependencies === */
|
24
25
|
#include <stddef.h> /* size_t */
|
@@ -67,7 +68,7 @@ ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
|
|
67
68
|
const void* src, size_t srcSize,
|
68
69
|
const ZSTD_CDict* cdict,
|
69
70
|
ZSTD_parameters const params,
|
70
|
-
unsigned
|
71
|
+
unsigned overlapLog);
|
71
72
|
|
72
73
|
ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
|
73
74
|
const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
|
@@ -32,38 +32,22 @@
|
|
32
32
|
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
33
33
|
****************************************************************** */
|
34
34
|
|
35
|
-
/* **************************************************************
|
36
|
-
* Compiler specifics
|
37
|
-
****************************************************************/
|
38
|
-
#ifdef _MSC_VER /* Visual Studio */
|
39
|
-
# define FORCE_INLINE static __forceinline
|
40
|
-
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
41
|
-
#else
|
42
|
-
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
43
|
-
# ifdef __GNUC__
|
44
|
-
# define FORCE_INLINE static inline __attribute__((always_inline))
|
45
|
-
# else
|
46
|
-
# define FORCE_INLINE static inline
|
47
|
-
# endif
|
48
|
-
# else
|
49
|
-
# define FORCE_INLINE static
|
50
|
-
# endif /* __STDC_VERSION__ */
|
51
|
-
#endif
|
52
|
-
|
53
|
-
|
54
35
|
/* **************************************************************
|
55
36
|
* Dependencies
|
56
37
|
****************************************************************/
|
57
38
|
#include <string.h> /* memcpy, memset */
|
58
39
|
#include "bitstream.h" /* BIT_* */
|
40
|
+
#include "compiler.h"
|
59
41
|
#include "fse.h" /* header compression */
|
60
42
|
#define HUF_STATIC_LINKING_ONLY
|
61
43
|
#include "huf.h"
|
44
|
+
#include "error_private.h"
|
62
45
|
|
63
46
|
|
64
47
|
/* **************************************************************
|
65
48
|
* Error Management
|
66
49
|
****************************************************************/
|
50
|
+
#define HUF_isError ERR_isError
|
67
51
|
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
68
52
|
|
69
53
|
|
@@ -180,7 +164,7 @@ static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, con
|
|
180
164
|
if (MEM_64bits()) \
|
181
165
|
HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
|
182
166
|
|
183
|
-
|
167
|
+
HINT_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
|
184
168
|
{
|
185
169
|
BYTE* const pStart = p;
|
186
170
|
|
@@ -639,7 +623,7 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
|
|
639
623
|
if (MEM_64bits()) \
|
640
624
|
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
641
625
|
|
642
|
-
|
626
|
+
HINT_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
|
643
627
|
{
|
644
628
|
BYTE* const pStart = p;
|
645
629
|
|
@@ -917,11 +901,11 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
|
|
917
901
|
* Tells which decoder is likely to decode faster,
|
918
902
|
* based on a set of pre-determined metrics.
|
919
903
|
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
|
920
|
-
* Assumption : 0 < cSrcSize
|
904
|
+
* Assumption : 0 < cSrcSize, dstSize <= 128 KB */
|
921
905
|
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
|
922
906
|
{
|
923
907
|
/* decoder timing evaluation */
|
924
|
-
U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16
|
908
|
+
U32 const Q = cSrcSize >= dstSize ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
|
925
909
|
U32 const D256 = (U32)(dstSize >> 8);
|
926
910
|
U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
|
927
911
|
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
|
@@ -977,7 +961,7 @@ size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
|
|
977
961
|
{
|
978
962
|
/* validation checks */
|
979
963
|
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
980
|
-
if (
|
964
|
+
if (cSrcSize == 0) return ERROR(corruption_detected);
|
981
965
|
|
982
966
|
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
983
967
|
return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize):
|
@@ -1,10 +1,10 @@
|
|
1
|
-
|
1
|
+
/*
|
2
2
|
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
6
|
-
* LICENSE file in the root directory of this source tree
|
7
|
-
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
8
|
*/
|
9
9
|
|
10
10
|
|
@@ -53,15 +53,6 @@
|
|
53
53
|
# include "zstd_legacy.h"
|
54
54
|
#endif
|
55
55
|
|
56
|
-
#if defined(_MSC_VER) && !defined(_M_IA64) /* _mm_prefetch() is not defined for ia64 */
|
57
|
-
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
|
58
|
-
# define ZSTD_PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
|
59
|
-
#elif defined(__GNUC__)
|
60
|
-
# define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
|
61
|
-
#else
|
62
|
-
# define ZSTD_PREFETCH(ptr) /* disabled */
|
63
|
-
#endif
|
64
|
-
|
65
56
|
|
66
57
|
/*-*************************************
|
67
58
|
* Errors
|
@@ -95,7 +86,7 @@ typedef struct {
|
|
95
86
|
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
|
96
87
|
U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
|
97
88
|
U32 rep[ZSTD_REP_NUM];
|
98
|
-
}
|
89
|
+
} ZSTD_entropyDTables_t;
|
99
90
|
|
100
91
|
struct ZSTD_DCtx_s
|
101
92
|
{
|
@@ -103,7 +94,7 @@ struct ZSTD_DCtx_s
|
|
103
94
|
const FSE_DTable* MLTptr;
|
104
95
|
const FSE_DTable* OFTptr;
|
105
96
|
const HUF_DTable* HUFptr;
|
106
|
-
|
97
|
+
ZSTD_entropyDTables_t entropy;
|
107
98
|
const void* previousDstEnd; /* detect continuity */
|
108
99
|
const void* base; /* start of current segment */
|
109
100
|
const void* vBase; /* virtual start of previous segment if it was just before current one */
|
@@ -304,15 +295,18 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
|
|
304
295
|
return ZSTD_skippableHeaderSize; /* magic number + frame length */
|
305
296
|
memset(zfhPtr, 0, sizeof(*zfhPtr));
|
306
297
|
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
307
|
-
zfhPtr->
|
298
|
+
zfhPtr->frameType = ZSTD_skippableFrame;
|
299
|
+
zfhPtr->windowSize = 0;
|
308
300
|
return 0;
|
309
301
|
}
|
310
302
|
return ERROR(prefix_unknown);
|
311
303
|
}
|
312
304
|
|
313
305
|
/* ensure there is enough `srcSize` to fully read/decode frame header */
|
314
|
-
{
|
315
|
-
|
306
|
+
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
|
307
|
+
if (srcSize < fhsize) return fhsize;
|
308
|
+
zfhPtr->headerSize = (U32)fhsize;
|
309
|
+
}
|
316
310
|
|
317
311
|
{ BYTE const fhdByte = ip[4];
|
318
312
|
size_t pos = 5;
|
@@ -320,24 +314,23 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
|
|
320
314
|
U32 const checksumFlag = (fhdByte>>2)&1;
|
321
315
|
U32 const singleSegment = (fhdByte>>5)&1;
|
322
316
|
U32 const fcsID = fhdByte>>6;
|
323
|
-
|
324
|
-
U32 windowSize = 0;
|
317
|
+
U64 windowSize = 0;
|
325
318
|
U32 dictID = 0;
|
326
|
-
U64 frameContentSize =
|
319
|
+
U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
|
327
320
|
if ((fhdByte & 0x08) != 0)
|
328
|
-
return ERROR(frameParameter_unsupported);
|
321
|
+
return ERROR(frameParameter_unsupported); /* reserved bits, must be zero */
|
322
|
+
|
329
323
|
if (!singleSegment) {
|
330
324
|
BYTE const wlByte = ip[pos++];
|
331
325
|
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
332
326
|
if (windowLog > ZSTD_WINDOWLOG_MAX)
|
333
327
|
return ERROR(frameParameter_windowTooLarge);
|
334
|
-
windowSize = (
|
328
|
+
windowSize = (1ULL << windowLog);
|
335
329
|
windowSize += (windowSize >> 3) * (wlByte&7);
|
336
330
|
}
|
337
|
-
|
338
331
|
switch(dictIDSizeCode)
|
339
332
|
{
|
340
|
-
default:
|
333
|
+
default: assert(0); /* impossible */
|
341
334
|
case 0 : break;
|
342
335
|
case 1 : dictID = ip[pos]; pos++; break;
|
343
336
|
case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
|
@@ -345,14 +338,15 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
|
|
345
338
|
}
|
346
339
|
switch(fcsID)
|
347
340
|
{
|
348
|
-
default:
|
341
|
+
default: assert(0); /* impossible */
|
349
342
|
case 0 : if (singleSegment) frameContentSize = ip[pos]; break;
|
350
343
|
case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
|
351
344
|
case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
|
352
345
|
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
353
346
|
}
|
354
|
-
if (
|
355
|
-
|
347
|
+
if (singleSegment) windowSize = frameContentSize;
|
348
|
+
|
349
|
+
zfhPtr->frameType = ZSTD_frame;
|
356
350
|
zfhPtr->frameContentSize = frameContentSize;
|
357
351
|
zfhPtr->windowSize = windowSize;
|
358
352
|
zfhPtr->dictID = dictID;
|
@@ -362,10 +356,10 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
|
|
362
356
|
}
|
363
357
|
|
364
358
|
/** ZSTD_getFrameContentSize() :
|
365
|
-
*
|
366
|
-
*
|
367
|
-
*
|
368
|
-
*
|
359
|
+
* compatible with legacy mode
|
360
|
+
* @return : decompressed size of the single frame pointed to be `src` if known, otherwise
|
361
|
+
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
|
362
|
+
* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
|
369
363
|
unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
|
370
364
|
{
|
371
365
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
@@ -374,17 +368,14 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
|
|
374
368
|
return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
|
375
369
|
}
|
376
370
|
#endif
|
377
|
-
{ ZSTD_frameHeader
|
378
|
-
if (ZSTD_getFrameHeader(&
|
379
|
-
|
380
|
-
|
371
|
+
{ ZSTD_frameHeader zfh;
|
372
|
+
if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)
|
373
|
+
return ZSTD_CONTENTSIZE_ERROR;
|
374
|
+
if (zfh.frameType == ZSTD_skippableFrame) {
|
381
375
|
return 0;
|
382
|
-
} else if (fParams.frameContentSize != 0) {
|
383
|
-
return fParams.frameContentSize;
|
384
376
|
} else {
|
385
|
-
return
|
386
|
-
|
387
|
-
}
|
377
|
+
return zfh.frameContentSize;
|
378
|
+
} }
|
388
379
|
}
|
389
380
|
|
390
381
|
/** ZSTD_findDecompressedSize() :
|
@@ -442,7 +433,8 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
|
442
433
|
* compatible with legacy mode
|
443
434
|
* @return : decompressed size if known, 0 otherwise
|
444
435
|
note : 0 can mean any of the following :
|
445
|
-
-
|
436
|
+
- frame content is empty
|
437
|
+
- decompressed size field is not present in frame header
|
446
438
|
- frame header unknown / not supported
|
447
439
|
- frame header not complete (`srcSize` too small) */
|
448
440
|
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
|
@@ -460,19 +452,13 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
|
|
460
452
|
size_t const result = ZSTD_getFrameHeader(&(dctx->fParams), src, headerSize);
|
461
453
|
if (ZSTD_isError(result)) return result; /* invalid header */
|
462
454
|
if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */
|
463
|
-
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID))
|
455
|
+
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID))
|
456
|
+
return ERROR(dictionary_wrong);
|
464
457
|
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
|
465
458
|
return 0;
|
466
459
|
}
|
467
460
|
|
468
461
|
|
469
|
-
typedef struct
|
470
|
-
{
|
471
|
-
blockType_e blockType;
|
472
|
-
U32 lastBlock;
|
473
|
-
U32 origSize;
|
474
|
-
} blockProperties_t;
|
475
|
-
|
476
462
|
/*! ZSTD_getcBlockSize() :
|
477
463
|
* Provides the size of compressed block from block header `src` */
|
478
464
|
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
|
@@ -958,7 +944,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
|
|
958
944
|
}
|
959
945
|
|
960
946
|
|
961
|
-
|
947
|
+
HINT_INLINE
|
962
948
|
size_t ZSTD_execSequence(BYTE* op,
|
963
949
|
BYTE* const oend, seq_t sequence,
|
964
950
|
const BYTE** litPtr, const BYTE* const litLimit,
|
@@ -1102,7 +1088,7 @@ static size_t ZSTD_decompressSequences(
|
|
1102
1088
|
}
|
1103
1089
|
|
1104
1090
|
|
1105
|
-
|
1091
|
+
FORCE_INLINE_TEMPLATE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int const longOffsets)
|
1106
1092
|
{
|
1107
1093
|
seq_t seq;
|
1108
1094
|
|
@@ -1202,7 +1188,7 @@ static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, unsigned const window
|
|
1202
1188
|
}
|
1203
1189
|
}
|
1204
1190
|
|
1205
|
-
|
1191
|
+
HINT_INLINE
|
1206
1192
|
size_t ZSTD_execSequenceLong(BYTE* op,
|
1207
1193
|
BYTE* const oend, seq_t sequence,
|
1208
1194
|
const BYTE** litPtr, const BYTE* const litLimit,
|
@@ -1338,7 +1324,7 @@ static size_t ZSTD_decompressSequencesLong(
|
|
1338
1324
|
seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize32);
|
1339
1325
|
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
|
1340
1326
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
1341
|
-
|
1327
|
+
PREFETCH(sequence.match);
|
1342
1328
|
sequences[seqNb&STOSEQ_MASK] = sequence;
|
1343
1329
|
op += oneSeqSize;
|
1344
1330
|
}
|
@@ -1440,28 +1426,26 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
|
|
1440
1426
|
size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
1441
1427
|
{
|
1442
1428
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
1443
|
-
if (ZSTD_isLegacy(src, srcSize))
|
1429
|
+
if (ZSTD_isLegacy(src, srcSize))
|
1430
|
+
return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
|
1444
1431
|
#endif
|
1445
|
-
if (srcSize >= ZSTD_skippableHeaderSize
|
1446
|
-
|
1432
|
+
if ( (srcSize >= ZSTD_skippableHeaderSize)
|
1433
|
+
&& (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) {
|
1447
1434
|
return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + 4);
|
1448
1435
|
} else {
|
1449
1436
|
const BYTE* ip = (const BYTE*)src;
|
1450
1437
|
const BYTE* const ipstart = ip;
|
1451
1438
|
size_t remainingSize = srcSize;
|
1452
|
-
ZSTD_frameHeader
|
1453
|
-
|
1454
|
-
size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
|
1455
|
-
if (ZSTD_isError(headerSize)) return headerSize;
|
1439
|
+
ZSTD_frameHeader zfh;
|
1456
1440
|
|
1457
|
-
/* Frame Header */
|
1458
|
-
{ size_t const ret = ZSTD_getFrameHeader(&
|
1441
|
+
/* Extract Frame Header */
|
1442
|
+
{ size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
|
1459
1443
|
if (ZSTD_isError(ret)) return ret;
|
1460
1444
|
if (ret > 0) return ERROR(srcSize_wrong);
|
1461
1445
|
}
|
1462
1446
|
|
1463
|
-
ip += headerSize;
|
1464
|
-
remainingSize -= headerSize;
|
1447
|
+
ip += zfh.headerSize;
|
1448
|
+
remainingSize -= zfh.headerSize;
|
1465
1449
|
|
1466
1450
|
/* Loop on each block */
|
1467
1451
|
while (1) {
|
@@ -1469,7 +1453,8 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
|
1469
1453
|
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
1470
1454
|
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
1471
1455
|
|
1472
|
-
if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
|
1456
|
+
if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
|
1457
|
+
return ERROR(srcSize_wrong);
|
1473
1458
|
|
1474
1459
|
ip += ZSTD_blockHeaderSize + cBlockSize;
|
1475
1460
|
remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
|
@@ -1477,7 +1462,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
|
1477
1462
|
if (blockProperties.lastBlock) break;
|
1478
1463
|
}
|
1479
1464
|
|
1480
|
-
if (
|
1465
|
+
if (zfh.checksumFlag) { /* Final frame content checksum */
|
1481
1466
|
if (remainingSize < 4) return ERROR(srcSize_wrong);
|
1482
1467
|
ip += 4;
|
1483
1468
|
remainingSize -= 4;
|
@@ -1490,8 +1475,8 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
|
1490
1475
|
/*! ZSTD_decompressFrame() :
|
1491
1476
|
* @dctx must be properly initialized */
|
1492
1477
|
static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
1493
|
-
|
1494
|
-
|
1478
|
+
void* dst, size_t dstCapacity,
|
1479
|
+
const void** srcPtr, size_t *srcSizePtr)
|
1495
1480
|
{
|
1496
1481
|
const BYTE* ip = (const BYTE*)(*srcPtr);
|
1497
1482
|
BYTE* const ostart = (BYTE* const)dst;
|
@@ -1500,13 +1485,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|
1500
1485
|
size_t remainingSize = *srcSizePtr;
|
1501
1486
|
|
1502
1487
|
/* check */
|
1503
|
-
if (remainingSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize)
|
1488
|
+
if (remainingSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize)
|
1489
|
+
return ERROR(srcSize_wrong);
|
1504
1490
|
|
1505
1491
|
/* Frame Header */
|
1506
1492
|
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix);
|
1507
1493
|
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
|
1508
|
-
if (remainingSize < frameHeaderSize+ZSTD_blockHeaderSize)
|
1509
|
-
|
1494
|
+
if (remainingSize < frameHeaderSize+ZSTD_blockHeaderSize)
|
1495
|
+
return ERROR(srcSize_wrong);
|
1496
|
+
CHECK_F( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) );
|
1510
1497
|
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
1511
1498
|
}
|
1512
1499
|
|
@@ -1538,14 +1525,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|
1538
1525
|
}
|
1539
1526
|
|
1540
1527
|
if (ZSTD_isError(decodedSize)) return decodedSize;
|
1541
|
-
if (dctx->fParams.checksumFlag)
|
1528
|
+
if (dctx->fParams.checksumFlag)
|
1529
|
+
XXH64_update(&dctx->xxhState, op, decodedSize);
|
1542
1530
|
op += decodedSize;
|
1543
1531
|
ip += cBlockSize;
|
1544
1532
|
remainingSize -= cBlockSize;
|
1545
1533
|
if (blockProperties.lastBlock) break;
|
1546
1534
|
}
|
1547
1535
|
|
1548
|
-
if (dctx->fParams.checksumFlag) {
|
1536
|
+
if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
|
1549
1537
|
U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
|
1550
1538
|
U32 checkRead;
|
1551
1539
|
if (remainingSize<4) return ERROR(checksum_wrong);
|
@@ -1567,17 +1555,13 @@ static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict);
|
|
1567
1555
|
static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
1568
1556
|
void* dst, size_t dstCapacity,
|
1569
1557
|
const void* src, size_t srcSize,
|
1570
|
-
const void
|
1558
|
+
const void* dict, size_t dictSize,
|
1571
1559
|
const ZSTD_DDict* ddict)
|
1572
1560
|
{
|
1573
1561
|
void* const dststart = dst;
|
1562
|
+
assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */
|
1574
1563
|
|
1575
1564
|
if (ddict) {
|
1576
|
-
if (dict) {
|
1577
|
-
/* programmer error, these two cases should be mutually exclusive */
|
1578
|
-
return ERROR(GENERIC);
|
1579
|
-
}
|
1580
|
-
|
1581
1565
|
dict = ZSTD_DDictDictContent(ddict);
|
1582
1566
|
dictSize = ZSTD_DDictDictSize(ddict);
|
1583
1567
|
}
|
@@ -1590,7 +1574,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|
1590
1574
|
size_t decodedSize;
|
1591
1575
|
size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
|
1592
1576
|
if (ZSTD_isError(frameSize)) return frameSize;
|
1593
|
-
/* legacy support is
|
1577
|
+
/* legacy support is not compatible with static dctx */
|
1594
1578
|
if (dctx->staticSize) return ERROR(memory_allocation);
|
1595
1579
|
|
1596
1580
|
decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
|
@@ -1613,16 +1597,13 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|
1613
1597
|
return ERROR(srcSize_wrong);
|
1614
1598
|
skippableSize = MEM_readLE32((const BYTE *)src + 4) +
|
1615
1599
|
ZSTD_skippableHeaderSize;
|
1616
|
-
if (srcSize < skippableSize)
|
1617
|
-
return ERROR(srcSize_wrong);
|
1618
|
-
}
|
1600
|
+
if (srcSize < skippableSize) return ERROR(srcSize_wrong);
|
1619
1601
|
|
1620
1602
|
src = (const BYTE *)src + skippableSize;
|
1621
1603
|
srcSize -= skippableSize;
|
1622
1604
|
continue;
|
1623
|
-
} else {
|
1624
|
-
return ERROR(prefix_unknown);
|
1625
1605
|
}
|
1606
|
+
return ERROR(prefix_unknown);
|
1626
1607
|
}
|
1627
1608
|
|
1628
1609
|
if (ddict) {
|
@@ -1638,12 +1619,11 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|
1638
1619
|
{ const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
|
1639
1620
|
&src, &srcSize);
|
1640
1621
|
if (ZSTD_isError(res)) return res;
|
1641
|
-
/*
|
1642
|
-
* already */
|
1622
|
+
/* no need to bound check, ZSTD_decompressFrame already has */
|
1643
1623
|
dst = (BYTE*)dst + res;
|
1644
1624
|
dstCapacity -= res;
|
1645
1625
|
}
|
1646
|
-
}
|
1626
|
+
} /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
|
1647
1627
|
|
1648
1628
|
if (srcSize) return ERROR(srcSize_wrong); /* input not entirely consumed */
|
1649
1629
|
|
@@ -1742,7 +1722,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|
1742
1722
|
return 0;
|
1743
1723
|
}
|
1744
1724
|
dctx->expected = 0; /* not necessary to copy more */
|
1745
|
-
|
1725
|
+
/* fall-through */
|
1746
1726
|
case ZSTDds_decodeFrameHeader:
|
1747
1727
|
assert(src != NULL);
|
1748
1728
|
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
|
@@ -1853,7 +1833,7 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
|
|
1853
1833
|
/* ZSTD_loadEntropy() :
|
1854
1834
|
* dict : must point at beginning of a valid zstd dictionary
|
1855
1835
|
* @return : size of entropy tables read */
|
1856
|
-
static size_t ZSTD_loadEntropy(
|
1836
|
+
static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, const void* const dict, size_t const dictSize)
|
1857
1837
|
{
|
1858
1838
|
const BYTE* dictPtr = (const BYTE*)dict;
|
1859
1839
|
const BYTE* const dictEnd = dictPtr + dictSize;
|
@@ -1931,8 +1911,9 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
|
|
1931
1911
|
|
1932
1912
|
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
1933
1913
|
{
|
1934
|
-
CHECK_F(ZSTD_decompressBegin(dctx));
|
1935
|
-
if (dict && dictSize)
|
1914
|
+
CHECK_F( ZSTD_decompressBegin(dctx) );
|
1915
|
+
if (dict && dictSize)
|
1916
|
+
CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted);
|
1936
1917
|
return 0;
|
1937
1918
|
}
|
1938
1919
|
|
@@ -1943,7 +1924,7 @@ struct ZSTD_DDict_s {
|
|
1943
1924
|
void* dictBuffer;
|
1944
1925
|
const void* dictContent;
|
1945
1926
|
size_t dictSize;
|
1946
|
-
|
1927
|
+
ZSTD_entropyDTables_t entropy;
|
1947
1928
|
U32 dictID;
|
1948
1929
|
U32 entropyPresent;
|
1949
1930
|
ZSTD_customMem cMem;
|
@@ -1961,7 +1942,7 @@ static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict)
|
|
1961
1942
|
|
1962
1943
|
size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
|
1963
1944
|
{
|
1964
|
-
CHECK_F(ZSTD_decompressBegin(dstDCtx));
|
1945
|
+
CHECK_F( ZSTD_decompressBegin(dstDCtx) );
|
1965
1946
|
if (ddict) { /* support begin on NULL */
|
1966
1947
|
dstDCtx->dictID = ddict->dictID;
|
1967
1948
|
dstDCtx->base = ddict->dictContent;
|
@@ -2142,7 +2123,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
|
|
2142
2123
|
* ZSTD_getFrameHeader(), which will provide a more precise error code. */
|
2143
2124
|
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
|
2144
2125
|
{
|
2145
|
-
ZSTD_frameHeader zfp = { 0
|
2126
|
+
ZSTD_frameHeader zfp = { 0, 0, ZSTD_frame, 0, 0, 0 };
|
2146
2127
|
size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
|
2147
2128
|
if (ZSTD_isError(hError)) return 0;
|
2148
2129
|
return zfp.dictID;
|
@@ -2237,7 +2218,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
|
|
2237
2218
|
{
|
2238
2219
|
switch(paramType)
|
2239
2220
|
{
|
2240
|
-
default : return ERROR(
|
2221
|
+
default : return ERROR(parameter_unsupported);
|
2241
2222
|
case DStream_p_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
|
2242
2223
|
}
|
2243
2224
|
return 0;
|
@@ -2254,16 +2235,19 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize)
|
|
2254
2235
|
size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
|
2255
2236
|
size_t const inBuffSize = blockSize; /* no block can be larger */
|
2256
2237
|
size_t const outBuffSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
|
2257
|
-
return
|
2238
|
+
return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
|
2258
2239
|
}
|
2259
2240
|
|
2260
2241
|
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
|
2261
2242
|
{
|
2262
|
-
|
2263
|
-
|
2243
|
+
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
|
2244
|
+
ZSTD_frameHeader zfh;
|
2245
|
+
size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
|
2264
2246
|
if (ZSTD_isError(err)) return err;
|
2265
2247
|
if (err>0) return ERROR(srcSize_wrong);
|
2266
|
-
|
2248
|
+
if (zfh.windowSize > windowSizeMax)
|
2249
|
+
return ERROR(frameParameter_windowTooLarge);
|
2250
|
+
return ZSTD_estimateDStreamSize((size_t)zfh.windowSize);
|
2267
2251
|
}
|
2268
2252
|
|
2269
2253
|
|
@@ -2314,16 +2298,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
2314
2298
|
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
|
2315
2299
|
/* legacy support is incompatible with static dctx */
|
2316
2300
|
if (zds->staticSize) return ERROR(memory_allocation);
|
2317
|
-
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext,
|
2318
|
-
|
2301
|
+
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext,
|
2302
|
+
zds->previousLegacyVersion, legacyVersion,
|
2303
|
+
dict, dictSize));
|
2319
2304
|
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
|
2320
|
-
return ZSTD_decompressLegacyStream(zds->legacyContext,
|
2321
|
-
} else {
|
2322
|
-
return hSize; /* error */
|
2305
|
+
return ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
|
2323
2306
|
}
|
2324
|
-
#else
|
2325
|
-
return hSize;
|
2326
2307
|
#endif
|
2308
|
+
return hSize; /* error */
|
2327
2309
|
}
|
2328
2310
|
if (hSize != 0) { /* need more input */
|
2329
2311
|
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
|
@@ -2374,8 +2356,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
2374
2356
|
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
|
2375
2357
|
|
2376
2358
|
/* Adapt buffer sizes to frame header instructions */
|
2377
|
-
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
|
2378
|
-
size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
|
2359
|
+
{ size_t const blockSize = (size_t)(MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_MAX));
|
2360
|
+
size_t const neededOutSize = (size_t)(zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2);
|
2379
2361
|
zds->blockSize = blockSize;
|
2380
2362
|
if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) {
|
2381
2363
|
size_t const bufferSize = blockSize + neededOutSize;
|
@@ -2400,7 +2382,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
2400
2382
|
zds->outBuffSize = neededOutSize;
|
2401
2383
|
} }
|
2402
2384
|
zds->streamStage = zdss_read;
|
2403
|
-
/*
|
2385
|
+
/* fall-through */
|
2404
2386
|
|
2405
2387
|
case zdss_read:
|
2406
2388
|
DEBUGLOG(5, "stage zdss_read");
|
@@ -2425,8 +2407,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
2425
2407
|
} }
|
2426
2408
|
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
|
2427
2409
|
zds->streamStage = zdss_load;
|
2428
|
-
/*
|
2429
|
-
|
2410
|
+
/* fall-through */
|
2430
2411
|
case zdss_load:
|
2431
2412
|
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
|
2432
2413
|
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
|
@@ -2448,8 +2429,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
2448
2429
|
zds->outEnd = zds->outStart + decodedSize;
|
2449
2430
|
} }
|
2450
2431
|
zds->streamStage = zdss_flush;
|
2451
|
-
/*
|
2452
|
-
|
2432
|
+
/* fall-through */
|
2453
2433
|
case zdss_flush:
|
2454
2434
|
{ size_t const toFlushSize = zds->outEnd - zds->outStart;
|
2455
2435
|
size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
|