extzstd 0.0.3.CONCEPT → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +39 -0
- data/LICENSE +6 -6
- data/README.md +26 -45
- data/contrib/zstd/CHANGELOG +555 -0
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/CONTRIBUTING.md +392 -0
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/LICENSE +13 -9
- data/contrib/zstd/Makefile +414 -0
- data/contrib/zstd/README.md +170 -45
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +289 -0
- data/contrib/zstd/lib/BUCK +234 -0
- data/contrib/zstd/lib/Makefile +354 -0
- data/contrib/zstd/lib/README.md +179 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +170 -130
- data/contrib/zstd/lib/common/compiler.h +175 -0
- data/contrib/zstd/lib/common/cpu.h +215 -0
- data/contrib/zstd/lib/common/debug.c +24 -0
- data/contrib/zstd/lib/common/debug.h +114 -0
- data/contrib/zstd/{common → lib/common}/entropy_common.c +79 -94
- data/contrib/zstd/lib/common/error_private.c +55 -0
- data/contrib/zstd/lib/common/error_private.h +80 -0
- data/contrib/zstd/{common → lib/common}/fse.h +153 -93
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +37 -82
- data/contrib/zstd/lib/common/huf.h +340 -0
- data/contrib/zstd/{common → lib/common}/mem.h +154 -78
- data/contrib/zstd/lib/common/pool.c +344 -0
- data/contrib/zstd/lib/common/pool.h +84 -0
- data/contrib/zstd/lib/common/threading.c +121 -0
- data/contrib/zstd/lib/common/threading.h +155 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +85 -75
- data/contrib/zstd/{common → lib/common}/xxhash.h +85 -73
- data/contrib/zstd/lib/common/zstd_common.c +83 -0
- data/contrib/zstd/lib/common/zstd_errors.h +94 -0
- data/contrib/zstd/lib/common/zstd_internal.h +447 -0
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +194 -303
- data/contrib/zstd/lib/compress/hist.c +183 -0
- data/contrib/zstd/lib/compress/hist.h +75 -0
- data/contrib/zstd/lib/compress/huf_compress.c +798 -0
- data/contrib/zstd/lib/compress/zstd_compress.c +4278 -0
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +1125 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +419 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +845 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +525 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +1138 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +67 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +619 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +110 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +1200 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.c +2143 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +192 -0
- data/contrib/zstd/lib/decompress/huf_decompress.c +1248 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +1885 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1432 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +189 -0
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +86 -69
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1236 -0
- data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +5 -5
- data/contrib/zstd/lib/dictBuilder/fastcover.c +757 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +437 -347
- data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
- data/contrib/zstd/lib/legacy/zstd_legacy.h +415 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +272 -292
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +26 -32
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +162 -392
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +26 -32
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +162 -391
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +27 -33
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +195 -604
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +26 -32
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +300 -575
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +22 -31
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +165 -592
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +54 -67
- data/contrib/zstd/lib/legacy/zstd_v07.c +4541 -0
- data/contrib/zstd/lib/legacy/zstd_v07.h +187 -0
- data/contrib/zstd/lib/libzstd.pc.in +15 -0
- data/contrib/zstd/lib/zstd.h +2090 -0
- data/ext/depend +2 -0
- data/ext/extconf.rb +18 -5
- data/ext/extzstd.c +296 -214
- data/ext/extzstd.h +81 -36
- data/ext/extzstd_nogvls.h +0 -117
- data/ext/extzstd_stream.c +622 -0
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +11 -0
- data/ext/zstd_compress.c +15 -0
- data/ext/zstd_decompress.c +6 -0
- data/ext/zstd_dictbuilder.c +10 -0
- data/ext/zstd_dictbuilder_fastcover.c +3 -0
- data/ext/zstd_legacy_v01.c +3 -1
- data/ext/zstd_legacy_v02.c +3 -1
- data/ext/zstd_legacy_v03.c +3 -1
- data/ext/zstd_legacy_v04.c +3 -1
- data/ext/zstd_legacy_v05.c +3 -1
- data/ext/zstd_legacy_v06.c +3 -1
- data/ext/zstd_legacy_v07.c +3 -0
- data/gemstub.rb +27 -21
- data/lib/extzstd.rb +82 -161
- data/lib/extzstd/version.rb +1 -1
- data/test/test_basic.rb +19 -6
- metadata +127 -59
- data/contrib/zstd/common/error_private.h +0 -125
- data/contrib/zstd/common/error_public.h +0 -77
- data/contrib/zstd/common/huf.h +0 -228
- data/contrib/zstd/common/zstd.h +0 -475
- data/contrib/zstd/common/zstd_common.c +0 -91
- data/contrib/zstd/common/zstd_internal.h +0 -238
- data/contrib/zstd/compress/huf_compress.c +0 -577
- data/contrib/zstd/compress/zbuff_compress.c +0 -327
- data/contrib/zstd/compress/zstd_compress.c +0 -3074
- data/contrib/zstd/compress/zstd_opt.h +0 -1046
- data/contrib/zstd/decompress/huf_decompress.c +0 -894
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -294
- data/contrib/zstd/decompress/zstd_decompress.c +0 -1362
- data/contrib/zstd/dictBuilder/zdict.h +0 -113
- data/contrib/zstd/legacy/zstd_legacy.h +0 -140
- data/ext/extzstd_buffered.c +0 -265
- data/ext/zstd_amalgam.c +0 -18
@@ -1,43 +1,20 @@
|
|
1
|
-
/*
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
* Redistributions of source code must retain the above copyright
|
13
|
-
notice, this list of conditions and the following disclaimer.
|
14
|
-
* Redistributions in binary form must reproduce the above
|
15
|
-
copyright notice, this list of conditions and the following disclaimer
|
16
|
-
in the documentation and/or other materials provided with the
|
17
|
-
distribution.
|
18
|
-
|
19
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
-
|
31
|
-
You can contact the author at :
|
32
|
-
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
33
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
34
|
-
****************************************************************** */
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
35
11
|
|
36
12
|
/******************************************
|
37
13
|
* Includes
|
38
14
|
******************************************/
|
39
15
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
40
16
|
#include "zstd_v01.h"
|
17
|
+
#include "../common/error_private.h"
|
41
18
|
|
42
19
|
|
43
20
|
/******************************************
|
@@ -164,11 +141,15 @@ typedef struct
|
|
164
141
|
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
|
165
142
|
#else
|
166
143
|
# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
167
|
-
#
|
168
|
-
#
|
144
|
+
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
145
|
+
# ifdef __GNUC__
|
146
|
+
# define FORCE_INLINE static inline __attribute__((always_inline))
|
147
|
+
# else
|
148
|
+
# define FORCE_INLINE static inline
|
149
|
+
# endif
|
169
150
|
# else
|
170
|
-
# define FORCE_INLINE static
|
171
|
-
# endif
|
151
|
+
# define FORCE_INLINE static
|
152
|
+
# endif /* __STDC_VERSION__ */
|
172
153
|
#endif
|
173
154
|
|
174
155
|
|
@@ -225,7 +206,7 @@ typedef signed long long S64;
|
|
225
206
|
#ifndef FSE_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
226
207
|
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
227
208
|
# define FSE_FORCE_MEMORY_ACCESS 2
|
228
|
-
# elif defined(__INTEL_COMPILER) || \
|
209
|
+
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
229
210
|
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
230
211
|
# define FSE_FORCE_MEMORY_ACCESS 1
|
231
212
|
# endif
|
@@ -276,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
|
|
276
257
|
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
277
258
|
}
|
278
259
|
|
279
|
-
#endif
|
260
|
+
#endif /* FSE_FORCE_MEMORY_ACCESS */
|
280
261
|
|
281
262
|
static U16 FSE_readLE16(const void* memPtr)
|
282
263
|
{
|
@@ -358,14 +339,14 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
|
358
339
|
/****************************************************************
|
359
340
|
* Internal functions
|
360
341
|
****************************************************************/
|
361
|
-
FORCE_INLINE unsigned FSE_highbit32 (
|
342
|
+
FORCE_INLINE unsigned FSE_highbit32 (U32 val)
|
362
343
|
{
|
363
344
|
# if defined(_MSC_VER) /* Visual */
|
364
345
|
unsigned long r;
|
365
346
|
_BitScanReverse ( &r, val );
|
366
347
|
return (unsigned) r;
|
367
348
|
# elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */
|
368
|
-
return
|
349
|
+
return __builtin_clz (val) ^ 31;
|
369
350
|
# else /* Software version */
|
370
351
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
371
352
|
U32 v = val;
|
@@ -687,11 +668,17 @@ static size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t
|
|
687
668
|
switch(srcSize)
|
688
669
|
{
|
689
670
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
671
|
+
/* fallthrough */
|
690
672
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
673
|
+
/* fallthrough */
|
691
674
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
675
|
+
/* fallthrough */
|
692
676
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
677
|
+
/* fallthrough */
|
693
678
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
679
|
+
/* fallthrough */
|
694
680
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
681
|
+
/* fallthrough */
|
695
682
|
default:;
|
696
683
|
}
|
697
684
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
@@ -978,13 +965,16 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
|
|
978
965
|
U32 weightTotal;
|
979
966
|
U32 maxBits;
|
980
967
|
const BYTE* ip = (const BYTE*) src;
|
981
|
-
size_t iSize
|
968
|
+
size_t iSize;
|
982
969
|
size_t oSize;
|
983
970
|
U32 n;
|
984
971
|
U32 nextRankStart;
|
985
972
|
void* ptr = DTable+1;
|
986
973
|
HUF_DElt* const dt = (HUF_DElt*)ptr;
|
987
974
|
|
975
|
+
if (!srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
976
|
+
iSize = ip[0];
|
977
|
+
|
988
978
|
FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */
|
989
979
|
//memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */
|
990
980
|
if (iSize >= 128) /* special header */
|
@@ -1025,6 +1015,7 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
|
|
1025
1015
|
rankVal[huffWeight[n]]++;
|
1026
1016
|
weightTotal += (1 << huffWeight[n]) >> 1;
|
1027
1017
|
}
|
1018
|
+
if (weightTotal == 0) return (size_t)-FSE_ERROR_corruptionDetected;
|
1028
1019
|
|
1029
1020
|
/* get last non-null symbol weight (implied, total must be 2^n) */
|
1030
1021
|
maxBits = FSE_highbit32(weightTotal) + 1;
|
@@ -1082,99 +1073,102 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
1082
1073
|
const void* cSrc, size_t cSrcSize,
|
1083
1074
|
const U16* DTable)
|
1084
1075
|
{
|
1085
|
-
|
1086
|
-
BYTE* op = ostart;
|
1087
|
-
BYTE* const omax = op + maxDstSize;
|
1088
|
-
BYTE* const olimit = omax-15;
|
1089
|
-
|
1090
|
-
const void* ptr = DTable;
|
1091
|
-
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
1092
|
-
const U32 dtLog = DTable[0];
|
1093
|
-
size_t errorCode;
|
1094
|
-
U32 reloadStatus;
|
1095
|
-
|
1096
|
-
/* Init */
|
1097
|
-
|
1098
|
-
const U16* jumpTable = (const U16*)cSrc;
|
1099
|
-
const size_t length1 = FSE_readLE16(jumpTable);
|
1100
|
-
const size_t length2 = FSE_readLE16(jumpTable+1);
|
1101
|
-
const size_t length3 = FSE_readLE16(jumpTable+2);
|
1102
|
-
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
|
1103
|
-
const char* const start1 = (const char*)(cSrc) + 6;
|
1104
|
-
const char* const start2 = start1 + length1;
|
1105
|
-
const char* const start3 = start2 + length2;
|
1106
|
-
const char* const start4 = start3 + length3;
|
1107
|
-
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
|
1108
|
-
|
1109
|
-
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
1110
|
-
|
1111
|
-
errorCode = FSE_initDStream(&bitD1, start1, length1);
|
1112
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1113
|
-
errorCode = FSE_initDStream(&bitD2, start2, length2);
|
1114
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1115
|
-
errorCode = FSE_initDStream(&bitD3, start3, length3);
|
1116
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1117
|
-
errorCode = FSE_initDStream(&bitD4, start4, length4);
|
1118
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1119
|
-
|
1120
|
-
reloadStatus=FSE_reloadDStream(&bitD2);
|
1121
|
-
|
1122
|
-
/* 16 symbols per loop */
|
1123
|
-
for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
|
1124
|
-
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
|
1076
|
+
if (cSrcSize < 6) return (size_t)-FSE_ERROR_srcSize_wrong;
|
1125
1077
|
{
|
1126
|
-
|
1127
|
-
op
|
1128
|
-
|
1129
|
-
|
1130
|
-
|
1131
|
-
|
1132
|
-
|
1133
|
-
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1078
|
+
BYTE* const ostart = (BYTE*) dst;
|
1079
|
+
BYTE* op = ostart;
|
1080
|
+
BYTE* const omax = op + maxDstSize;
|
1081
|
+
BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
|
1082
|
+
|
1083
|
+
const void* ptr = DTable;
|
1084
|
+
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
1085
|
+
const U32 dtLog = DTable[0];
|
1086
|
+
size_t errorCode;
|
1087
|
+
U32 reloadStatus;
|
1088
|
+
|
1089
|
+
/* Init */
|
1090
|
+
|
1091
|
+
const U16* jumpTable = (const U16*)cSrc;
|
1092
|
+
const size_t length1 = FSE_readLE16(jumpTable);
|
1093
|
+
const size_t length2 = FSE_readLE16(jumpTable+1);
|
1094
|
+
const size_t length3 = FSE_readLE16(jumpTable+2);
|
1095
|
+
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
|
1096
|
+
const char* const start1 = (const char*)(cSrc) + 6;
|
1097
|
+
const char* const start2 = start1 + length1;
|
1098
|
+
const char* const start3 = start2 + length2;
|
1099
|
+
const char* const start4 = start3 + length3;
|
1100
|
+
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
|
1101
|
+
|
1102
|
+
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
1103
|
+
|
1104
|
+
errorCode = FSE_initDStream(&bitD1, start1, length1);
|
1105
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1106
|
+
errorCode = FSE_initDStream(&bitD2, start2, length2);
|
1107
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1108
|
+
errorCode = FSE_initDStream(&bitD3, start3, length3);
|
1109
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1110
|
+
errorCode = FSE_initDStream(&bitD4, start4, length4);
|
1111
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1112
|
+
|
1113
|
+
reloadStatus=FSE_reloadDStream(&bitD2);
|
1114
|
+
|
1115
|
+
/* 16 symbols per loop */
|
1116
|
+
for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
|
1117
|
+
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
|
1118
|
+
{
|
1119
|
+
#define HUF_DECODE_SYMBOL_0(n, Dstream) \
|
1120
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
|
1121
|
+
|
1122
|
+
#define HUF_DECODE_SYMBOL_1(n, Dstream) \
|
1123
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
|
1124
|
+
if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
|
1125
|
+
|
1126
|
+
#define HUF_DECODE_SYMBOL_2(n, Dstream) \
|
1127
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
|
1128
|
+
if (FSE_32bits()) FSE_reloadDStream(&Dstream)
|
1129
|
+
|
1130
|
+
HUF_DECODE_SYMBOL_1( 0, bitD1);
|
1131
|
+
HUF_DECODE_SYMBOL_1( 1, bitD2);
|
1132
|
+
HUF_DECODE_SYMBOL_1( 2, bitD3);
|
1133
|
+
HUF_DECODE_SYMBOL_1( 3, bitD4);
|
1134
|
+
HUF_DECODE_SYMBOL_2( 4, bitD1);
|
1135
|
+
HUF_DECODE_SYMBOL_2( 5, bitD2);
|
1136
|
+
HUF_DECODE_SYMBOL_2( 6, bitD3);
|
1137
|
+
HUF_DECODE_SYMBOL_2( 7, bitD4);
|
1138
|
+
HUF_DECODE_SYMBOL_1( 8, bitD1);
|
1139
|
+
HUF_DECODE_SYMBOL_1( 9, bitD2);
|
1140
|
+
HUF_DECODE_SYMBOL_1(10, bitD3);
|
1141
|
+
HUF_DECODE_SYMBOL_1(11, bitD4);
|
1142
|
+
HUF_DECODE_SYMBOL_0(12, bitD1);
|
1143
|
+
HUF_DECODE_SYMBOL_0(13, bitD2);
|
1144
|
+
HUF_DECODE_SYMBOL_0(14, bitD3);
|
1145
|
+
HUF_DECODE_SYMBOL_0(15, bitD4);
|
1146
|
+
}
|
1154
1147
|
|
1155
|
-
|
1156
|
-
|
1148
|
+
if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
|
1149
|
+
return (size_t)-FSE_ERROR_corruptionDetected;
|
1157
1150
|
|
1158
|
-
|
1159
|
-
{
|
1160
|
-
// bitTail = bitD1; // *much* slower : -20% !??!
|
1161
|
-
FSE_DStream_t bitTail;
|
1162
|
-
bitTail.ptr = bitD1.ptr;
|
1163
|
-
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
1164
|
-
bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
|
1165
|
-
bitTail.start = start1;
|
1166
|
-
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
1151
|
+
/* tail */
|
1167
1152
|
{
|
1168
|
-
|
1169
|
-
|
1153
|
+
/* bitTail = bitD1; */ /* *much* slower : -20% !??! */
|
1154
|
+
FSE_DStream_t bitTail;
|
1155
|
+
bitTail.ptr = bitD1.ptr;
|
1156
|
+
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
1157
|
+
bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
|
1158
|
+
bitTail.start = start1;
|
1159
|
+
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
1160
|
+
{
|
1161
|
+
HUF_DECODE_SYMBOL_0(0, bitTail);
|
1162
|
+
}
|
1170
1163
|
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1164
|
+
if (FSE_endOfDStream(&bitTail))
|
1165
|
+
return op-ostart;
|
1166
|
+
}
|
1174
1167
|
|
1175
|
-
|
1168
|
+
if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
|
1176
1169
|
|
1177
|
-
|
1170
|
+
return (size_t)-FSE_ERROR_corruptionDetected;
|
1171
|
+
}
|
1178
1172
|
}
|
1179
1173
|
|
1180
1174
|
|
@@ -1196,57 +1190,6 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
|
|
1196
1190
|
|
1197
1191
|
#endif /* FSE_COMMONDEFS_ONLY */
|
1198
1192
|
|
1199
|
-
/*
|
1200
|
-
zstd - standard compression library
|
1201
|
-
Header File for static linking only
|
1202
|
-
Copyright (C) 2014-2015, Yann Collet.
|
1203
|
-
|
1204
|
-
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
1205
|
-
|
1206
|
-
Redistribution and use in source and binary forms, with or without
|
1207
|
-
modification, are permitted provided that the following conditions are
|
1208
|
-
met:
|
1209
|
-
* Redistributions of source code must retain the above copyright
|
1210
|
-
notice, this list of conditions and the following disclaimer.
|
1211
|
-
* Redistributions in binary form must reproduce the above
|
1212
|
-
copyright notice, this list of conditions and the following disclaimer
|
1213
|
-
in the documentation and/or other materials provided with the
|
1214
|
-
distribution.
|
1215
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
1216
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
1217
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
1218
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
1219
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
1220
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
1221
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
1222
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
1223
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
1224
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
1225
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
1226
|
-
|
1227
|
-
You can contact the author at :
|
1228
|
-
- zstd source repository : https://github.com/Cyan4973/zstd
|
1229
|
-
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
|
1230
|
-
*/
|
1231
|
-
|
1232
|
-
/* The objects defined into this file should be considered experimental.
|
1233
|
-
* They are not labelled stable, as their prototype may change in the future.
|
1234
|
-
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
1235
|
-
*/
|
1236
|
-
|
1237
|
-
/**************************************
|
1238
|
-
* Error management
|
1239
|
-
**************************************/
|
1240
|
-
#define ZSTD_LIST_ERRORS(ITEM) \
|
1241
|
-
ITEM(ZSTD_OK_NoError) ITEM(ZSTD_ERROR_GENERIC) \
|
1242
|
-
ITEM(ZSTD_ERROR_MagicNumber) \
|
1243
|
-
ITEM(ZSTD_ERROR_SrcSize) ITEM(ZSTD_ERROR_maxDstSize_tooSmall) \
|
1244
|
-
ITEM(ZSTD_ERROR_corruption) \
|
1245
|
-
ITEM(ZSTD_ERROR_maxCode)
|
1246
|
-
|
1247
|
-
#define ZSTD_GENERATE_ENUM(ENUM) ENUM,
|
1248
|
-
typedef enum { ZSTD_LIST_ERRORS(ZSTD_GENERATE_ENUM) } ZSTD_errorCodes; /* exposed list of errors; static linking only */
|
1249
|
-
|
1250
1193
|
/*
|
1251
1194
|
zstd - standard compression library
|
1252
1195
|
Copyright (C) 2014-2015, Yann Collet.
|
@@ -1325,17 +1268,9 @@ typedef enum { ZSTD_LIST_ERRORS(ZSTD_GENERATE_ENUM) } ZSTD_errorCodes; /* expo
|
|
1325
1268
|
#endif
|
1326
1269
|
|
1327
1270
|
#ifdef _MSC_VER /* Visual Studio */
|
1328
|
-
# define FORCE_INLINE static __forceinline
|
1329
1271
|
# include <intrin.h> /* For Visual 2005 */
|
1330
1272
|
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
1331
1273
|
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
|
1332
|
-
#else
|
1333
|
-
# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
1334
|
-
# ifdef __GNUC__
|
1335
|
-
# define FORCE_INLINE static inline __attribute__((always_inline))
|
1336
|
-
# else
|
1337
|
-
# define FORCE_INLINE static inline
|
1338
|
-
# endif
|
1339
1274
|
#endif
|
1340
1275
|
|
1341
1276
|
|
@@ -1404,6 +1339,8 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51E; /* 3rd version : seqNb header
|
|
1404
1339
|
#define LITERAL_NOENTROPY 63
|
1405
1340
|
#define COMMAND_NOENTROPY 7 /* to remove */
|
1406
1341
|
|
1342
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
1343
|
+
|
1407
1344
|
static const size_t ZSTD_blockHeaderSize = 3;
|
1408
1345
|
static const size_t ZSTD_frameHeaderSize = 4;
|
1409
1346
|
|
@@ -1421,15 +1358,13 @@ static unsigned ZSTD_isLittleEndian(void)
|
|
1421
1358
|
|
1422
1359
|
static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); return r; }
|
1423
1360
|
|
1424
|
-
static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; }
|
1425
|
-
|
1426
1361
|
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
1427
1362
|
|
1428
1363
|
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
1429
1364
|
|
1430
1365
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
1431
1366
|
|
1432
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
1367
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
1433
1368
|
{
|
1434
1369
|
const BYTE* ip = (const BYTE*)src;
|
1435
1370
|
BYTE* op = (BYTE*)dst;
|
@@ -1447,16 +1382,9 @@ static U16 ZSTD_readLE16(const void* memPtr)
|
|
1447
1382
|
}
|
1448
1383
|
}
|
1449
1384
|
|
1450
|
-
|
1451
|
-
static U32 ZSTD_readLE32(const void* memPtr)
|
1385
|
+
static U32 ZSTD_readLE24(const void* memPtr)
|
1452
1386
|
{
|
1453
|
-
|
1454
|
-
return ZSTD_read32(memPtr);
|
1455
|
-
else
|
1456
|
-
{
|
1457
|
-
const BYTE* p = (const BYTE*)memPtr;
|
1458
|
-
return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
|
1459
|
-
}
|
1387
|
+
return ZSTD_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
1460
1388
|
}
|
1461
1389
|
|
1462
1390
|
static U32 ZSTD_readBE32(const void* memPtr)
|
@@ -1507,7 +1435,7 @@ typedef struct ZSTD_Cctx_s
|
|
1507
1435
|
#else
|
1508
1436
|
U32 hashTable[HASH_TABLESIZE];
|
1509
1437
|
#endif
|
1510
|
-
|
1438
|
+
BYTE buffer[WORKPLACESIZE];
|
1511
1439
|
} cctxi_t;
|
1512
1440
|
|
1513
1441
|
|
@@ -1516,11 +1444,8 @@ typedef struct ZSTD_Cctx_s
|
|
1516
1444
|
/**************************************
|
1517
1445
|
* Error Management
|
1518
1446
|
**************************************/
|
1519
|
-
/* tells if a return value is an error code */
|
1520
|
-
static unsigned ZSTD_isError(size_t code) { return (code > (size_t)(-ZSTD_ERROR_maxCode)); }
|
1521
|
-
|
1522
1447
|
/* published entry point */
|
1523
|
-
unsigned ZSTDv01_isError(size_t code) { return
|
1448
|
+
unsigned ZSTDv01_isError(size_t code) { return ERR_isError(code); }
|
1524
1449
|
|
1525
1450
|
|
1526
1451
|
/**************************************
|
@@ -1535,13 +1460,13 @@ unsigned ZSTDv01_isError(size_t code) { return ZSTD_isError(code); }
|
|
1535
1460
|
* Decompression code
|
1536
1461
|
**************************************************************/
|
1537
1462
|
|
1538
|
-
static size_t
|
1463
|
+
static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
1539
1464
|
{
|
1540
1465
|
const BYTE* const in = (const BYTE* const)src;
|
1541
1466
|
BYTE headerFlags;
|
1542
1467
|
U32 cSize;
|
1543
1468
|
|
1544
|
-
if (srcSize < 3) return (
|
1469
|
+
if (srcSize < 3) return ERROR(srcSize_wrong);
|
1545
1470
|
|
1546
1471
|
headerFlags = *in;
|
1547
1472
|
cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16);
|
@@ -1557,8 +1482,10 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
|
|
1557
1482
|
|
1558
1483
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
1559
1484
|
{
|
1560
|
-
if (srcSize > maxDstSize) return (
|
1561
|
-
|
1485
|
+
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
1486
|
+
if (srcSize > 0) {
|
1487
|
+
memcpy(dst, src, srcSize);
|
1488
|
+
}
|
1562
1489
|
return srcSize;
|
1563
1490
|
}
|
1564
1491
|
|
@@ -1574,21 +1501,21 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
|
1574
1501
|
size_t litSize;
|
1575
1502
|
|
1576
1503
|
/* check : minimum 2, for litSize, +1, for content */
|
1577
|
-
if (srcSize <= 3) return (
|
1504
|
+
if (srcSize <= 3) return ERROR(corruption_detected);
|
1578
1505
|
|
1579
1506
|
litSize = ip[1] + (ip[0]<<8);
|
1580
|
-
litSize += ((ip[-3] >> 3) & 7) << 16;
|
1507
|
+
litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
|
1581
1508
|
op = oend - litSize;
|
1582
1509
|
|
1583
1510
|
(void)ctx;
|
1584
|
-
if (litSize > maxDstSize) return (
|
1511
|
+
if (litSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
1585
1512
|
errorCode = HUF_decompress(op, litSize, ip+2, srcSize-2);
|
1586
|
-
if (FSE_isError(errorCode)) return (
|
1513
|
+
if (FSE_isError(errorCode)) return ERROR(GENERIC);
|
1587
1514
|
return litSize;
|
1588
1515
|
}
|
1589
1516
|
|
1590
1517
|
|
1591
|
-
static size_t
|
1518
|
+
static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
1592
1519
|
void* dst, size_t maxDstSize,
|
1593
1520
|
const BYTE** litStart, size_t* litSize,
|
1594
1521
|
const void* src, size_t srcSize)
|
@@ -1599,9 +1526,9 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
1599
1526
|
BYTE* const oend = ostart + maxDstSize;
|
1600
1527
|
blockProperties_t litbp;
|
1601
1528
|
|
1602
|
-
size_t litcSize =
|
1603
|
-
if (
|
1604
|
-
if (litcSize > srcSize - ZSTD_blockHeaderSize) return (
|
1529
|
+
size_t litcSize = ZSTDv01_getcBlockSize(src, srcSize, &litbp);
|
1530
|
+
if (ZSTDv01_isError(litcSize)) return litcSize;
|
1531
|
+
if (litcSize > srcSize - ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
1605
1532
|
ip += ZSTD_blockHeaderSize;
|
1606
1533
|
|
1607
1534
|
switch(litbp.blockType)
|
@@ -1614,8 +1541,11 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
1614
1541
|
case bt_rle:
|
1615
1542
|
{
|
1616
1543
|
size_t rleSize = litbp.origSize;
|
1617
|
-
if (rleSize>maxDstSize) return (
|
1618
|
-
|
1544
|
+
if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
|
1545
|
+
if (!srcSize) return ERROR(srcSize_wrong);
|
1546
|
+
if (rleSize > 0) {
|
1547
|
+
memset(oend - rleSize, *ip, rleSize);
|
1548
|
+
}
|
1619
1549
|
*litStart = oend - rleSize;
|
1620
1550
|
*litSize = rleSize;
|
1621
1551
|
ip++;
|
@@ -1624,7 +1554,7 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
1624
1554
|
case bt_compressed:
|
1625
1555
|
{
|
1626
1556
|
size_t decodedLitSize = ZSTD_decompressLiterals(ctx, dst, maxDstSize, ip, litcSize);
|
1627
|
-
if (
|
1557
|
+
if (ZSTDv01_isError(decodedLitSize)) return decodedLitSize;
|
1628
1558
|
*litStart = oend - decodedLitSize;
|
1629
1559
|
*litSize = decodedLitSize;
|
1630
1560
|
ip += litcSize;
|
@@ -1632,14 +1562,14 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
1632
1562
|
}
|
1633
1563
|
case bt_end:
|
1634
1564
|
default:
|
1635
|
-
return (
|
1565
|
+
return ERROR(GENERIC);
|
1636
1566
|
}
|
1637
1567
|
|
1638
1568
|
return ip-istart;
|
1639
1569
|
}
|
1640
1570
|
|
1641
1571
|
|
1642
|
-
static size_t
|
1572
|
+
static size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
1643
1573
|
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
|
1644
1574
|
const void* src, size_t srcSize)
|
1645
1575
|
{
|
@@ -1651,7 +1581,7 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
|
|
1651
1581
|
size_t dumpsLength;
|
1652
1582
|
|
1653
1583
|
/* check */
|
1654
|
-
if (srcSize < 5) return (
|
1584
|
+
if (srcSize < 5) return ERROR(srcSize_wrong);
|
1655
1585
|
|
1656
1586
|
/* SeqHead */
|
1657
1587
|
*nbSeq = ZSTD_readLE16(ip); ip+=2;
|
@@ -1675,7 +1605,7 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
|
|
1675
1605
|
*dumpsLengthPtr = dumpsLength;
|
1676
1606
|
|
1677
1607
|
/* check */
|
1678
|
-
if (ip > iend-3) return (
|
1608
|
+
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
1679
1609
|
|
1680
1610
|
/* sequences */
|
1681
1611
|
{
|
@@ -1685,7 +1615,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
|
|
1685
1615
|
/* Build DTables */
|
1686
1616
|
switch(LLtype)
|
1687
1617
|
{
|
1688
|
-
U32 max;
|
1689
1618
|
case bt_rle :
|
1690
1619
|
LLlog = 0;
|
1691
1620
|
FSE_buildDTable_rle(DTableLL, *ip++); break;
|
@@ -1693,52 +1622,49 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
|
|
1693
1622
|
LLlog = LLbits;
|
1694
1623
|
FSE_buildDTable_raw(DTableLL, LLbits); break;
|
1695
1624
|
default :
|
1696
|
-
max = MaxLL;
|
1697
|
-
|
1698
|
-
|
1699
|
-
|
1700
|
-
|
1701
|
-
|
1702
|
-
}
|
1625
|
+
{ U32 max = MaxLL;
|
1626
|
+
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
|
1627
|
+
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
1628
|
+
if (LLlog > LLFSELog) return ERROR(corruption_detected);
|
1629
|
+
ip += headerSize;
|
1630
|
+
FSE_buildDTable(DTableLL, norm, max, LLlog);
|
1631
|
+
} }
|
1703
1632
|
|
1704
1633
|
switch(Offtype)
|
1705
1634
|
{
|
1706
|
-
U32 max;
|
1707
1635
|
case bt_rle :
|
1708
1636
|
Offlog = 0;
|
1709
|
-
if (ip > iend-2) return (
|
1637
|
+
if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
|
1710
1638
|
FSE_buildDTable_rle(DTableOffb, *ip++); break;
|
1711
1639
|
case bt_raw :
|
1712
1640
|
Offlog = Offbits;
|
1713
1641
|
FSE_buildDTable_raw(DTableOffb, Offbits); break;
|
1714
1642
|
default :
|
1715
|
-
max = MaxOff;
|
1716
|
-
|
1717
|
-
|
1718
|
-
|
1719
|
-
|
1720
|
-
|
1721
|
-
}
|
1643
|
+
{ U32 max = MaxOff;
|
1644
|
+
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
|
1645
|
+
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
1646
|
+
if (Offlog > OffFSELog) return ERROR(corruption_detected);
|
1647
|
+
ip += headerSize;
|
1648
|
+
FSE_buildDTable(DTableOffb, norm, max, Offlog);
|
1649
|
+
} }
|
1722
1650
|
|
1723
1651
|
switch(MLtype)
|
1724
1652
|
{
|
1725
|
-
U32 max;
|
1726
1653
|
case bt_rle :
|
1727
1654
|
MLlog = 0;
|
1728
|
-
if (ip > iend-2) return (
|
1655
|
+
if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
|
1729
1656
|
FSE_buildDTable_rle(DTableML, *ip++); break;
|
1730
1657
|
case bt_raw :
|
1731
1658
|
MLlog = MLbits;
|
1732
1659
|
FSE_buildDTable_raw(DTableML, MLbits); break;
|
1733
1660
|
default :
|
1734
|
-
max = MaxML;
|
1735
|
-
|
1736
|
-
|
1737
|
-
|
1738
|
-
|
1739
|
-
|
1740
|
-
|
1741
|
-
}
|
1661
|
+
{ U32 max = MaxML;
|
1662
|
+
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
|
1663
|
+
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
1664
|
+
if (MLlog > MLFSELog) return ERROR(corruption_detected);
|
1665
|
+
ip += headerSize;
|
1666
|
+
FSE_buildDTable(DTableML, norm, max, MLlog);
|
1667
|
+
} } }
|
1742
1668
|
|
1743
1669
|
return ip-istart;
|
1744
1670
|
}
|
@@ -1776,13 +1702,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
1776
1702
|
seqState->prevOffset = seq->offset;
|
1777
1703
|
if (litLength == MaxLL)
|
1778
1704
|
{
|
1779
|
-
U32 add = dumps<de ? *dumps++ : 0;
|
1705
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
1780
1706
|
if (add < 255) litLength += add;
|
1781
1707
|
else
|
1782
1708
|
{
|
1783
1709
|
if (dumps<=(de-3))
|
1784
1710
|
{
|
1785
|
-
litLength =
|
1711
|
+
litLength = ZSTD_readLE24(dumps);
|
1786
1712
|
dumps += 3;
|
1787
1713
|
}
|
1788
1714
|
}
|
@@ -1804,13 +1730,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
1804
1730
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
1805
1731
|
if (matchLength == MaxML)
|
1806
1732
|
{
|
1807
|
-
U32 add = dumps<de ? *dumps++ : 0;
|
1733
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
1808
1734
|
if (add < 255) matchLength += add;
|
1809
1735
|
else
|
1810
1736
|
{
|
1811
1737
|
if (dumps<=(de-3))
|
1812
1738
|
{
|
1813
|
-
matchLength =
|
1739
|
+
matchLength = ZSTD_readLE24(dumps);
|
1814
1740
|
dumps += 3;
|
1815
1741
|
}
|
1816
1742
|
}
|
@@ -1831,16 +1757,16 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
1831
1757
|
BYTE* const base, BYTE* const oend)
|
1832
1758
|
{
|
1833
1759
|
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
1834
|
-
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /*
|
1760
|
+
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
1835
1761
|
const BYTE* const ostart = op;
|
1836
1762
|
const size_t litLength = sequence.litLength;
|
1837
1763
|
BYTE* const endMatch = op + litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
1838
1764
|
const BYTE* const litEnd = *litPtr + litLength;
|
1839
1765
|
|
1840
1766
|
/* check */
|
1841
|
-
if (endMatch > oend) return (
|
1842
|
-
if (litEnd > litLimit) return (
|
1843
|
-
if (sequence.matchLength > (size_t)(*litPtr-op)) return (
|
1767
|
+
if (endMatch > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
1768
|
+
if (litEnd > litLimit) return ERROR(corruption_detected);
|
1769
|
+
if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall); /* overwrite literal segment */
|
1844
1770
|
|
1845
1771
|
/* copy Literals */
|
1846
1772
|
if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8))
|
@@ -1851,7 +1777,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
1851
1777
|
*litPtr = litEnd; /* update for next sequence */
|
1852
1778
|
|
1853
1779
|
/* check : last match must be at a minimum distance of 8 from end of dest buffer */
|
1854
|
-
if (oend-op < 8) return (
|
1780
|
+
if (oend-op < 8) return ERROR(dstSize_tooSmall);
|
1855
1781
|
|
1856
1782
|
/* copy Match */
|
1857
1783
|
{
|
@@ -1861,8 +1787,8 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
1861
1787
|
U64 saved[2];
|
1862
1788
|
|
1863
1789
|
/* check */
|
1864
|
-
if (match < base) return (
|
1865
|
-
if (sequence.offset > (size_t)base) return (
|
1790
|
+
if (match < base) return ERROR(corruption_detected);
|
1791
|
+
if (sequence.offset > (size_t)base) return ERROR(corruption_detected);
|
1866
1792
|
|
1867
1793
|
/* save beginning of literal sequence, in case of write overlap */
|
1868
1794
|
if (overlapRisk)
|
@@ -1884,7 +1810,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
1884
1810
|
} else { ZSTD_copy8(op, match); }
|
1885
1811
|
op += 8; match += 8;
|
1886
1812
|
|
1887
|
-
if (endMatch > oend-
|
1813
|
+
if (endMatch > oend-(16-MINMATCH))
|
1888
1814
|
{
|
1889
1815
|
if (op < oend-8)
|
1890
1816
|
{
|
@@ -1895,7 +1821,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
1895
1821
|
while (op<endMatch) *op++ = *match++;
|
1896
1822
|
}
|
1897
1823
|
else
|
1898
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
1824
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
1899
1825
|
|
1900
1826
|
/* restore, in case of overlap */
|
1901
1827
|
if (overlapRisk) memcpy(endMatch, saved, qutt);
|
@@ -1940,10 +1866,10 @@ static size_t ZSTD_decompressSequences(
|
|
1940
1866
|
BYTE* const base = (BYTE*) (dctx->base);
|
1941
1867
|
|
1942
1868
|
/* Build Decoding Tables */
|
1943
|
-
errorCode =
|
1869
|
+
errorCode = ZSTDv01_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
|
1944
1870
|
DTableLL, DTableML, DTableOffb,
|
1945
1871
|
ip, iend-ip);
|
1946
|
-
if (
|
1872
|
+
if (ZSTDv01_isError(errorCode)) return errorCode;
|
1947
1873
|
ip += errorCode;
|
1948
1874
|
|
1949
1875
|
/* Regen sequences */
|
@@ -1956,7 +1882,7 @@ static size_t ZSTD_decompressSequences(
|
|
1956
1882
|
seqState.dumpsEnd = dumps + dumpsLength;
|
1957
1883
|
seqState.prevOffset = 1;
|
1958
1884
|
errorCode = FSE_initDStream(&(seqState.DStream), ip, iend-ip);
|
1959
|
-
if (FSE_isError(errorCode)) return (
|
1885
|
+
if (FSE_isError(errorCode)) return ERROR(corruption_detected);
|
1960
1886
|
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
|
1961
1887
|
FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
|
1962
1888
|
FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
|
@@ -1967,20 +1893,22 @@ static size_t ZSTD_decompressSequences(
|
|
1967
1893
|
nbSeq--;
|
1968
1894
|
ZSTD_decodeSequence(&sequence, &seqState);
|
1969
1895
|
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
1970
|
-
if (
|
1896
|
+
if (ZSTDv01_isError(oneSeqSize)) return oneSeqSize;
|
1971
1897
|
op += oneSeqSize;
|
1972
1898
|
}
|
1973
1899
|
|
1974
1900
|
/* check if reached exact end */
|
1975
|
-
if ( !FSE_endOfDStream(&(seqState.DStream)) ) return (
|
1976
|
-
if (nbSeq<0) return (
|
1901
|
+
if ( !FSE_endOfDStream(&(seqState.DStream)) ) return ERROR(corruption_detected); /* requested too much : data is corrupted */
|
1902
|
+
if (nbSeq<0) return ERROR(corruption_detected); /* requested too many sequences : data is corrupted */
|
1977
1903
|
|
1978
1904
|
/* last literal segment */
|
1979
1905
|
{
|
1980
1906
|
size_t lastLLSize = litEnd - litPtr;
|
1981
|
-
if (op+lastLLSize > oend) return (
|
1982
|
-
if (
|
1983
|
-
|
1907
|
+
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
1908
|
+
if (lastLLSize > 0) {
|
1909
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
1910
|
+
op += lastLLSize;
|
1911
|
+
}
|
1984
1912
|
}
|
1985
1913
|
}
|
1986
1914
|
|
@@ -2000,8 +1928,8 @@ static size_t ZSTD_decompressBlock(
|
|
2000
1928
|
size_t errorCode;
|
2001
1929
|
|
2002
1930
|
/* Decode literals sub-block */
|
2003
|
-
errorCode =
|
2004
|
-
if (
|
1931
|
+
errorCode = ZSTDv01_decodeLiteralsBlock(ctx, dst, maxDstSize, &litPtr, &litSize, src, srcSize);
|
1932
|
+
if (ZSTDv01_isError(errorCode)) return errorCode;
|
2005
1933
|
ip += errorCode;
|
2006
1934
|
srcSize -= errorCode;
|
2007
1935
|
|
@@ -2022,20 +1950,20 @@ size_t ZSTDv01_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const voi
|
|
2022
1950
|
blockProperties_t blockProperties;
|
2023
1951
|
|
2024
1952
|
/* Frame Header */
|
2025
|
-
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return (
|
1953
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
2026
1954
|
magicNumber = ZSTD_readBE32(src);
|
2027
|
-
if (magicNumber != ZSTD_magicNumber) return (
|
1955
|
+
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
2028
1956
|
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
2029
1957
|
|
2030
1958
|
/* Loop on each block */
|
2031
1959
|
while (1)
|
2032
1960
|
{
|
2033
|
-
size_t blockSize =
|
2034
|
-
if (
|
1961
|
+
size_t blockSize = ZSTDv01_getcBlockSize(ip, iend-ip, &blockProperties);
|
1962
|
+
if (ZSTDv01_isError(blockSize)) return blockSize;
|
2035
1963
|
|
2036
1964
|
ip += ZSTD_blockHeaderSize;
|
2037
1965
|
remainingSize -= ZSTD_blockHeaderSize;
|
2038
|
-
if (blockSize > remainingSize) return (
|
1966
|
+
if (blockSize > remainingSize) return ERROR(srcSize_wrong);
|
2039
1967
|
|
2040
1968
|
switch(blockProperties.blockType)
|
2041
1969
|
{
|
@@ -2046,18 +1974,18 @@ size_t ZSTDv01_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const voi
|
|
2046
1974
|
errorCode = ZSTD_copyUncompressedBlock(op, oend-op, ip, blockSize);
|
2047
1975
|
break;
|
2048
1976
|
case bt_rle :
|
2049
|
-
return (
|
1977
|
+
return ERROR(GENERIC); /* not yet supported */
|
2050
1978
|
break;
|
2051
1979
|
case bt_end :
|
2052
1980
|
/* end of frame */
|
2053
|
-
if (remainingSize) return (
|
1981
|
+
if (remainingSize) return ERROR(srcSize_wrong);
|
2054
1982
|
break;
|
2055
1983
|
default:
|
2056
|
-
return (
|
1984
|
+
return ERROR(GENERIC);
|
2057
1985
|
}
|
2058
1986
|
if (blockSize == 0) break; /* bt_end */
|
2059
1987
|
|
2060
|
-
if (
|
1988
|
+
if (ZSTDv01_isError(errorCode)) return errorCode;
|
2061
1989
|
op += errorCode;
|
2062
1990
|
ip += blockSize;
|
2063
1991
|
remainingSize -= blockSize;
|
@@ -2073,6 +2001,60 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|
2073
2001
|
return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
2074
2002
|
}
|
2075
2003
|
|
2004
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
2005
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
2006
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
2007
|
+
{
|
2008
|
+
*cSize = ret;
|
2009
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
2010
|
+
}
|
2011
|
+
|
2012
|
+
void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
2013
|
+
{
|
2014
|
+
const BYTE* ip = (const BYTE*)src;
|
2015
|
+
size_t remainingSize = srcSize;
|
2016
|
+
size_t nbBlocks = 0;
|
2017
|
+
U32 magicNumber;
|
2018
|
+
blockProperties_t blockProperties;
|
2019
|
+
|
2020
|
+
/* Frame Header */
|
2021
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
|
2022
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
2023
|
+
return;
|
2024
|
+
}
|
2025
|
+
magicNumber = ZSTD_readBE32(src);
|
2026
|
+
if (magicNumber != ZSTD_magicNumber) {
|
2027
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
2028
|
+
return;
|
2029
|
+
}
|
2030
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
2031
|
+
|
2032
|
+
/* Loop on each block */
|
2033
|
+
while (1)
|
2034
|
+
{
|
2035
|
+
size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties);
|
2036
|
+
if (ZSTDv01_isError(blockSize)) {
|
2037
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, blockSize);
|
2038
|
+
return;
|
2039
|
+
}
|
2040
|
+
|
2041
|
+
ip += ZSTD_blockHeaderSize;
|
2042
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
2043
|
+
if (blockSize > remainingSize) {
|
2044
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
2045
|
+
return;
|
2046
|
+
}
|
2047
|
+
|
2048
|
+
if (blockSize == 0) break; /* bt_end */
|
2049
|
+
|
2050
|
+
ip += blockSize;
|
2051
|
+
remainingSize -= blockSize;
|
2052
|
+
nbBlocks++;
|
2053
|
+
}
|
2054
|
+
|
2055
|
+
*cSize = ip - (const BYTE*)src;
|
2056
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
2057
|
+
}
|
2076
2058
|
|
2077
2059
|
/*******************************
|
2078
2060
|
* Streaming Decompression API
|
@@ -2111,7 +2093,7 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
|
|
2111
2093
|
dctx_t* ctx = (dctx_t*)dctx;
|
2112
2094
|
|
2113
2095
|
/* Sanity check */
|
2114
|
-
if (srcSize != ctx->expected) return (
|
2096
|
+
if (srcSize != ctx->expected) return ERROR(srcSize_wrong);
|
2115
2097
|
if (dst != ctx->previousDstEnd) /* not contiguous */
|
2116
2098
|
ctx->base = dst;
|
2117
2099
|
|
@@ -2120,7 +2102,7 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
|
|
2120
2102
|
{
|
2121
2103
|
/* Check frame magic header */
|
2122
2104
|
U32 magicNumber = ZSTD_readBE32(src);
|
2123
|
-
if (magicNumber != ZSTD_magicNumber) return (
|
2105
|
+
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
2124
2106
|
ctx->phase = 1;
|
2125
2107
|
ctx->expected = ZSTD_blockHeaderSize;
|
2126
2108
|
return 0;
|
@@ -2130,8 +2112,8 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
|
|
2130
2112
|
if (ctx->phase == 1)
|
2131
2113
|
{
|
2132
2114
|
blockProperties_t bp;
|
2133
|
-
size_t blockSize =
|
2134
|
-
if (
|
2115
|
+
size_t blockSize = ZSTDv01_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
2116
|
+
if (ZSTDv01_isError(blockSize)) return blockSize;
|
2135
2117
|
if (bp.blockType == bt_end)
|
2136
2118
|
{
|
2137
2119
|
ctx->expected = 0;
|
@@ -2159,13 +2141,13 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
|
|
2159
2141
|
rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, srcSize);
|
2160
2142
|
break;
|
2161
2143
|
case bt_rle :
|
2162
|
-
return (
|
2144
|
+
return ERROR(GENERIC); /* not yet handled */
|
2163
2145
|
break;
|
2164
2146
|
case bt_end : /* should never happen (filtered at phase 1) */
|
2165
2147
|
rSize = 0;
|
2166
2148
|
break;
|
2167
2149
|
default:
|
2168
|
-
return (
|
2150
|
+
return ERROR(GENERIC);
|
2169
2151
|
}
|
2170
2152
|
ctx->phase = 1;
|
2171
2153
|
ctx->expected = ZSTD_blockHeaderSize;
|
@@ -2174,5 +2156,3 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
|
|
2174
2156
|
}
|
2175
2157
|
|
2176
2158
|
}
|
2177
|
-
|
2178
|
-
|