extzstd 0.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +39 -0
- data/README.md +38 -56
- data/contrib/zstd/CHANGELOG +613 -0
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/CONTRIBUTING.md +406 -0
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/Makefile +420 -0
- data/contrib/zstd/README.md +179 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +292 -0
- data/contrib/zstd/lib/BUCK +234 -0
- data/contrib/zstd/lib/Makefile +451 -0
- data/contrib/zstd/lib/README.md +207 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
- data/contrib/zstd/lib/common/compiler.h +288 -0
- data/contrib/zstd/lib/common/cpu.h +213 -0
- data/contrib/zstd/lib/common/debug.c +24 -0
- data/contrib/zstd/lib/common/debug.h +107 -0
- data/contrib/zstd/lib/common/entropy_common.c +362 -0
- data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
- data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
- data/contrib/zstd/{common → lib/common}/fse.h +173 -92
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
- data/contrib/zstd/lib/common/huf.h +361 -0
- data/contrib/zstd/{common → lib/common}/mem.h +115 -59
- data/contrib/zstd/lib/common/pool.c +350 -0
- data/contrib/zstd/lib/common/pool.h +84 -0
- data/contrib/zstd/lib/common/threading.c +122 -0
- data/contrib/zstd/lib/common/threading.h +155 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
- data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
- data/contrib/zstd/lib/common/zstd_common.c +83 -0
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_errors.h +95 -0
- data/contrib/zstd/lib/common/zstd_internal.h +478 -0
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
- data/contrib/zstd/lib/compress/hist.c +181 -0
- data/contrib/zstd/lib/compress/hist.h +75 -0
- data/contrib/zstd/lib/compress/huf_compress.c +913 -0
- data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
- data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
- data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
- data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
- data/contrib/zstd/lib/libzstd.pc.in +15 -0
- data/contrib/zstd/lib/zstd.h +2391 -0
- data/ext/depend +2 -0
- data/ext/extconf.rb +15 -6
- data/ext/extzstd.c +76 -145
- data/ext/extzstd.h +80 -31
- data/ext/extzstd_stream.c +417 -142
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +10 -7
- data/ext/zstd_compress.c +14 -5
- data/ext/zstd_decompress.c +5 -4
- data/ext/zstd_dictbuilder.c +9 -4
- data/ext/zstd_dictbuilder_fastcover.c +3 -0
- data/ext/zstd_legacy_v01.c +3 -1
- data/ext/zstd_legacy_v02.c +3 -1
- data/ext/zstd_legacy_v03.c +3 -1
- data/ext/zstd_legacy_v04.c +3 -1
- data/ext/zstd_legacy_v05.c +3 -1
- data/ext/zstd_legacy_v06.c +3 -1
- data/ext/zstd_legacy_v07.c +3 -1
- data/gemstub.rb +10 -24
- data/lib/extzstd.rb +64 -179
- data/lib/extzstd/version.rb +6 -1
- data/test/test_basic.rb +9 -6
- metadata +113 -57
- data/HISTORY.ja +0 -5
- data/contrib/zstd/common/entropy_common.c +0 -225
- data/contrib/zstd/common/huf.h +0 -228
- data/contrib/zstd/common/zstd_common.c +0 -83
- data/contrib/zstd/common/zstd_errors.h +0 -60
- data/contrib/zstd/common/zstd_internal.h +0 -267
- data/contrib/zstd/compress/huf_compress.c +0 -533
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/compress/zstd_compress.c +0 -3264
- data/contrib/zstd/compress/zstd_opt.h +0 -900
- data/contrib/zstd/decompress/huf_decompress.c +0 -883
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
- data/contrib/zstd/zstd.h +0 -640
@@ -1,10 +1,11 @@
|
|
1
|
-
|
2
|
-
* Copyright (c) 2016-
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
6
|
-
* LICENSE file in the root directory of this source tree
|
7
|
-
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
8
9
|
*/
|
9
10
|
|
10
11
|
|
@@ -13,7 +14,7 @@
|
|
13
14
|
******************************************/
|
14
15
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
15
16
|
#include "zstd_v01.h"
|
16
|
-
#include "error_private.h"
|
17
|
+
#include "../common/error_private.h"
|
17
18
|
|
18
19
|
|
19
20
|
/******************************************
|
@@ -256,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
|
|
256
257
|
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
257
258
|
}
|
258
259
|
|
259
|
-
#endif
|
260
|
+
#endif /* FSE_FORCE_MEMORY_ACCESS */
|
260
261
|
|
261
262
|
static U16 FSE_readLE16(const void* memPtr)
|
262
263
|
{
|
@@ -338,14 +339,14 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
|
338
339
|
/****************************************************************
|
339
340
|
* Internal functions
|
340
341
|
****************************************************************/
|
341
|
-
FORCE_INLINE unsigned FSE_highbit32 (
|
342
|
+
FORCE_INLINE unsigned FSE_highbit32 (U32 val)
|
342
343
|
{
|
343
344
|
# if defined(_MSC_VER) /* Visual */
|
344
345
|
unsigned long r;
|
345
346
|
_BitScanReverse ( &r, val );
|
346
347
|
return (unsigned) r;
|
347
348
|
# elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */
|
348
|
-
return
|
349
|
+
return __builtin_clz (val) ^ 31;
|
349
350
|
# else /* Software version */
|
350
351
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
351
352
|
U32 v = val;
|
@@ -667,11 +668,17 @@ static size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t
|
|
667
668
|
switch(srcSize)
|
668
669
|
{
|
669
670
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
671
|
+
/* fallthrough */
|
670
672
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
673
|
+
/* fallthrough */
|
671
674
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
675
|
+
/* fallthrough */
|
672
676
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
677
|
+
/* fallthrough */
|
673
678
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
679
|
+
/* fallthrough */
|
674
680
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
681
|
+
/* fallthrough */
|
675
682
|
default:;
|
676
683
|
}
|
677
684
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
@@ -1066,99 +1073,102 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
1066
1073
|
const void* cSrc, size_t cSrcSize,
|
1067
1074
|
const U16* DTable)
|
1068
1075
|
{
|
1069
|
-
|
1070
|
-
BYTE* op = ostart;
|
1071
|
-
BYTE* const omax = op + maxDstSize;
|
1072
|
-
BYTE* const olimit = omax-15;
|
1073
|
-
|
1074
|
-
const void* ptr = DTable;
|
1075
|
-
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
1076
|
-
const U32 dtLog = DTable[0];
|
1077
|
-
size_t errorCode;
|
1078
|
-
U32 reloadStatus;
|
1079
|
-
|
1080
|
-
/* Init */
|
1081
|
-
|
1082
|
-
const U16* jumpTable = (const U16*)cSrc;
|
1083
|
-
const size_t length1 = FSE_readLE16(jumpTable);
|
1084
|
-
const size_t length2 = FSE_readLE16(jumpTable+1);
|
1085
|
-
const size_t length3 = FSE_readLE16(jumpTable+2);
|
1086
|
-
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
|
1087
|
-
const char* const start1 = (const char*)(cSrc) + 6;
|
1088
|
-
const char* const start2 = start1 + length1;
|
1089
|
-
const char* const start3 = start2 + length2;
|
1090
|
-
const char* const start4 = start3 + length3;
|
1091
|
-
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
|
1092
|
-
|
1093
|
-
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
1094
|
-
|
1095
|
-
errorCode = FSE_initDStream(&bitD1, start1, length1);
|
1096
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1097
|
-
errorCode = FSE_initDStream(&bitD2, start2, length2);
|
1098
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1099
|
-
errorCode = FSE_initDStream(&bitD3, start3, length3);
|
1100
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1101
|
-
errorCode = FSE_initDStream(&bitD4, start4, length4);
|
1102
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1103
|
-
|
1104
|
-
reloadStatus=FSE_reloadDStream(&bitD2);
|
1105
|
-
|
1106
|
-
/* 16 symbols per loop */
|
1107
|
-
for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
|
1108
|
-
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
|
1076
|
+
if (cSrcSize < 6) return (size_t)-FSE_ERROR_srcSize_wrong;
|
1109
1077
|
{
|
1110
|
-
|
1111
|
-
op
|
1112
|
-
|
1113
|
-
|
1114
|
-
|
1115
|
-
|
1116
|
-
|
1117
|
-
|
1118
|
-
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
1126
|
-
|
1127
|
-
|
1128
|
-
|
1129
|
-
|
1130
|
-
|
1131
|
-
|
1132
|
-
|
1133
|
-
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1078
|
+
BYTE* const ostart = (BYTE*) dst;
|
1079
|
+
BYTE* op = ostart;
|
1080
|
+
BYTE* const omax = op + maxDstSize;
|
1081
|
+
BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
|
1082
|
+
|
1083
|
+
const void* ptr = DTable;
|
1084
|
+
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
1085
|
+
const U32 dtLog = DTable[0];
|
1086
|
+
size_t errorCode;
|
1087
|
+
U32 reloadStatus;
|
1088
|
+
|
1089
|
+
/* Init */
|
1090
|
+
|
1091
|
+
const U16* jumpTable = (const U16*)cSrc;
|
1092
|
+
const size_t length1 = FSE_readLE16(jumpTable);
|
1093
|
+
const size_t length2 = FSE_readLE16(jumpTable+1);
|
1094
|
+
const size_t length3 = FSE_readLE16(jumpTable+2);
|
1095
|
+
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
|
1096
|
+
const char* const start1 = (const char*)(cSrc) + 6;
|
1097
|
+
const char* const start2 = start1 + length1;
|
1098
|
+
const char* const start3 = start2 + length2;
|
1099
|
+
const char* const start4 = start3 + length3;
|
1100
|
+
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
|
1101
|
+
|
1102
|
+
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
1103
|
+
|
1104
|
+
errorCode = FSE_initDStream(&bitD1, start1, length1);
|
1105
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1106
|
+
errorCode = FSE_initDStream(&bitD2, start2, length2);
|
1107
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1108
|
+
errorCode = FSE_initDStream(&bitD3, start3, length3);
|
1109
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1110
|
+
errorCode = FSE_initDStream(&bitD4, start4, length4);
|
1111
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1112
|
+
|
1113
|
+
reloadStatus=FSE_reloadDStream(&bitD2);
|
1114
|
+
|
1115
|
+
/* 16 symbols per loop */
|
1116
|
+
for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
|
1117
|
+
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
|
1118
|
+
{
|
1119
|
+
#define HUF_DECODE_SYMBOL_0(n, Dstream) \
|
1120
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
|
1121
|
+
|
1122
|
+
#define HUF_DECODE_SYMBOL_1(n, Dstream) \
|
1123
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
|
1124
|
+
if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
|
1125
|
+
|
1126
|
+
#define HUF_DECODE_SYMBOL_2(n, Dstream) \
|
1127
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
|
1128
|
+
if (FSE_32bits()) FSE_reloadDStream(&Dstream)
|
1129
|
+
|
1130
|
+
HUF_DECODE_SYMBOL_1( 0, bitD1);
|
1131
|
+
HUF_DECODE_SYMBOL_1( 1, bitD2);
|
1132
|
+
HUF_DECODE_SYMBOL_1( 2, bitD3);
|
1133
|
+
HUF_DECODE_SYMBOL_1( 3, bitD4);
|
1134
|
+
HUF_DECODE_SYMBOL_2( 4, bitD1);
|
1135
|
+
HUF_DECODE_SYMBOL_2( 5, bitD2);
|
1136
|
+
HUF_DECODE_SYMBOL_2( 6, bitD3);
|
1137
|
+
HUF_DECODE_SYMBOL_2( 7, bitD4);
|
1138
|
+
HUF_DECODE_SYMBOL_1( 8, bitD1);
|
1139
|
+
HUF_DECODE_SYMBOL_1( 9, bitD2);
|
1140
|
+
HUF_DECODE_SYMBOL_1(10, bitD3);
|
1141
|
+
HUF_DECODE_SYMBOL_1(11, bitD4);
|
1142
|
+
HUF_DECODE_SYMBOL_0(12, bitD1);
|
1143
|
+
HUF_DECODE_SYMBOL_0(13, bitD2);
|
1144
|
+
HUF_DECODE_SYMBOL_0(14, bitD3);
|
1145
|
+
HUF_DECODE_SYMBOL_0(15, bitD4);
|
1146
|
+
}
|
1138
1147
|
|
1139
|
-
|
1140
|
-
|
1148
|
+
if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
|
1149
|
+
return (size_t)-FSE_ERROR_corruptionDetected;
|
1141
1150
|
|
1142
|
-
|
1143
|
-
{
|
1144
|
-
// bitTail = bitD1; // *much* slower : -20% !??!
|
1145
|
-
FSE_DStream_t bitTail;
|
1146
|
-
bitTail.ptr = bitD1.ptr;
|
1147
|
-
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
1148
|
-
bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
|
1149
|
-
bitTail.start = start1;
|
1150
|
-
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
1151
|
+
/* tail */
|
1151
1152
|
{
|
1152
|
-
|
1153
|
-
|
1153
|
+
/* bitTail = bitD1; */ /* *much* slower : -20% !??! */
|
1154
|
+
FSE_DStream_t bitTail;
|
1155
|
+
bitTail.ptr = bitD1.ptr;
|
1156
|
+
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
1157
|
+
bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
|
1158
|
+
bitTail.start = start1;
|
1159
|
+
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
1160
|
+
{
|
1161
|
+
HUF_DECODE_SYMBOL_0(0, bitTail);
|
1162
|
+
}
|
1154
1163
|
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1164
|
+
if (FSE_endOfDStream(&bitTail))
|
1165
|
+
return op-ostart;
|
1166
|
+
}
|
1158
1167
|
|
1159
|
-
|
1168
|
+
if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
|
1160
1169
|
|
1161
|
-
|
1170
|
+
return (size_t)-FSE_ERROR_corruptionDetected;
|
1171
|
+
}
|
1162
1172
|
}
|
1163
1173
|
|
1164
1174
|
|
@@ -1270,7 +1280,11 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
|
|
1270
1280
|
* Basic Types
|
1271
1281
|
*********************************************************/
|
1272
1282
|
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
1273
|
-
#
|
1283
|
+
# if defined(_AIX)
|
1284
|
+
# include <inttypes.h>
|
1285
|
+
# else
|
1286
|
+
# include <stdint.h> /* intptr_t */
|
1287
|
+
# endif
|
1274
1288
|
typedef uint8_t BYTE;
|
1275
1289
|
typedef uint16_t U16;
|
1276
1290
|
typedef int16_t S16;
|
@@ -1329,6 +1343,8 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51E; /* 3rd version : seqNb header
|
|
1329
1343
|
#define LITERAL_NOENTROPY 63
|
1330
1344
|
#define COMMAND_NOENTROPY 7 /* to remove */
|
1331
1345
|
|
1346
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
1347
|
+
|
1332
1348
|
static const size_t ZSTD_blockHeaderSize = 3;
|
1333
1349
|
static const size_t ZSTD_frameHeaderSize = 4;
|
1334
1350
|
|
@@ -1346,15 +1362,13 @@ static unsigned ZSTD_isLittleEndian(void)
|
|
1346
1362
|
|
1347
1363
|
static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); return r; }
|
1348
1364
|
|
1349
|
-
static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; }
|
1350
|
-
|
1351
1365
|
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
1352
1366
|
|
1353
1367
|
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
1354
1368
|
|
1355
1369
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
1356
1370
|
|
1357
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
1371
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
1358
1372
|
{
|
1359
1373
|
const BYTE* ip = (const BYTE*)src;
|
1360
1374
|
BYTE* op = (BYTE*)dst;
|
@@ -1372,16 +1386,9 @@ static U16 ZSTD_readLE16(const void* memPtr)
|
|
1372
1386
|
}
|
1373
1387
|
}
|
1374
1388
|
|
1375
|
-
|
1376
|
-
static U32 ZSTD_readLE32(const void* memPtr)
|
1389
|
+
static U32 ZSTD_readLE24(const void* memPtr)
|
1377
1390
|
{
|
1378
|
-
|
1379
|
-
return ZSTD_read32(memPtr);
|
1380
|
-
else
|
1381
|
-
{
|
1382
|
-
const BYTE* p = (const BYTE*)memPtr;
|
1383
|
-
return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
|
1384
|
-
}
|
1391
|
+
return ZSTD_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
1385
1392
|
}
|
1386
1393
|
|
1387
1394
|
static U32 ZSTD_readBE32(const void* memPtr)
|
@@ -1432,7 +1439,7 @@ typedef struct ZSTD_Cctx_s
|
|
1432
1439
|
#else
|
1433
1440
|
U32 hashTable[HASH_TABLESIZE];
|
1434
1441
|
#endif
|
1435
|
-
|
1442
|
+
BYTE buffer[WORKPLACESIZE];
|
1436
1443
|
} cctxi_t;
|
1437
1444
|
|
1438
1445
|
|
@@ -1457,7 +1464,7 @@ unsigned ZSTDv01_isError(size_t code) { return ERR_isError(code); }
|
|
1457
1464
|
* Decompression code
|
1458
1465
|
**************************************************************/
|
1459
1466
|
|
1460
|
-
size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
1467
|
+
static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
1461
1468
|
{
|
1462
1469
|
const BYTE* const in = (const BYTE* const)src;
|
1463
1470
|
BYTE headerFlags;
|
@@ -1480,7 +1487,9 @@ size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|
1480
1487
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
1481
1488
|
{
|
1482
1489
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
1483
|
-
|
1490
|
+
if (srcSize > 0) {
|
1491
|
+
memcpy(dst, src, srcSize);
|
1492
|
+
}
|
1484
1493
|
return srcSize;
|
1485
1494
|
}
|
1486
1495
|
|
@@ -1499,7 +1508,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
|
1499
1508
|
if (srcSize <= 3) return ERROR(corruption_detected);
|
1500
1509
|
|
1501
1510
|
litSize = ip[1] + (ip[0]<<8);
|
1502
|
-
litSize += ((ip[-3] >> 3) & 7) << 16;
|
1511
|
+
litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
|
1503
1512
|
op = oend - litSize;
|
1504
1513
|
|
1505
1514
|
(void)ctx;
|
@@ -1510,7 +1519,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
|
1510
1519
|
}
|
1511
1520
|
|
1512
1521
|
|
1513
|
-
size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
1522
|
+
static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
1514
1523
|
void* dst, size_t maxDstSize,
|
1515
1524
|
const BYTE** litStart, size_t* litSize,
|
1516
1525
|
const void* src, size_t srcSize)
|
@@ -1538,7 +1547,9 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
1538
1547
|
size_t rleSize = litbp.origSize;
|
1539
1548
|
if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
|
1540
1549
|
if (!srcSize) return ERROR(srcSize_wrong);
|
1541
|
-
|
1550
|
+
if (rleSize > 0) {
|
1551
|
+
memset(oend - rleSize, *ip, rleSize);
|
1552
|
+
}
|
1542
1553
|
*litStart = oend - rleSize;
|
1543
1554
|
*litSize = rleSize;
|
1544
1555
|
ip++;
|
@@ -1562,7 +1573,7 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
1562
1573
|
}
|
1563
1574
|
|
1564
1575
|
|
1565
|
-
size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
1576
|
+
static size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
1566
1577
|
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
|
1567
1578
|
const void* src, size_t srcSize)
|
1568
1579
|
{
|
@@ -1695,13 +1706,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
1695
1706
|
seqState->prevOffset = seq->offset;
|
1696
1707
|
if (litLength == MaxLL)
|
1697
1708
|
{
|
1698
|
-
U32 add = dumps<de ? *dumps++ : 0;
|
1709
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
1699
1710
|
if (add < 255) litLength += add;
|
1700
1711
|
else
|
1701
1712
|
{
|
1702
1713
|
if (dumps<=(de-3))
|
1703
1714
|
{
|
1704
|
-
litLength =
|
1715
|
+
litLength = ZSTD_readLE24(dumps);
|
1705
1716
|
dumps += 3;
|
1706
1717
|
}
|
1707
1718
|
}
|
@@ -1723,13 +1734,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
1723
1734
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
1724
1735
|
if (matchLength == MaxML)
|
1725
1736
|
{
|
1726
|
-
U32 add = dumps<de ? *dumps++ : 0;
|
1737
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
1727
1738
|
if (add < 255) matchLength += add;
|
1728
1739
|
else
|
1729
1740
|
{
|
1730
1741
|
if (dumps<=(de-3))
|
1731
1742
|
{
|
1732
|
-
matchLength =
|
1743
|
+
matchLength = ZSTD_readLE24(dumps);
|
1733
1744
|
dumps += 3;
|
1734
1745
|
}
|
1735
1746
|
}
|
@@ -1750,7 +1761,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
1750
1761
|
BYTE* const base, BYTE* const oend)
|
1751
1762
|
{
|
1752
1763
|
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
1753
|
-
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /*
|
1764
|
+
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
1754
1765
|
const BYTE* const ostart = op;
|
1755
1766
|
const size_t litLength = sequence.litLength;
|
1756
1767
|
BYTE* const endMatch = op + litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
@@ -1803,7 +1814,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
1803
1814
|
} else { ZSTD_copy8(op, match); }
|
1804
1815
|
op += 8; match += 8;
|
1805
1816
|
|
1806
|
-
if (endMatch > oend-
|
1817
|
+
if (endMatch > oend-(16-MINMATCH))
|
1807
1818
|
{
|
1808
1819
|
if (op < oend-8)
|
1809
1820
|
{
|
@@ -1814,7 +1825,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
1814
1825
|
while (op<endMatch) *op++ = *match++;
|
1815
1826
|
}
|
1816
1827
|
else
|
1817
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
1828
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
1818
1829
|
|
1819
1830
|
/* restore, in case of overlap */
|
1820
1831
|
if (overlapRisk) memcpy(endMatch, saved, qutt);
|
@@ -1898,8 +1909,10 @@ static size_t ZSTD_decompressSequences(
|
|
1898
1909
|
{
|
1899
1910
|
size_t lastLLSize = litEnd - litPtr;
|
1900
1911
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
1901
|
-
if (
|
1902
|
-
|
1912
|
+
if (lastLLSize > 0) {
|
1913
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
1914
|
+
op += lastLLSize;
|
1915
|
+
}
|
1903
1916
|
}
|
1904
1917
|
}
|
1905
1918
|
|
@@ -1992,6 +2005,60 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|
1992
2005
|
return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
1993
2006
|
}
|
1994
2007
|
|
2008
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
2009
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
2010
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
2011
|
+
{
|
2012
|
+
*cSize = ret;
|
2013
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
2014
|
+
}
|
2015
|
+
|
2016
|
+
void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
2017
|
+
{
|
2018
|
+
const BYTE* ip = (const BYTE*)src;
|
2019
|
+
size_t remainingSize = srcSize;
|
2020
|
+
size_t nbBlocks = 0;
|
2021
|
+
U32 magicNumber;
|
2022
|
+
blockProperties_t blockProperties;
|
2023
|
+
|
2024
|
+
/* Frame Header */
|
2025
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
|
2026
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
2027
|
+
return;
|
2028
|
+
}
|
2029
|
+
magicNumber = ZSTD_readBE32(src);
|
2030
|
+
if (magicNumber != ZSTD_magicNumber) {
|
2031
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
2032
|
+
return;
|
2033
|
+
}
|
2034
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
2035
|
+
|
2036
|
+
/* Loop on each block */
|
2037
|
+
while (1)
|
2038
|
+
{
|
2039
|
+
size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties);
|
2040
|
+
if (ZSTDv01_isError(blockSize)) {
|
2041
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, blockSize);
|
2042
|
+
return;
|
2043
|
+
}
|
2044
|
+
|
2045
|
+
ip += ZSTD_blockHeaderSize;
|
2046
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
2047
|
+
if (blockSize > remainingSize) {
|
2048
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
2049
|
+
return;
|
2050
|
+
}
|
2051
|
+
|
2052
|
+
if (blockSize == 0) break; /* bt_end */
|
2053
|
+
|
2054
|
+
ip += blockSize;
|
2055
|
+
remainingSize -= blockSize;
|
2056
|
+
nbBlocks++;
|
2057
|
+
}
|
2058
|
+
|
2059
|
+
*cSize = ip - (const BYTE*)src;
|
2060
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
2061
|
+
}
|
1995
2062
|
|
1996
2063
|
/*******************************
|
1997
2064
|
* Streaming Decompression API
|