extzstd 0.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +39 -0
- data/README.md +38 -56
- data/contrib/zstd/CHANGELOG +613 -0
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/CONTRIBUTING.md +406 -0
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/Makefile +420 -0
- data/contrib/zstd/README.md +179 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +292 -0
- data/contrib/zstd/lib/BUCK +234 -0
- data/contrib/zstd/lib/Makefile +451 -0
- data/contrib/zstd/lib/README.md +207 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
- data/contrib/zstd/lib/common/compiler.h +288 -0
- data/contrib/zstd/lib/common/cpu.h +213 -0
- data/contrib/zstd/lib/common/debug.c +24 -0
- data/contrib/zstd/lib/common/debug.h +107 -0
- data/contrib/zstd/lib/common/entropy_common.c +362 -0
- data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
- data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
- data/contrib/zstd/{common → lib/common}/fse.h +173 -92
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
- data/contrib/zstd/lib/common/huf.h +361 -0
- data/contrib/zstd/{common → lib/common}/mem.h +115 -59
- data/contrib/zstd/lib/common/pool.c +350 -0
- data/contrib/zstd/lib/common/pool.h +84 -0
- data/contrib/zstd/lib/common/threading.c +122 -0
- data/contrib/zstd/lib/common/threading.h +155 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
- data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
- data/contrib/zstd/lib/common/zstd_common.c +83 -0
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_errors.h +95 -0
- data/contrib/zstd/lib/common/zstd_internal.h +478 -0
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
- data/contrib/zstd/lib/compress/hist.c +181 -0
- data/contrib/zstd/lib/compress/hist.h +75 -0
- data/contrib/zstd/lib/compress/huf_compress.c +913 -0
- data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
- data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
- data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
- data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
- data/contrib/zstd/lib/libzstd.pc.in +15 -0
- data/contrib/zstd/lib/zstd.h +2391 -0
- data/ext/depend +2 -0
- data/ext/extconf.rb +15 -6
- data/ext/extzstd.c +76 -145
- data/ext/extzstd.h +80 -31
- data/ext/extzstd_stream.c +417 -142
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +10 -7
- data/ext/zstd_compress.c +14 -5
- data/ext/zstd_decompress.c +5 -4
- data/ext/zstd_dictbuilder.c +9 -4
- data/ext/zstd_dictbuilder_fastcover.c +3 -0
- data/ext/zstd_legacy_v01.c +3 -1
- data/ext/zstd_legacy_v02.c +3 -1
- data/ext/zstd_legacy_v03.c +3 -1
- data/ext/zstd_legacy_v04.c +3 -1
- data/ext/zstd_legacy_v05.c +3 -1
- data/ext/zstd_legacy_v06.c +3 -1
- data/ext/zstd_legacy_v07.c +3 -1
- data/gemstub.rb +10 -24
- data/lib/extzstd.rb +64 -179
- data/lib/extzstd/version.rb +6 -1
- data/test/test_basic.rb +9 -6
- metadata +113 -57
- data/HISTORY.ja +0 -5
- data/contrib/zstd/common/entropy_common.c +0 -225
- data/contrib/zstd/common/huf.h +0 -228
- data/contrib/zstd/common/zstd_common.c +0 -83
- data/contrib/zstd/common/zstd_errors.h +0 -60
- data/contrib/zstd/common/zstd_internal.h +0 -267
- data/contrib/zstd/compress/huf_compress.c +0 -533
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/compress/zstd_compress.c +0 -3264
- data/contrib/zstd/compress/zstd_opt.h +0 -900
- data/contrib/zstd/decompress/huf_decompress.c +0 -883
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
- data/contrib/zstd/zstd.h +0 -640
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
* Copyright (c) 2016-
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
|
|
@@ -13,7 +14,7 @@
|
|
|
13
14
|
******************************************/
|
|
14
15
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
|
15
16
|
#include "zstd_v01.h"
|
|
16
|
-
#include "error_private.h"
|
|
17
|
+
#include "../common/error_private.h"
|
|
17
18
|
|
|
18
19
|
|
|
19
20
|
/******************************************
|
|
@@ -256,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
|
|
|
256
257
|
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
|
257
258
|
}
|
|
258
259
|
|
|
259
|
-
#endif
|
|
260
|
+
#endif /* FSE_FORCE_MEMORY_ACCESS */
|
|
260
261
|
|
|
261
262
|
static U16 FSE_readLE16(const void* memPtr)
|
|
262
263
|
{
|
|
@@ -338,14 +339,14 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
|
|
338
339
|
/****************************************************************
|
|
339
340
|
* Internal functions
|
|
340
341
|
****************************************************************/
|
|
341
|
-
FORCE_INLINE unsigned FSE_highbit32 (
|
|
342
|
+
FORCE_INLINE unsigned FSE_highbit32 (U32 val)
|
|
342
343
|
{
|
|
343
344
|
# if defined(_MSC_VER) /* Visual */
|
|
344
345
|
unsigned long r;
|
|
345
346
|
_BitScanReverse ( &r, val );
|
|
346
347
|
return (unsigned) r;
|
|
347
348
|
# elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */
|
|
348
|
-
return
|
|
349
|
+
return __builtin_clz (val) ^ 31;
|
|
349
350
|
# else /* Software version */
|
|
350
351
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
|
351
352
|
U32 v = val;
|
|
@@ -667,11 +668,17 @@ static size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t
|
|
|
667
668
|
switch(srcSize)
|
|
668
669
|
{
|
|
669
670
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
|
671
|
+
/* fallthrough */
|
|
670
672
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
|
673
|
+
/* fallthrough */
|
|
671
674
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
|
675
|
+
/* fallthrough */
|
|
672
676
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
|
677
|
+
/* fallthrough */
|
|
673
678
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
|
679
|
+
/* fallthrough */
|
|
674
680
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
|
681
|
+
/* fallthrough */
|
|
675
682
|
default:;
|
|
676
683
|
}
|
|
677
684
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
|
@@ -1066,99 +1073,102 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
|
1066
1073
|
const void* cSrc, size_t cSrcSize,
|
|
1067
1074
|
const U16* DTable)
|
|
1068
1075
|
{
|
|
1069
|
-
|
|
1070
|
-
BYTE* op = ostart;
|
|
1071
|
-
BYTE* const omax = op + maxDstSize;
|
|
1072
|
-
BYTE* const olimit = omax-15;
|
|
1073
|
-
|
|
1074
|
-
const void* ptr = DTable;
|
|
1075
|
-
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
|
1076
|
-
const U32 dtLog = DTable[0];
|
|
1077
|
-
size_t errorCode;
|
|
1078
|
-
U32 reloadStatus;
|
|
1079
|
-
|
|
1080
|
-
/* Init */
|
|
1081
|
-
|
|
1082
|
-
const U16* jumpTable = (const U16*)cSrc;
|
|
1083
|
-
const size_t length1 = FSE_readLE16(jumpTable);
|
|
1084
|
-
const size_t length2 = FSE_readLE16(jumpTable+1);
|
|
1085
|
-
const size_t length3 = FSE_readLE16(jumpTable+2);
|
|
1086
|
-
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
|
|
1087
|
-
const char* const start1 = (const char*)(cSrc) + 6;
|
|
1088
|
-
const char* const start2 = start1 + length1;
|
|
1089
|
-
const char* const start3 = start2 + length2;
|
|
1090
|
-
const char* const start4 = start3 + length3;
|
|
1091
|
-
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
|
|
1092
|
-
|
|
1093
|
-
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
|
1094
|
-
|
|
1095
|
-
errorCode = FSE_initDStream(&bitD1, start1, length1);
|
|
1096
|
-
if (FSE_isError(errorCode)) return errorCode;
|
|
1097
|
-
errorCode = FSE_initDStream(&bitD2, start2, length2);
|
|
1098
|
-
if (FSE_isError(errorCode)) return errorCode;
|
|
1099
|
-
errorCode = FSE_initDStream(&bitD3, start3, length3);
|
|
1100
|
-
if (FSE_isError(errorCode)) return errorCode;
|
|
1101
|
-
errorCode = FSE_initDStream(&bitD4, start4, length4);
|
|
1102
|
-
if (FSE_isError(errorCode)) return errorCode;
|
|
1103
|
-
|
|
1104
|
-
reloadStatus=FSE_reloadDStream(&bitD2);
|
|
1105
|
-
|
|
1106
|
-
/* 16 symbols per loop */
|
|
1107
|
-
for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
|
|
1108
|
-
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
|
|
1076
|
+
if (cSrcSize < 6) return (size_t)-FSE_ERROR_srcSize_wrong;
|
|
1109
1077
|
{
|
|
1110
|
-
|
|
1111
|
-
op
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1078
|
+
BYTE* const ostart = (BYTE*) dst;
|
|
1079
|
+
BYTE* op = ostart;
|
|
1080
|
+
BYTE* const omax = op + maxDstSize;
|
|
1081
|
+
BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
|
|
1082
|
+
|
|
1083
|
+
const void* ptr = DTable;
|
|
1084
|
+
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
|
1085
|
+
const U32 dtLog = DTable[0];
|
|
1086
|
+
size_t errorCode;
|
|
1087
|
+
U32 reloadStatus;
|
|
1088
|
+
|
|
1089
|
+
/* Init */
|
|
1090
|
+
|
|
1091
|
+
const U16* jumpTable = (const U16*)cSrc;
|
|
1092
|
+
const size_t length1 = FSE_readLE16(jumpTable);
|
|
1093
|
+
const size_t length2 = FSE_readLE16(jumpTable+1);
|
|
1094
|
+
const size_t length3 = FSE_readLE16(jumpTable+2);
|
|
1095
|
+
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
|
|
1096
|
+
const char* const start1 = (const char*)(cSrc) + 6;
|
|
1097
|
+
const char* const start2 = start1 + length1;
|
|
1098
|
+
const char* const start3 = start2 + length2;
|
|
1099
|
+
const char* const start4 = start3 + length3;
|
|
1100
|
+
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
|
|
1101
|
+
|
|
1102
|
+
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
|
1103
|
+
|
|
1104
|
+
errorCode = FSE_initDStream(&bitD1, start1, length1);
|
|
1105
|
+
if (FSE_isError(errorCode)) return errorCode;
|
|
1106
|
+
errorCode = FSE_initDStream(&bitD2, start2, length2);
|
|
1107
|
+
if (FSE_isError(errorCode)) return errorCode;
|
|
1108
|
+
errorCode = FSE_initDStream(&bitD3, start3, length3);
|
|
1109
|
+
if (FSE_isError(errorCode)) return errorCode;
|
|
1110
|
+
errorCode = FSE_initDStream(&bitD4, start4, length4);
|
|
1111
|
+
if (FSE_isError(errorCode)) return errorCode;
|
|
1112
|
+
|
|
1113
|
+
reloadStatus=FSE_reloadDStream(&bitD2);
|
|
1114
|
+
|
|
1115
|
+
/* 16 symbols per loop */
|
|
1116
|
+
for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
|
|
1117
|
+
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
|
|
1118
|
+
{
|
|
1119
|
+
#define HUF_DECODE_SYMBOL_0(n, Dstream) \
|
|
1120
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
|
|
1121
|
+
|
|
1122
|
+
#define HUF_DECODE_SYMBOL_1(n, Dstream) \
|
|
1123
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
|
|
1124
|
+
if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
|
|
1125
|
+
|
|
1126
|
+
#define HUF_DECODE_SYMBOL_2(n, Dstream) \
|
|
1127
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
|
|
1128
|
+
if (FSE_32bits()) FSE_reloadDStream(&Dstream)
|
|
1129
|
+
|
|
1130
|
+
HUF_DECODE_SYMBOL_1( 0, bitD1);
|
|
1131
|
+
HUF_DECODE_SYMBOL_1( 1, bitD2);
|
|
1132
|
+
HUF_DECODE_SYMBOL_1( 2, bitD3);
|
|
1133
|
+
HUF_DECODE_SYMBOL_1( 3, bitD4);
|
|
1134
|
+
HUF_DECODE_SYMBOL_2( 4, bitD1);
|
|
1135
|
+
HUF_DECODE_SYMBOL_2( 5, bitD2);
|
|
1136
|
+
HUF_DECODE_SYMBOL_2( 6, bitD3);
|
|
1137
|
+
HUF_DECODE_SYMBOL_2( 7, bitD4);
|
|
1138
|
+
HUF_DECODE_SYMBOL_1( 8, bitD1);
|
|
1139
|
+
HUF_DECODE_SYMBOL_1( 9, bitD2);
|
|
1140
|
+
HUF_DECODE_SYMBOL_1(10, bitD3);
|
|
1141
|
+
HUF_DECODE_SYMBOL_1(11, bitD4);
|
|
1142
|
+
HUF_DECODE_SYMBOL_0(12, bitD1);
|
|
1143
|
+
HUF_DECODE_SYMBOL_0(13, bitD2);
|
|
1144
|
+
HUF_DECODE_SYMBOL_0(14, bitD3);
|
|
1145
|
+
HUF_DECODE_SYMBOL_0(15, bitD4);
|
|
1146
|
+
}
|
|
1138
1147
|
|
|
1139
|
-
|
|
1140
|
-
|
|
1148
|
+
if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
|
|
1149
|
+
return (size_t)-FSE_ERROR_corruptionDetected;
|
|
1141
1150
|
|
|
1142
|
-
|
|
1143
|
-
{
|
|
1144
|
-
// bitTail = bitD1; // *much* slower : -20% !??!
|
|
1145
|
-
FSE_DStream_t bitTail;
|
|
1146
|
-
bitTail.ptr = bitD1.ptr;
|
|
1147
|
-
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
|
1148
|
-
bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
|
|
1149
|
-
bitTail.start = start1;
|
|
1150
|
-
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
|
1151
|
+
/* tail */
|
|
1151
1152
|
{
|
|
1152
|
-
|
|
1153
|
-
|
|
1153
|
+
/* bitTail = bitD1; */ /* *much* slower : -20% !??! */
|
|
1154
|
+
FSE_DStream_t bitTail;
|
|
1155
|
+
bitTail.ptr = bitD1.ptr;
|
|
1156
|
+
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
|
1157
|
+
bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
|
|
1158
|
+
bitTail.start = start1;
|
|
1159
|
+
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
|
1160
|
+
{
|
|
1161
|
+
HUF_DECODE_SYMBOL_0(0, bitTail);
|
|
1162
|
+
}
|
|
1154
1163
|
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1164
|
+
if (FSE_endOfDStream(&bitTail))
|
|
1165
|
+
return op-ostart;
|
|
1166
|
+
}
|
|
1158
1167
|
|
|
1159
|
-
|
|
1168
|
+
if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
|
|
1160
1169
|
|
|
1161
|
-
|
|
1170
|
+
return (size_t)-FSE_ERROR_corruptionDetected;
|
|
1171
|
+
}
|
|
1162
1172
|
}
|
|
1163
1173
|
|
|
1164
1174
|
|
|
@@ -1270,7 +1280,11 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
|
|
|
1270
1280
|
* Basic Types
|
|
1271
1281
|
*********************************************************/
|
|
1272
1282
|
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
|
1273
|
-
#
|
|
1283
|
+
# if defined(_AIX)
|
|
1284
|
+
# include <inttypes.h>
|
|
1285
|
+
# else
|
|
1286
|
+
# include <stdint.h> /* intptr_t */
|
|
1287
|
+
# endif
|
|
1274
1288
|
typedef uint8_t BYTE;
|
|
1275
1289
|
typedef uint16_t U16;
|
|
1276
1290
|
typedef int16_t S16;
|
|
@@ -1329,6 +1343,8 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51E; /* 3rd version : seqNb header
|
|
|
1329
1343
|
#define LITERAL_NOENTROPY 63
|
|
1330
1344
|
#define COMMAND_NOENTROPY 7 /* to remove */
|
|
1331
1345
|
|
|
1346
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
|
1347
|
+
|
|
1332
1348
|
static const size_t ZSTD_blockHeaderSize = 3;
|
|
1333
1349
|
static const size_t ZSTD_frameHeaderSize = 4;
|
|
1334
1350
|
|
|
@@ -1346,15 +1362,13 @@ static unsigned ZSTD_isLittleEndian(void)
|
|
|
1346
1362
|
|
|
1347
1363
|
static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); return r; }
|
|
1348
1364
|
|
|
1349
|
-
static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; }
|
|
1350
|
-
|
|
1351
1365
|
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
|
1352
1366
|
|
|
1353
1367
|
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
1354
1368
|
|
|
1355
1369
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
|
1356
1370
|
|
|
1357
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
|
1371
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
|
1358
1372
|
{
|
|
1359
1373
|
const BYTE* ip = (const BYTE*)src;
|
|
1360
1374
|
BYTE* op = (BYTE*)dst;
|
|
@@ -1372,16 +1386,9 @@ static U16 ZSTD_readLE16(const void* memPtr)
|
|
|
1372
1386
|
}
|
|
1373
1387
|
}
|
|
1374
1388
|
|
|
1375
|
-
|
|
1376
|
-
static U32 ZSTD_readLE32(const void* memPtr)
|
|
1389
|
+
static U32 ZSTD_readLE24(const void* memPtr)
|
|
1377
1390
|
{
|
|
1378
|
-
|
|
1379
|
-
return ZSTD_read32(memPtr);
|
|
1380
|
-
else
|
|
1381
|
-
{
|
|
1382
|
-
const BYTE* p = (const BYTE*)memPtr;
|
|
1383
|
-
return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
|
|
1384
|
-
}
|
|
1391
|
+
return ZSTD_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
|
1385
1392
|
}
|
|
1386
1393
|
|
|
1387
1394
|
static U32 ZSTD_readBE32(const void* memPtr)
|
|
@@ -1432,7 +1439,7 @@ typedef struct ZSTD_Cctx_s
|
|
|
1432
1439
|
#else
|
|
1433
1440
|
U32 hashTable[HASH_TABLESIZE];
|
|
1434
1441
|
#endif
|
|
1435
|
-
|
|
1442
|
+
BYTE buffer[WORKPLACESIZE];
|
|
1436
1443
|
} cctxi_t;
|
|
1437
1444
|
|
|
1438
1445
|
|
|
@@ -1457,7 +1464,7 @@ unsigned ZSTDv01_isError(size_t code) { return ERR_isError(code); }
|
|
|
1457
1464
|
* Decompression code
|
|
1458
1465
|
**************************************************************/
|
|
1459
1466
|
|
|
1460
|
-
size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
|
1467
|
+
static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
|
1461
1468
|
{
|
|
1462
1469
|
const BYTE* const in = (const BYTE* const)src;
|
|
1463
1470
|
BYTE headerFlags;
|
|
@@ -1480,7 +1487,9 @@ size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|
|
1480
1487
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
|
1481
1488
|
{
|
|
1482
1489
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
|
1483
|
-
|
|
1490
|
+
if (srcSize > 0) {
|
|
1491
|
+
memcpy(dst, src, srcSize);
|
|
1492
|
+
}
|
|
1484
1493
|
return srcSize;
|
|
1485
1494
|
}
|
|
1486
1495
|
|
|
@@ -1499,7 +1508,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
|
|
1499
1508
|
if (srcSize <= 3) return ERROR(corruption_detected);
|
|
1500
1509
|
|
|
1501
1510
|
litSize = ip[1] + (ip[0]<<8);
|
|
1502
|
-
litSize += ((ip[-3] >> 3) & 7) << 16;
|
|
1511
|
+
litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
|
|
1503
1512
|
op = oend - litSize;
|
|
1504
1513
|
|
|
1505
1514
|
(void)ctx;
|
|
@@ -1510,7 +1519,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
|
|
1510
1519
|
}
|
|
1511
1520
|
|
|
1512
1521
|
|
|
1513
|
-
size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
1522
|
+
static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
1514
1523
|
void* dst, size_t maxDstSize,
|
|
1515
1524
|
const BYTE** litStart, size_t* litSize,
|
|
1516
1525
|
const void* src, size_t srcSize)
|
|
@@ -1538,7 +1547,9 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
|
1538
1547
|
size_t rleSize = litbp.origSize;
|
|
1539
1548
|
if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
|
|
1540
1549
|
if (!srcSize) return ERROR(srcSize_wrong);
|
|
1541
|
-
|
|
1550
|
+
if (rleSize > 0) {
|
|
1551
|
+
memset(oend - rleSize, *ip, rleSize);
|
|
1552
|
+
}
|
|
1542
1553
|
*litStart = oend - rleSize;
|
|
1543
1554
|
*litSize = rleSize;
|
|
1544
1555
|
ip++;
|
|
@@ -1562,7 +1573,7 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
|
1562
1573
|
}
|
|
1563
1574
|
|
|
1564
1575
|
|
|
1565
|
-
size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
|
1576
|
+
static size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
|
1566
1577
|
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
|
|
1567
1578
|
const void* src, size_t srcSize)
|
|
1568
1579
|
{
|
|
@@ -1695,13 +1706,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
1695
1706
|
seqState->prevOffset = seq->offset;
|
|
1696
1707
|
if (litLength == MaxLL)
|
|
1697
1708
|
{
|
|
1698
|
-
U32 add = dumps<de ? *dumps++ : 0;
|
|
1709
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
|
1699
1710
|
if (add < 255) litLength += add;
|
|
1700
1711
|
else
|
|
1701
1712
|
{
|
|
1702
1713
|
if (dumps<=(de-3))
|
|
1703
1714
|
{
|
|
1704
|
-
litLength =
|
|
1715
|
+
litLength = ZSTD_readLE24(dumps);
|
|
1705
1716
|
dumps += 3;
|
|
1706
1717
|
}
|
|
1707
1718
|
}
|
|
@@ -1723,13 +1734,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
1723
1734
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
|
1724
1735
|
if (matchLength == MaxML)
|
|
1725
1736
|
{
|
|
1726
|
-
U32 add = dumps<de ? *dumps++ : 0;
|
|
1737
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
|
1727
1738
|
if (add < 255) matchLength += add;
|
|
1728
1739
|
else
|
|
1729
1740
|
{
|
|
1730
1741
|
if (dumps<=(de-3))
|
|
1731
1742
|
{
|
|
1732
|
-
matchLength =
|
|
1743
|
+
matchLength = ZSTD_readLE24(dumps);
|
|
1733
1744
|
dumps += 3;
|
|
1734
1745
|
}
|
|
1735
1746
|
}
|
|
@@ -1750,7 +1761,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
1750
1761
|
BYTE* const base, BYTE* const oend)
|
|
1751
1762
|
{
|
|
1752
1763
|
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
|
1753
|
-
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /*
|
|
1764
|
+
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
|
1754
1765
|
const BYTE* const ostart = op;
|
|
1755
1766
|
const size_t litLength = sequence.litLength;
|
|
1756
1767
|
BYTE* const endMatch = op + litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
|
@@ -1803,7 +1814,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
1803
1814
|
} else { ZSTD_copy8(op, match); }
|
|
1804
1815
|
op += 8; match += 8;
|
|
1805
1816
|
|
|
1806
|
-
if (endMatch > oend-
|
|
1817
|
+
if (endMatch > oend-(16-MINMATCH))
|
|
1807
1818
|
{
|
|
1808
1819
|
if (op < oend-8)
|
|
1809
1820
|
{
|
|
@@ -1814,7 +1825,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
1814
1825
|
while (op<endMatch) *op++ = *match++;
|
|
1815
1826
|
}
|
|
1816
1827
|
else
|
|
1817
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
1828
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
|
1818
1829
|
|
|
1819
1830
|
/* restore, in case of overlap */
|
|
1820
1831
|
if (overlapRisk) memcpy(endMatch, saved, qutt);
|
|
@@ -1898,8 +1909,10 @@ static size_t ZSTD_decompressSequences(
|
|
|
1898
1909
|
{
|
|
1899
1910
|
size_t lastLLSize = litEnd - litPtr;
|
|
1900
1911
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
|
1901
|
-
if (
|
|
1902
|
-
|
|
1912
|
+
if (lastLLSize > 0) {
|
|
1913
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
|
1914
|
+
op += lastLLSize;
|
|
1915
|
+
}
|
|
1903
1916
|
}
|
|
1904
1917
|
}
|
|
1905
1918
|
|
|
@@ -1992,6 +2005,60 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|
|
1992
2005
|
return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
|
1993
2006
|
}
|
|
1994
2007
|
|
|
2008
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
|
2009
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
|
2010
|
+
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
|
2011
|
+
{
|
|
2012
|
+
*cSize = ret;
|
|
2013
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
|
2014
|
+
}
|
|
2015
|
+
|
|
2016
|
+
void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
|
2017
|
+
{
|
|
2018
|
+
const BYTE* ip = (const BYTE*)src;
|
|
2019
|
+
size_t remainingSize = srcSize;
|
|
2020
|
+
size_t nbBlocks = 0;
|
|
2021
|
+
U32 magicNumber;
|
|
2022
|
+
blockProperties_t blockProperties;
|
|
2023
|
+
|
|
2024
|
+
/* Frame Header */
|
|
2025
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
|
|
2026
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
2027
|
+
return;
|
|
2028
|
+
}
|
|
2029
|
+
magicNumber = ZSTD_readBE32(src);
|
|
2030
|
+
if (magicNumber != ZSTD_magicNumber) {
|
|
2031
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
|
2032
|
+
return;
|
|
2033
|
+
}
|
|
2034
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
|
2035
|
+
|
|
2036
|
+
/* Loop on each block */
|
|
2037
|
+
while (1)
|
|
2038
|
+
{
|
|
2039
|
+
size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties);
|
|
2040
|
+
if (ZSTDv01_isError(blockSize)) {
|
|
2041
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, blockSize);
|
|
2042
|
+
return;
|
|
2043
|
+
}
|
|
2044
|
+
|
|
2045
|
+
ip += ZSTD_blockHeaderSize;
|
|
2046
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
|
2047
|
+
if (blockSize > remainingSize) {
|
|
2048
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
|
2049
|
+
return;
|
|
2050
|
+
}
|
|
2051
|
+
|
|
2052
|
+
if (blockSize == 0) break; /* bt_end */
|
|
2053
|
+
|
|
2054
|
+
ip += blockSize;
|
|
2055
|
+
remainingSize -= blockSize;
|
|
2056
|
+
nbBlocks++;
|
|
2057
|
+
}
|
|
2058
|
+
|
|
2059
|
+
*cSize = ip - (const BYTE*)src;
|
|
2060
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
|
2061
|
+
}
|
|
1995
2062
|
|
|
1996
2063
|
/*******************************
|
|
1997
2064
|
* Streaming Decompression API
|