extzstd 0.0.3.CONCEPT → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +39 -0
- data/LICENSE +6 -6
- data/README.md +26 -45
- data/contrib/zstd/CHANGELOG +555 -0
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/CONTRIBUTING.md +392 -0
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/LICENSE +13 -9
- data/contrib/zstd/Makefile +414 -0
- data/contrib/zstd/README.md +170 -45
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +289 -0
- data/contrib/zstd/lib/BUCK +234 -0
- data/contrib/zstd/lib/Makefile +354 -0
- data/contrib/zstd/lib/README.md +179 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +170 -130
- data/contrib/zstd/lib/common/compiler.h +175 -0
- data/contrib/zstd/lib/common/cpu.h +215 -0
- data/contrib/zstd/lib/common/debug.c +24 -0
- data/contrib/zstd/lib/common/debug.h +114 -0
- data/contrib/zstd/{common → lib/common}/entropy_common.c +79 -94
- data/contrib/zstd/lib/common/error_private.c +55 -0
- data/contrib/zstd/lib/common/error_private.h +80 -0
- data/contrib/zstd/{common → lib/common}/fse.h +153 -93
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +37 -82
- data/contrib/zstd/lib/common/huf.h +340 -0
- data/contrib/zstd/{common → lib/common}/mem.h +154 -78
- data/contrib/zstd/lib/common/pool.c +344 -0
- data/contrib/zstd/lib/common/pool.h +84 -0
- data/contrib/zstd/lib/common/threading.c +121 -0
- data/contrib/zstd/lib/common/threading.h +155 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +85 -75
- data/contrib/zstd/{common → lib/common}/xxhash.h +85 -73
- data/contrib/zstd/lib/common/zstd_common.c +83 -0
- data/contrib/zstd/lib/common/zstd_errors.h +94 -0
- data/contrib/zstd/lib/common/zstd_internal.h +447 -0
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +194 -303
- data/contrib/zstd/lib/compress/hist.c +183 -0
- data/contrib/zstd/lib/compress/hist.h +75 -0
- data/contrib/zstd/lib/compress/huf_compress.c +798 -0
- data/contrib/zstd/lib/compress/zstd_compress.c +4278 -0
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +1125 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +419 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +845 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +525 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +1138 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +67 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +619 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +110 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +1200 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.c +2143 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +192 -0
- data/contrib/zstd/lib/decompress/huf_decompress.c +1248 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +1885 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1432 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +189 -0
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +86 -69
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1236 -0
- data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +5 -5
- data/contrib/zstd/lib/dictBuilder/fastcover.c +757 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +437 -347
- data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
- data/contrib/zstd/lib/legacy/zstd_legacy.h +415 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +272 -292
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +26 -32
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +162 -392
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +26 -32
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +162 -391
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +27 -33
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +195 -604
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +26 -32
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +300 -575
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +22 -31
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +165 -592
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +54 -67
- data/contrib/zstd/lib/legacy/zstd_v07.c +4541 -0
- data/contrib/zstd/lib/legacy/zstd_v07.h +187 -0
- data/contrib/zstd/lib/libzstd.pc.in +15 -0
- data/contrib/zstd/lib/zstd.h +2090 -0
- data/ext/depend +2 -0
- data/ext/extconf.rb +18 -5
- data/ext/extzstd.c +296 -214
- data/ext/extzstd.h +81 -36
- data/ext/extzstd_nogvls.h +0 -117
- data/ext/extzstd_stream.c +622 -0
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +11 -0
- data/ext/zstd_compress.c +15 -0
- data/ext/zstd_decompress.c +6 -0
- data/ext/zstd_dictbuilder.c +10 -0
- data/ext/zstd_dictbuilder_fastcover.c +3 -0
- data/ext/zstd_legacy_v01.c +3 -1
- data/ext/zstd_legacy_v02.c +3 -1
- data/ext/zstd_legacy_v03.c +3 -1
- data/ext/zstd_legacy_v04.c +3 -1
- data/ext/zstd_legacy_v05.c +3 -1
- data/ext/zstd_legacy_v06.c +3 -1
- data/ext/zstd_legacy_v07.c +3 -0
- data/gemstub.rb +27 -21
- data/lib/extzstd.rb +82 -161
- data/lib/extzstd/version.rb +1 -1
- data/test/test_basic.rb +19 -6
- metadata +127 -59
- data/contrib/zstd/common/error_private.h +0 -125
- data/contrib/zstd/common/error_public.h +0 -77
- data/contrib/zstd/common/huf.h +0 -228
- data/contrib/zstd/common/zstd.h +0 -475
- data/contrib/zstd/common/zstd_common.c +0 -91
- data/contrib/zstd/common/zstd_internal.h +0 -238
- data/contrib/zstd/compress/huf_compress.c +0 -577
- data/contrib/zstd/compress/zbuff_compress.c +0 -327
- data/contrib/zstd/compress/zstd_compress.c +0 -3074
- data/contrib/zstd/compress/zstd_opt.h +0 -1046
- data/contrib/zstd/decompress/huf_decompress.c +0 -894
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -294
- data/contrib/zstd/decompress/zstd_decompress.c +0 -1362
- data/contrib/zstd/dictBuilder/zdict.h +0 -113
- data/contrib/zstd/legacy/zstd_legacy.h +0 -140
- data/ext/extzstd_buffered.c +0 -265
- data/ext/zstd_amalgam.c +0 -18
@@ -1,36 +1,15 @@
|
|
1
1
|
/*
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
* Redistributions in binary form must reproduce the above
|
14
|
-
copyright notice, this list of conditions and the following disclaimer
|
15
|
-
in the documentation and/or other materials provided with the
|
16
|
-
distribution.
|
17
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
18
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
19
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
20
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
21
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
22
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
23
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
24
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
25
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
26
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
27
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
28
|
-
|
29
|
-
You can contact the author at :
|
30
|
-
- zstd source repository : https://github.com/Cyan4973/zstd
|
31
|
-
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
|
32
|
-
*/
|
33
|
-
#pragma once
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
11
|
+
#ifndef ZSTD_V02_H_4174539423
|
12
|
+
#define ZSTD_V02_H_4174539423
|
34
13
|
|
35
14
|
#if defined (__cplusplus)
|
36
15
|
extern "C" {
|
@@ -56,6 +35,19 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
|
|
56
35
|
size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
57
36
|
const void* src, size_t compressedSize);
|
58
37
|
|
38
|
+
/**
|
39
|
+
ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format
|
40
|
+
srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
41
|
+
cSize (output parameter) : the number of bytes that would be read to decompress this frame
|
42
|
+
or an error code if it fails (which can be tested using ZSTDv01_isError())
|
43
|
+
dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
|
44
|
+
or ZSTD_CONTENTSIZE_ERROR if an error occurs
|
45
|
+
|
46
|
+
note : assumes `cSize` and `dBound` are _not_ NULL.
|
47
|
+
*/
|
48
|
+
void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
|
49
|
+
size_t* cSize, unsigned long long* dBound);
|
50
|
+
|
59
51
|
/**
|
60
52
|
ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
|
61
53
|
*/
|
@@ -97,3 +89,5 @@ size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSi
|
|
97
89
|
#if defined (__cplusplus)
|
98
90
|
}
|
99
91
|
#endif
|
92
|
+
|
93
|
+
#endif /* ZSTD_V02_H_4174539423 */
|
@@ -1,89 +1,27 @@
|
|
1
|
-
/*
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
* Redistributions of source code must retain the above copyright
|
12
|
-
notice, this list of conditions and the following disclaimer.
|
13
|
-
* Redistributions in binary form must reproduce the above
|
14
|
-
copyright notice, this list of conditions and the following disclaimer
|
15
|
-
in the documentation and/or other materials provided with the
|
16
|
-
distribution.
|
17
|
-
|
18
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
-
|
30
|
-
You can contact the author at :
|
31
|
-
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
32
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
33
|
-
****************************************************************** */
|
34
|
-
#ifndef ERROR_H_MODULE
|
35
|
-
#define ERROR_H_MODULE
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
36
10
|
|
37
|
-
#if defined (__cplusplus)
|
38
|
-
extern "C" {
|
39
|
-
#endif
|
40
11
|
|
41
12
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
42
13
|
#include "zstd_v03.h"
|
43
|
-
|
44
|
-
/******************************************
|
45
|
-
* Compiler-specific
|
46
|
-
******************************************/
|
47
|
-
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
48
|
-
# define ERR_STATIC static inline
|
49
|
-
#elif defined(_MSC_VER)
|
50
|
-
# define ERR_STATIC static __inline
|
51
|
-
#elif defined(__GNUC__)
|
52
|
-
# define ERR_STATIC static __attribute__((unused))
|
53
|
-
#else
|
54
|
-
# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
55
|
-
#endif
|
14
|
+
#include "../common/error_private.h"
|
56
15
|
|
57
16
|
|
58
17
|
/******************************************
|
59
|
-
*
|
18
|
+
* Compiler-specific
|
60
19
|
******************************************/
|
61
|
-
#
|
62
|
-
|
63
|
-
#
|
64
|
-
|
65
|
-
#define ERROR_LIST(ITEM) \
|
66
|
-
ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
|
67
|
-
ITEM(PREFIX(memory_allocation)) \
|
68
|
-
ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
|
69
|
-
ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \
|
70
|
-
ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \
|
71
|
-
ITEM(PREFIX(maxCode))
|
72
|
-
|
73
|
-
#define ERROR_GENERATE_ENUM(ENUM) ENUM,
|
74
|
-
typedef enum { ERROR_LIST(ERROR_GENERATE_ENUM) } ERR_codes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */
|
75
|
-
|
76
|
-
#define ERROR_CONVERTTOSTRING(STRING) #STRING,
|
77
|
-
#define ERROR_GENERATE_STRING(EXPR) ERROR_CONVERTTOSTRING(EXPR)
|
78
|
-
|
79
|
-
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
|
80
|
-
|
81
|
-
|
82
|
-
#if defined (__cplusplus)
|
83
|
-
}
|
20
|
+
#if defined(_MSC_VER) /* Visual Studio */
|
21
|
+
# include <stdlib.h> /* _byteswap_ulong */
|
22
|
+
# include <intrin.h> /* _byteswap_* */
|
84
23
|
#endif
|
85
24
|
|
86
|
-
#endif /* ERROR_H_MODULE */
|
87
25
|
|
88
26
|
|
89
27
|
/* ******************************************************************
|
@@ -190,7 +128,7 @@ extern "C" {
|
|
190
128
|
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
191
129
|
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
192
130
|
# define MEM_FORCE_MEMORY_ACCESS 2
|
193
|
-
# elif defined(__INTEL_COMPILER) || \
|
131
|
+
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
194
132
|
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
195
133
|
# define MEM_FORCE_MEMORY_ACCESS 1
|
196
134
|
# endif
|
@@ -214,8 +152,6 @@ MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
|
214
152
|
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
215
153
|
|
216
154
|
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
217
|
-
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
|
218
|
-
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
|
219
155
|
|
220
156
|
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
221
157
|
|
@@ -228,8 +164,6 @@ MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32;
|
|
228
164
|
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
229
165
|
|
230
166
|
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
231
|
-
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
|
232
|
-
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
|
233
167
|
|
234
168
|
#else
|
235
169
|
|
@@ -256,17 +190,8 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
|
256
190
|
memcpy(memPtr, &value, sizeof(value));
|
257
191
|
}
|
258
192
|
|
259
|
-
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
|
260
|
-
{
|
261
|
-
memcpy(memPtr, &value, sizeof(value));
|
262
|
-
}
|
263
|
-
|
264
|
-
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
|
265
|
-
{
|
266
|
-
memcpy(memPtr, &value, sizeof(value));
|
267
|
-
}
|
268
193
|
|
269
|
-
#endif
|
194
|
+
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
270
195
|
|
271
196
|
|
272
197
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
@@ -294,6 +219,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
|
294
219
|
}
|
295
220
|
}
|
296
221
|
|
222
|
+
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
|
223
|
+
{
|
224
|
+
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
225
|
+
}
|
226
|
+
|
297
227
|
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
298
228
|
{
|
299
229
|
if (MEM_isLittleEndian())
|
@@ -305,22 +235,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
|
305
235
|
}
|
306
236
|
}
|
307
237
|
|
308
|
-
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
|
309
|
-
{
|
310
|
-
if (MEM_isLittleEndian())
|
311
|
-
{
|
312
|
-
MEM_write32(memPtr, val32);
|
313
|
-
}
|
314
|
-
else
|
315
|
-
{
|
316
|
-
BYTE* p = (BYTE*)memPtr;
|
317
|
-
p[0] = (BYTE)val32;
|
318
|
-
p[1] = (BYTE)(val32>>8);
|
319
|
-
p[2] = (BYTE)(val32>>16);
|
320
|
-
p[3] = (BYTE)(val32>>24);
|
321
|
-
}
|
322
|
-
}
|
323
|
-
|
324
238
|
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
|
325
239
|
{
|
326
240
|
if (MEM_isLittleEndian())
|
@@ -333,25 +247,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
|
|
333
247
|
}
|
334
248
|
}
|
335
249
|
|
336
|
-
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
|
337
|
-
{
|
338
|
-
if (MEM_isLittleEndian())
|
339
|
-
{
|
340
|
-
MEM_write64(memPtr, val64);
|
341
|
-
}
|
342
|
-
else
|
343
|
-
{
|
344
|
-
BYTE* p = (BYTE*)memPtr;
|
345
|
-
p[0] = (BYTE)val64;
|
346
|
-
p[1] = (BYTE)(val64>>8);
|
347
|
-
p[2] = (BYTE)(val64>>16);
|
348
|
-
p[3] = (BYTE)(val64>>24);
|
349
|
-
p[4] = (BYTE)(val64>>32);
|
350
|
-
p[5] = (BYTE)(val64>>40);
|
351
|
-
p[6] = (BYTE)(val64>>48);
|
352
|
-
p[7] = (BYTE)(val64>>56);
|
353
|
-
}
|
354
|
-
}
|
355
250
|
|
356
251
|
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
357
252
|
{
|
@@ -361,13 +256,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
|
361
256
|
return (size_t)MEM_readLE64(memPtr);
|
362
257
|
}
|
363
258
|
|
364
|
-
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
|
365
|
-
{
|
366
|
-
if (MEM_32bits())
|
367
|
-
MEM_writeLE32(memPtr, (U32)val);
|
368
|
-
else
|
369
|
-
MEM_writeLE64(memPtr, (U64)val);
|
370
|
-
}
|
371
259
|
|
372
260
|
#if defined (__cplusplus)
|
373
261
|
}
|
@@ -449,17 +337,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
|
449
337
|
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
450
338
|
|
451
339
|
|
452
|
-
/*
|
453
|
-
* Start by invoking BIT_initDStream().
|
454
|
-
* A chunk of the bitStream is then stored into a local register.
|
455
|
-
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
456
|
-
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
457
|
-
* Local register is manually filled from memory by the BIT_reloadDStream() method.
|
458
|
-
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
459
|
-
* Otherwise, it can be less than that, so proceed accordingly.
|
460
|
-
* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
|
461
|
-
*/
|
462
|
-
|
463
340
|
|
464
341
|
/******************************************
|
465
342
|
* unsafe API
|
@@ -472,14 +349,14 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
|
472
349
|
/****************************************************************
|
473
350
|
* Helper functions
|
474
351
|
****************************************************************/
|
475
|
-
MEM_STATIC unsigned BIT_highbit32 (
|
352
|
+
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
476
353
|
{
|
477
354
|
# if defined(_MSC_VER) /* Visual */
|
478
355
|
unsigned long r=0;
|
479
356
|
_BitScanReverse ( &r, val );
|
480
357
|
return (unsigned) r;
|
481
358
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
482
|
-
return
|
359
|
+
return __builtin_clz (val) ^ 31;
|
483
360
|
# else /* Software version */
|
484
361
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
485
362
|
U32 v = val;
|
@@ -530,11 +407,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
530
407
|
switch(srcSize)
|
531
408
|
{
|
532
409
|
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
410
|
+
/* fallthrough */
|
533
411
|
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
412
|
+
/* fallthrough */
|
534
413
|
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
414
|
+
/* fallthrough */
|
535
415
|
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
416
|
+
/* fallthrough */
|
536
417
|
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
418
|
+
/* fallthrough */
|
537
419
|
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
420
|
+
/* fallthrough */
|
538
421
|
default:;
|
539
422
|
}
|
540
423
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
@@ -545,14 +428,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
545
428
|
|
546
429
|
return srcSize;
|
547
430
|
}
|
548
|
-
|
549
|
-
/*!BIT_lookBits
|
550
|
-
* Provides next n bits from local register
|
551
|
-
* local register is not modified (bits are still present for next read/look)
|
552
|
-
* On 32-bits, maxNbBits==25
|
553
|
-
* On 64-bits, maxNbBits==57
|
554
|
-
* @return : value extracted
|
555
|
-
*/
|
556
431
|
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
557
432
|
{
|
558
433
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
@@ -572,11 +447,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
572
447
|
bitD->bitsConsumed += nbBits;
|
573
448
|
}
|
574
449
|
|
575
|
-
/*!BIT_readBits
|
576
|
-
* Read next n bits from local register.
|
577
|
-
* pay attention to not read more than nbBits contained into local register.
|
578
|
-
* @return : extracted value.
|
579
|
-
*/
|
580
450
|
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
581
451
|
{
|
582
452
|
size_t value = BIT_lookBits(bitD, nbBits);
|
@@ -595,8 +465,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
|
595
465
|
|
596
466
|
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
597
467
|
{
|
598
|
-
|
599
|
-
|
468
|
+
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
469
|
+
return BIT_DStream_overflow;
|
600
470
|
|
601
471
|
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
|
602
472
|
{
|
@@ -814,55 +684,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
|
|
814
684
|
|
815
685
|
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
816
686
|
|
817
|
-
/*
|
818
|
-
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
819
|
-
You will decode FSE-encoded symbols from the bitStream,
|
820
|
-
and also any other bitFields you put in, **in reverse order**.
|
821
|
-
|
822
|
-
You will need a few variables to track your bitStream. They are :
|
823
|
-
|
824
|
-
BIT_DStream_t DStream; // Stream context
|
825
|
-
FSE_DState_t DState; // State context. Multiple ones are possible
|
826
|
-
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
827
|
-
|
828
|
-
The first thing to do is to init the bitStream.
|
829
|
-
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
830
|
-
|
831
|
-
You should then retrieve your initial state(s)
|
832
|
-
(in reverse flushing order if you have several ones) :
|
833
|
-
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
834
|
-
|
835
|
-
You can then decode your data, symbol after symbol.
|
836
|
-
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
837
|
-
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
838
|
-
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
839
|
-
|
840
|
-
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
841
|
-
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
842
|
-
size_t bitField = BIT_readBits(&DStream, nbBits);
|
843
|
-
|
844
|
-
All above operations only read from local register (which size depends on size_t).
|
845
|
-
Refueling the register from memory is manually performed by the reload method.
|
846
|
-
endSignal = FSE_reloadDStream(&DStream);
|
847
|
-
|
848
|
-
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
849
|
-
BIT_DStream_unfinished : there is still some data left into the DStream.
|
850
|
-
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
851
|
-
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
852
|
-
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
853
|
-
|
854
|
-
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
855
|
-
to properly detect the exact end of stream.
|
856
|
-
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
857
|
-
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
858
|
-
|
859
|
-
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
860
|
-
Checking if DStream has reached its end is performed by :
|
861
|
-
BIT_endOfDStream(&DStream);
|
862
|
-
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
863
|
-
FSE_endOfDState(&DState);
|
864
|
-
*/
|
865
|
-
|
866
687
|
|
867
688
|
/******************************************
|
868
689
|
* FSE unsafe API
|
@@ -1197,12 +1018,15 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
|
1197
1018
|
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
1198
1019
|
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
|
1199
1020
|
#else
|
1200
|
-
#
|
1201
|
-
#
|
1202
|
-
#
|
1021
|
+
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
1022
|
+
# ifdef __GNUC__
|
1023
|
+
# define FORCE_INLINE static inline __attribute__((always_inline))
|
1024
|
+
# else
|
1025
|
+
# define FORCE_INLINE static inline
|
1026
|
+
# endif
|
1203
1027
|
# else
|
1204
|
-
# define FORCE_INLINE static
|
1205
|
-
# endif
|
1028
|
+
# define FORCE_INLINE static
|
1029
|
+
# endif /* __STDC_VERSION__ */
|
1206
1030
|
#endif
|
1207
1031
|
|
1208
1032
|
|
@@ -1450,8 +1274,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
|
|
1450
1274
|
else
|
1451
1275
|
{
|
1452
1276
|
bitCount -= (int)(8 * (iend - 4 - ip));
|
1453
|
-
|
1454
|
-
|
1277
|
+
ip = iend - 4;
|
1278
|
+
}
|
1455
1279
|
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
|
1456
1280
|
}
|
1457
1281
|
}
|
@@ -1663,25 +1487,13 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
|
|
1663
1487
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
1664
1488
|
/* inline is defined */
|
1665
1489
|
#elif defined(_MSC_VER)
|
1490
|
+
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
1666
1491
|
# define inline __inline
|
1667
1492
|
#else
|
1668
1493
|
# define inline /* disable inline */
|
1669
1494
|
#endif
|
1670
1495
|
|
1671
1496
|
|
1672
|
-
#ifdef _MSC_VER /* Visual Studio */
|
1673
|
-
# define FORCE_INLINE static __forceinline
|
1674
|
-
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
1675
|
-
#else
|
1676
|
-
# ifdef __GNUC__
|
1677
|
-
# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
1678
|
-
# define FORCE_INLINE static inline __attribute__((always_inline))
|
1679
|
-
# else
|
1680
|
-
# define FORCE_INLINE static inline
|
1681
|
-
# endif
|
1682
|
-
#endif
|
1683
|
-
|
1684
|
-
|
1685
1497
|
/****************************************************************
|
1686
1498
|
* Includes
|
1687
1499
|
****************************************************************/
|
@@ -1731,10 +1543,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
|
1731
1543
|
U32 weightTotal;
|
1732
1544
|
U32 tableLog;
|
1733
1545
|
const BYTE* ip = (const BYTE*) src;
|
1734
|
-
size_t iSize
|
1546
|
+
size_t iSize;
|
1735
1547
|
size_t oSize;
|
1736
1548
|
U32 n;
|
1737
1549
|
|
1550
|
+
if (!srcSize) return ERROR(srcSize_wrong);
|
1551
|
+
iSize = ip[0];
|
1738
1552
|
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
|
1739
1553
|
|
1740
1554
|
if (iSize >= 128) /* special header */
|
@@ -1776,6 +1590,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
|
1776
1590
|
rankStats[huffWeight[n]]++;
|
1777
1591
|
weightTotal += (1 << huffWeight[n]) >> 1;
|
1778
1592
|
}
|
1593
|
+
if (weightTotal == 0) return ERROR(corruption_detected);
|
1779
1594
|
|
1780
1595
|
/* get last non-null symbol weight (implied, total must be 2^n) */
|
1781
1596
|
tableLog = BIT_highbit32(weightTotal) + 1;
|
@@ -2161,7 +1976,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
|
2161
1976
|
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
|
2162
1977
|
}
|
2163
1978
|
|
2164
|
-
|
1979
|
+
/* Build rankVal */
|
2165
1980
|
{
|
2166
1981
|
const U32 minBits = tableLog+1 - maxW;
|
2167
1982
|
U32 nextRankVal = 0;
|
@@ -2507,17 +2322,11 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
2507
2322
|
#endif
|
2508
2323
|
|
2509
2324
|
#ifdef _MSC_VER /* Visual Studio */
|
2510
|
-
# define FORCE_INLINE static __forceinline
|
2511
2325
|
# include <intrin.h> /* For Visual 2005 */
|
2512
2326
|
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
2513
2327
|
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
|
2514
2328
|
#else
|
2515
2329
|
# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
2516
|
-
# ifdef __GNUC__
|
2517
|
-
# define FORCE_INLINE static inline __attribute__((always_inline))
|
2518
|
-
# else
|
2519
|
-
# define FORCE_INLINE static inline
|
2520
|
-
# endif
|
2521
2330
|
#endif
|
2522
2331
|
|
2523
2332
|
|
@@ -2565,6 +2374,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
2565
2374
|
#define LITERAL_NOENTROPY 63
|
2566
2375
|
#define COMMAND_NOENTROPY 7 /* to remove */
|
2567
2376
|
|
2377
|
+
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
2378
|
+
|
2568
2379
|
static const size_t ZSTD_blockHeaderSize = 3;
|
2569
2380
|
static const size_t ZSTD_frameHeaderSize = 4;
|
2570
2381
|
|
@@ -2579,7 +2390,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
2579
2390
|
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
2580
2391
|
|
2581
2392
|
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
|
2582
|
-
static void ZSTD_wildcopy(void* dst, const void* src,
|
2393
|
+
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
2583
2394
|
{
|
2584
2395
|
const BYTE* ip = (const BYTE*)src;
|
2585
2396
|
BYTE* op = (BYTE*)dst;
|
@@ -2624,98 +2435,6 @@ typedef struct {
|
|
2624
2435
|
static unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
|
2625
2436
|
|
2626
2437
|
|
2627
|
-
/* *************************************
|
2628
|
-
* Function body to include
|
2629
|
-
***************************************/
|
2630
|
-
static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
|
2631
|
-
|
2632
|
-
MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
|
2633
|
-
{
|
2634
|
-
if (MEM_isLittleEndian())
|
2635
|
-
{
|
2636
|
-
if (MEM_64bits())
|
2637
|
-
{
|
2638
|
-
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
|
2639
|
-
unsigned long r = 0;
|
2640
|
-
_BitScanForward64( &r, (U64)val );
|
2641
|
-
return (int)(r>>3);
|
2642
|
-
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
|
2643
|
-
return (__builtin_ctzll((U64)val) >> 3);
|
2644
|
-
# else
|
2645
|
-
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
|
2646
|
-
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
|
2647
|
-
# endif
|
2648
|
-
}
|
2649
|
-
else /* 32 bits */
|
2650
|
-
{
|
2651
|
-
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
|
2652
|
-
unsigned long r;
|
2653
|
-
_BitScanForward( &r, (U32)val );
|
2654
|
-
return (int)(r>>3);
|
2655
|
-
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
|
2656
|
-
return (__builtin_ctz((U32)val) >> 3);
|
2657
|
-
# else
|
2658
|
-
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
|
2659
|
-
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
|
2660
|
-
# endif
|
2661
|
-
}
|
2662
|
-
}
|
2663
|
-
else /* Big Endian CPU */
|
2664
|
-
{
|
2665
|
-
if (MEM_32bits())
|
2666
|
-
{
|
2667
|
-
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
|
2668
|
-
unsigned long r = 0;
|
2669
|
-
_BitScanReverse64( &r, val );
|
2670
|
-
return (unsigned)(r>>3);
|
2671
|
-
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
|
2672
|
-
return (__builtin_clzll(val) >> 3);
|
2673
|
-
# else
|
2674
|
-
unsigned r;
|
2675
|
-
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
|
2676
|
-
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
|
2677
|
-
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
|
2678
|
-
r += (!val);
|
2679
|
-
return r;
|
2680
|
-
# endif
|
2681
|
-
}
|
2682
|
-
else /* 32 bits */
|
2683
|
-
{
|
2684
|
-
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
|
2685
|
-
unsigned long r = 0;
|
2686
|
-
_BitScanReverse( &r, (unsigned long)val );
|
2687
|
-
return (unsigned)(r>>3);
|
2688
|
-
# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
|
2689
|
-
return (__builtin_clz((U32)val) >> 3);
|
2690
|
-
# else
|
2691
|
-
unsigned r;
|
2692
|
-
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
|
2693
|
-
r += (!val);
|
2694
|
-
return r;
|
2695
|
-
# endif
|
2696
|
-
}
|
2697
|
-
}
|
2698
|
-
}
|
2699
|
-
|
2700
|
-
|
2701
|
-
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
|
2702
|
-
{
|
2703
|
-
const BYTE* const pStart = pIn;
|
2704
|
-
|
2705
|
-
while ((pIn<pInLimit-(sizeof(size_t)-1)))
|
2706
|
-
{
|
2707
|
-
size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
|
2708
|
-
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
|
2709
|
-
pIn += ZSTD_NbCommonBytes(diff);
|
2710
|
-
return (size_t)(pIn - pStart);
|
2711
|
-
}
|
2712
|
-
|
2713
|
-
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
|
2714
|
-
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
|
2715
|
-
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
|
2716
|
-
return (size_t)(pIn - pStart);
|
2717
|
-
}
|
2718
|
-
|
2719
2438
|
|
2720
2439
|
/* *************************************************************
|
2721
2440
|
* Decompression section
|
@@ -2731,7 +2450,6 @@ struct ZSTD_DCtx_s
|
|
2731
2450
|
blockType_t bType;
|
2732
2451
|
U32 phase;
|
2733
2452
|
const BYTE* litPtr;
|
2734
|
-
size_t litBufSize;
|
2735
2453
|
size_t litSize;
|
2736
2454
|
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
2737
2455
|
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
|
@@ -2759,7 +2477,9 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
|
|
2759
2477
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
2760
2478
|
{
|
2761
2479
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
2762
|
-
|
2480
|
+
if (srcSize > 0) {
|
2481
|
+
memcpy(dst, src, srcSize);
|
2482
|
+
}
|
2763
2483
|
return srcSize;
|
2764
2484
|
}
|
2765
2485
|
|
@@ -2803,8 +2523,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2803
2523
|
size_t litSize = BLOCKSIZE;
|
2804
2524
|
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
2805
2525
|
dctx->litPtr = dctx->litBuffer;
|
2806
|
-
dctx->litBufSize = BLOCKSIZE;
|
2807
2526
|
dctx->litSize = litSize;
|
2527
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2808
2528
|
return readSize; /* works if it's an error too */
|
2809
2529
|
}
|
2810
2530
|
case IS_RAW:
|
@@ -2812,16 +2532,16 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2812
2532
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
2813
2533
|
if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
|
2814
2534
|
{
|
2815
|
-
|
2816
|
-
|
2817
|
-
|
2818
|
-
|
2819
|
-
|
2820
|
-
|
2821
|
-
|
2822
|
-
|
2535
|
+
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
2536
|
+
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
2537
|
+
memcpy(dctx->litBuffer, istart, litSize);
|
2538
|
+
dctx->litPtr = dctx->litBuffer;
|
2539
|
+
dctx->litSize = litSize;
|
2540
|
+
memset(dctx->litBuffer + dctx->litSize, 0, 8);
|
2541
|
+
return litSize+3;
|
2542
|
+
}
|
2543
|
+
/* direct reference into compressed stream */
|
2823
2544
|
dctx->litPtr = istart+3;
|
2824
|
-
dctx->litBufSize = srcSize-3;
|
2825
2545
|
dctx->litSize = litSize;
|
2826
2546
|
return litSize+3;
|
2827
2547
|
}
|
@@ -2829,9 +2549,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
|
2829
2549
|
{
|
2830
2550
|
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
2831
2551
|
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
2832
|
-
memset(dctx->litBuffer, istart[3], litSize);
|
2552
|
+
memset(dctx->litBuffer, istart[3], litSize + 8);
|
2833
2553
|
dctx->litPtr = dctx->litBuffer;
|
2834
|
-
dctx->litBufSize = BLOCKSIZE;
|
2835
2554
|
dctx->litSize = litSize;
|
2836
2555
|
return 4;
|
2837
2556
|
}
|
@@ -2885,7 +2604,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
|
|
2885
2604
|
/* Build DTables */
|
2886
2605
|
switch(LLtype)
|
2887
2606
|
{
|
2888
|
-
U32 max;
|
2889
2607
|
case bt_rle :
|
2890
2608
|
LLlog = 0;
|
2891
2609
|
FSE_buildDTable_rle(DTableLL, *ip++); break;
|
@@ -2893,17 +2611,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
|
|
2893
2611
|
LLlog = LLbits;
|
2894
2612
|
FSE_buildDTable_raw(DTableLL, LLbits); break;
|
2895
2613
|
default :
|
2896
|
-
max = MaxLL;
|
2897
|
-
|
2898
|
-
|
2899
|
-
|
2900
|
-
|
2901
|
-
|
2902
|
-
}
|
2614
|
+
{ U32 max = MaxLL;
|
2615
|
+
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
|
2616
|
+
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
2617
|
+
if (LLlog > LLFSELog) return ERROR(corruption_detected);
|
2618
|
+
ip += headerSize;
|
2619
|
+
FSE_buildDTable(DTableLL, norm, max, LLlog);
|
2620
|
+
} }
|
2903
2621
|
|
2904
2622
|
switch(Offtype)
|
2905
2623
|
{
|
2906
|
-
U32 max;
|
2907
2624
|
case bt_rle :
|
2908
2625
|
Offlog = 0;
|
2909
2626
|
if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
|
@@ -2913,17 +2630,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
|
|
2913
2630
|
Offlog = Offbits;
|
2914
2631
|
FSE_buildDTable_raw(DTableOffb, Offbits); break;
|
2915
2632
|
default :
|
2916
|
-
max = MaxOff;
|
2917
|
-
|
2918
|
-
|
2919
|
-
|
2920
|
-
|
2921
|
-
|
2922
|
-
}
|
2633
|
+
{ U32 max = MaxOff;
|
2634
|
+
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
|
2635
|
+
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
2636
|
+
if (Offlog > OffFSELog) return ERROR(corruption_detected);
|
2637
|
+
ip += headerSize;
|
2638
|
+
FSE_buildDTable(DTableOffb, norm, max, Offlog);
|
2639
|
+
} }
|
2923
2640
|
|
2924
2641
|
switch(MLtype)
|
2925
2642
|
{
|
2926
|
-
U32 max;
|
2927
2643
|
case bt_rle :
|
2928
2644
|
MLlog = 0;
|
2929
2645
|
if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
|
@@ -2932,14 +2648,13 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
|
|
2932
2648
|
MLlog = MLbits;
|
2933
2649
|
FSE_buildDTable_raw(DTableML, MLbits); break;
|
2934
2650
|
default :
|
2935
|
-
max = MaxML;
|
2936
|
-
|
2937
|
-
|
2938
|
-
|
2939
|
-
|
2940
|
-
|
2941
|
-
|
2942
|
-
}
|
2651
|
+
{ U32 max = MaxML;
|
2652
|
+
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
|
2653
|
+
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
2654
|
+
if (MLlog > MLFSELog) return ERROR(corruption_detected);
|
2655
|
+
ip += headerSize;
|
2656
|
+
FSE_buildDTable(DTableML, norm, max, MLlog);
|
2657
|
+
} } }
|
2943
2658
|
|
2944
2659
|
return ip-istart;
|
2945
2660
|
}
|
@@ -2977,11 +2692,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
2977
2692
|
seqState->prevOffset = seq->offset;
|
2978
2693
|
if (litLength == MaxLL)
|
2979
2694
|
{
|
2980
|
-
U32 add = *dumps
|
2695
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
2981
2696
|
if (add < 255) litLength += add;
|
2982
|
-
else
|
2697
|
+
else if (dumps + 3 <= de)
|
2983
2698
|
{
|
2984
|
-
litLength =
|
2699
|
+
litLength = MEM_readLE24(dumps);
|
2985
2700
|
dumps += 3;
|
2986
2701
|
}
|
2987
2702
|
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
@@ -3007,11 +2722,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
3007
2722
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
3008
2723
|
if (matchLength == MaxML)
|
3009
2724
|
{
|
3010
|
-
U32 add = *dumps
|
2725
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
3011
2726
|
if (add < 255) matchLength += add;
|
3012
|
-
else
|
2727
|
+
else if (dumps + 3 <= de)
|
3013
2728
|
{
|
3014
|
-
matchLength =
|
2729
|
+
matchLength = MEM_readLE24(dumps);
|
3015
2730
|
dumps += 3;
|
3016
2731
|
}
|
3017
2732
|
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
@@ -3032,7 +2747,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3032
2747
|
BYTE* const base, BYTE* const oend)
|
3033
2748
|
{
|
3034
2749
|
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
3035
|
-
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /*
|
2750
|
+
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
|
3036
2751
|
const BYTE* const ostart = op;
|
3037
2752
|
BYTE* const oLitEnd = op + sequence.litLength;
|
3038
2753
|
BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
|
@@ -3042,7 +2757,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3042
2757
|
/* checks */
|
3043
2758
|
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
3044
2759
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
3045
|
-
if (litEnd > litLimit
|
2760
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
3046
2761
|
|
3047
2762
|
/* copy Literals */
|
3048
2763
|
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
@@ -3076,7 +2791,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3076
2791
|
}
|
3077
2792
|
op += 8; match += 8;
|
3078
2793
|
|
3079
|
-
if (oMatchEnd > oend-
|
2794
|
+
if (oMatchEnd > oend-(16-MINMATCH))
|
3080
2795
|
{
|
3081
2796
|
if (op < oend_8)
|
3082
2797
|
{
|
@@ -3088,7 +2803,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3088
2803
|
}
|
3089
2804
|
else
|
3090
2805
|
{
|
3091
|
-
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
2806
|
+
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
|
3092
2807
|
}
|
3093
2808
|
}
|
3094
2809
|
|
@@ -3108,7 +2823,6 @@ static size_t ZSTD_decompressSequences(
|
|
3108
2823
|
BYTE* const oend = ostart + maxDstSize;
|
3109
2824
|
size_t errorCode, dumpsLength;
|
3110
2825
|
const BYTE* litPtr = dctx->litPtr;
|
3111
|
-
const BYTE* const litMax = litPtr + dctx->litBufSize;
|
3112
2826
|
const BYTE* const litEnd = litPtr + dctx->litSize;
|
3113
2827
|
int nbSeq;
|
3114
2828
|
const BYTE* dumps;
|
@@ -3144,7 +2858,7 @@ static size_t ZSTD_decompressSequences(
|
|
3144
2858
|
size_t oneSeqSize;
|
3145
2859
|
nbSeq--;
|
3146
2860
|
ZSTD_decodeSequence(&sequence, &seqState);
|
3147
|
-
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr,
|
2861
|
+
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
3148
2862
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
3149
2863
|
op += oneSeqSize;
|
3150
2864
|
}
|
@@ -3158,8 +2872,10 @@ static size_t ZSTD_decompressSequences(
|
|
3158
2872
|
size_t lastLLSize = litEnd - litPtr;
|
3159
2873
|
if (litPtr > litEnd) return ERROR(corruption_detected);
|
3160
2874
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
3161
|
-
if (
|
3162
|
-
|
2875
|
+
if (lastLLSize > 0) {
|
2876
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
2877
|
+
op += lastLLSize;
|
2878
|
+
}
|
3163
2879
|
}
|
3164
2880
|
}
|
3165
2881
|
|
@@ -3249,6 +2965,61 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|
3249
2965
|
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
3250
2966
|
}
|
3251
2967
|
|
2968
|
+
/* ZSTD_errorFrameSizeInfoLegacy() :
|
2969
|
+
assumes `cSize` and `dBound` are _not_ NULL */
|
2970
|
+
MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
|
2971
|
+
{
|
2972
|
+
*cSize = ret;
|
2973
|
+
*dBound = ZSTD_CONTENTSIZE_ERROR;
|
2974
|
+
}
|
2975
|
+
|
2976
|
+
void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
|
2977
|
+
{
|
2978
|
+
const BYTE* ip = (const BYTE*)src;
|
2979
|
+
size_t remainingSize = srcSize;
|
2980
|
+
size_t nbBlocks = 0;
|
2981
|
+
U32 magicNumber;
|
2982
|
+
blockProperties_t blockProperties;
|
2983
|
+
|
2984
|
+
/* Frame Header */
|
2985
|
+
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
|
2986
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
2987
|
+
return;
|
2988
|
+
}
|
2989
|
+
magicNumber = MEM_readLE32(src);
|
2990
|
+
if (magicNumber != ZSTD_magicNumber) {
|
2991
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
|
2992
|
+
return;
|
2993
|
+
}
|
2994
|
+
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
2995
|
+
|
2996
|
+
/* Loop on each block */
|
2997
|
+
while (1)
|
2998
|
+
{
|
2999
|
+
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
3000
|
+
if (ZSTD_isError(cBlockSize)) {
|
3001
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
|
3002
|
+
return;
|
3003
|
+
}
|
3004
|
+
|
3005
|
+
ip += ZSTD_blockHeaderSize;
|
3006
|
+
remainingSize -= ZSTD_blockHeaderSize;
|
3007
|
+
if (cBlockSize > remainingSize) {
|
3008
|
+
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
|
3009
|
+
return;
|
3010
|
+
}
|
3011
|
+
|
3012
|
+
if (cBlockSize == 0) break; /* bt_end */
|
3013
|
+
|
3014
|
+
ip += cBlockSize;
|
3015
|
+
remainingSize -= cBlockSize;
|
3016
|
+
nbBlocks++;
|
3017
|
+
}
|
3018
|
+
|
3019
|
+
*cSize = ip - (const BYTE*)src;
|
3020
|
+
*dBound = nbBlocks * BLOCKSIZE;
|
3021
|
+
}
|
3022
|
+
|
3252
3023
|
|
3253
3024
|
/*******************************
|
3254
3025
|
* Streaming Decompression API
|
@@ -3354,36 +3125,36 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
|
|
3354
3125
|
|
3355
3126
|
unsigned ZSTDv03_isError(size_t code)
|
3356
3127
|
{
|
3357
|
-
|
3128
|
+
return ZSTD_isError(code);
|
3358
3129
|
}
|
3359
3130
|
|
3360
3131
|
size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
3361
3132
|
const void* src, size_t compressedSize)
|
3362
3133
|
{
|
3363
|
-
|
3134
|
+
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
3364
3135
|
}
|
3365
3136
|
|
3366
3137
|
ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
|
3367
3138
|
{
|
3368
|
-
|
3139
|
+
return (ZSTDv03_Dctx*)ZSTD_createDCtx();
|
3369
3140
|
}
|
3370
3141
|
|
3371
3142
|
size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx)
|
3372
3143
|
{
|
3373
|
-
|
3144
|
+
return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
|
3374
3145
|
}
|
3375
3146
|
|
3376
3147
|
size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx)
|
3377
3148
|
{
|
3378
|
-
|
3149
|
+
return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
|
3379
3150
|
}
|
3380
3151
|
|
3381
3152
|
size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx)
|
3382
3153
|
{
|
3383
|
-
|
3154
|
+
return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
|
3384
3155
|
}
|
3385
3156
|
|
3386
3157
|
size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
3387
3158
|
{
|
3388
|
-
|
3159
|
+
return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
|
3389
3160
|
}
|