extzstd 0.3.1 → 0.3.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +28 -14
- data/contrib/zstd/CHANGELOG +301 -56
- data/contrib/zstd/CONTRIBUTING.md +169 -72
- data/contrib/zstd/LICENSE +4 -4
- data/contrib/zstd/Makefile +116 -87
- data/contrib/zstd/Package.swift +36 -0
- data/contrib/zstd/README.md +62 -32
- data/contrib/zstd/TESTING.md +2 -3
- data/contrib/zstd/appveyor.yml +52 -136
- data/contrib/zstd/lib/BUCK +5 -7
- data/contrib/zstd/lib/Makefile +225 -222
- data/contrib/zstd/lib/README.md +51 -6
- data/contrib/zstd/lib/common/allocations.h +55 -0
- data/contrib/zstd/lib/common/bits.h +200 -0
- data/contrib/zstd/lib/common/bitstream.h +45 -62
- data/contrib/zstd/lib/common/compiler.h +205 -22
- data/contrib/zstd/lib/common/cpu.h +1 -3
- data/contrib/zstd/lib/common/debug.c +1 -1
- data/contrib/zstd/lib/common/debug.h +12 -19
- data/contrib/zstd/lib/common/entropy_common.c +172 -48
- data/contrib/zstd/lib/common/error_private.c +10 -2
- data/contrib/zstd/lib/common/error_private.h +82 -3
- data/contrib/zstd/lib/common/fse.h +37 -86
- data/contrib/zstd/lib/common/fse_decompress.c +117 -92
- data/contrib/zstd/lib/common/huf.h +99 -166
- data/contrib/zstd/lib/common/mem.h +124 -142
- data/contrib/zstd/lib/common/pool.c +54 -27
- data/contrib/zstd/lib/common/pool.h +10 -4
- data/contrib/zstd/lib/common/portability_macros.h +156 -0
- data/contrib/zstd/lib/common/threading.c +74 -19
- data/contrib/zstd/lib/common/threading.h +5 -10
- data/contrib/zstd/lib/common/xxhash.c +7 -847
- data/contrib/zstd/lib/common/xxhash.h +5568 -167
- data/contrib/zstd/lib/common/zstd_common.c +2 -37
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_internal.h +132 -187
- data/contrib/zstd/lib/common/zstd_trace.h +163 -0
- data/contrib/zstd/lib/compress/clevels.h +134 -0
- data/contrib/zstd/lib/compress/fse_compress.c +83 -157
- data/contrib/zstd/lib/compress/hist.c +27 -29
- data/contrib/zstd/lib/compress/hist.h +2 -2
- data/contrib/zstd/lib/compress/huf_compress.c +916 -279
- data/contrib/zstd/lib/compress/zstd_compress.c +3773 -1019
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +610 -203
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +119 -42
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +16 -6
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +42 -19
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +1 -1
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +49 -317
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +1 -1
- data/contrib/zstd/lib/compress/zstd_cwksp.h +320 -103
- data/contrib/zstd/lib/compress/zstd_double_fast.c +388 -151
- data/contrib/zstd/lib/compress/zstd_double_fast.h +3 -2
- data/contrib/zstd/lib/compress/zstd_fast.c +729 -265
- data/contrib/zstd/lib/compress/zstd_fast.h +3 -2
- data/contrib/zstd/lib/compress/zstd_lazy.c +1270 -251
- data/contrib/zstd/lib/compress/zstd_lazy.h +61 -1
- data/contrib/zstd/lib/compress/zstd_ldm.c +324 -219
- data/contrib/zstd/lib/compress/zstd_ldm.h +9 -2
- data/contrib/zstd/lib/compress/zstd_ldm_geartab.h +106 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +481 -209
- data/contrib/zstd/lib/compress/zstd_opt.h +1 -1
- data/contrib/zstd/lib/compress/zstdmt_compress.c +181 -457
- data/contrib/zstd/lib/compress/zstdmt_compress.h +34 -113
- data/contrib/zstd/lib/decompress/huf_decompress.c +1199 -565
- data/contrib/zstd/lib/decompress/huf_decompress_amd64.S +576 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +12 -12
- data/contrib/zstd/lib/decompress/zstd_ddict.h +2 -2
- data/contrib/zstd/lib/decompress/zstd_decompress.c +627 -157
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1086 -326
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +19 -5
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +62 -13
- data/contrib/zstd/lib/deprecated/zbuff.h +1 -1
- data/contrib/zstd/lib/deprecated/zbuff_common.c +1 -1
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +24 -4
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +3 -1
- data/contrib/zstd/lib/dictBuilder/cover.c +73 -52
- data/contrib/zstd/lib/dictBuilder/cover.h +7 -6
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +1 -1
- data/contrib/zstd/lib/dictBuilder/fastcover.c +44 -35
- data/contrib/zstd/lib/dictBuilder/zdict.c +103 -111
- data/contrib/zstd/lib/legacy/zstd_legacy.h +8 -1
- data/contrib/zstd/lib/legacy/zstd_v01.c +21 -54
- data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v02.c +29 -70
- data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v03.c +30 -73
- data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v04.c +29 -71
- data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v05.c +40 -86
- data/contrib/zstd/lib/legacy/zstd_v05.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v06.c +47 -88
- data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v07.c +40 -83
- data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
- data/contrib/zstd/lib/libzstd.mk +214 -0
- data/contrib/zstd/lib/libzstd.pc.in +7 -6
- data/contrib/zstd/lib/module.modulemap +35 -0
- data/contrib/zstd/lib/{dictBuilder/zdict.h → zdict.h} +203 -34
- data/contrib/zstd/lib/zstd.h +1217 -287
- data/contrib/zstd/lib/{common/zstd_errors.h → zstd_errors.h} +28 -8
- data/ext/extconf.rb +7 -6
- data/ext/extzstd.c +19 -10
- data/ext/extzstd.h +6 -0
- data/ext/libzstd_conf.h +0 -1
- data/ext/zstd_decompress_asm.S +1 -0
- data/gemstub.rb +3 -21
- data/lib/extzstd/version.rb +6 -1
- data/lib/extzstd.rb +0 -2
- data/test/test_basic.rb +0 -5
- metadata +18 -6
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -28,7 +28,7 @@
|
|
28
28
|
low-level memory access routines
|
29
29
|
Copyright (C) 2013-2015, Yann Collet.
|
30
30
|
|
31
|
-
BSD 2-Clause License (
|
31
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
32
32
|
|
33
33
|
Redistribution and use in source and binary forms, with or without
|
34
34
|
modification, are permitted provided that the following conditions are
|
@@ -89,7 +89,11 @@ extern "C" {
|
|
89
89
|
* Basic Types
|
90
90
|
*****************************************************************/
|
91
91
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
92
|
-
#
|
92
|
+
# if defined(_AIX)
|
93
|
+
# include <inttypes.h>
|
94
|
+
# else
|
95
|
+
# include <stdint.h> /* intptr_t */
|
96
|
+
# endif
|
93
97
|
typedef uint8_t BYTE;
|
94
98
|
typedef uint16_t U16;
|
95
99
|
typedef int16_t S16;
|
@@ -111,27 +115,6 @@ extern "C" {
|
|
111
115
|
/****************************************************************
|
112
116
|
* Memory I/O
|
113
117
|
*****************************************************************/
|
114
|
-
/* MEM_FORCE_MEMORY_ACCESS
|
115
|
-
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
116
|
-
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
117
|
-
* The below switch allow to select different access method for improved performance.
|
118
|
-
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
119
|
-
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
120
|
-
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
121
|
-
* Method 2 : direct access. This method is portable but violate C standard.
|
122
|
-
* It can generate buggy code on targets generating assembly depending on alignment.
|
123
|
-
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
124
|
-
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
125
|
-
* Prefer these methods in priority order (0 > 1 > 2)
|
126
|
-
*/
|
127
|
-
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
128
|
-
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
129
|
-
# define MEM_FORCE_MEMORY_ACCESS 2
|
130
|
-
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
131
|
-
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
132
|
-
# define MEM_FORCE_MEMORY_ACCESS 1
|
133
|
-
# endif
|
134
|
-
#endif
|
135
118
|
|
136
119
|
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
|
137
120
|
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
|
@@ -142,33 +125,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
|
|
142
125
|
return one.c[0];
|
143
126
|
}
|
144
127
|
|
145
|
-
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
146
|
-
|
147
|
-
/* violates C standard on structure alignment.
|
148
|
-
Only use if no other choice to achieve best performance on target platform */
|
149
|
-
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
150
|
-
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
151
|
-
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
152
|
-
|
153
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
154
|
-
|
155
|
-
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
156
|
-
|
157
|
-
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
158
|
-
/* currently only defined for gcc and icc */
|
159
|
-
typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
|
160
|
-
|
161
|
-
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
162
|
-
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
163
|
-
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
164
|
-
|
165
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
166
|
-
|
167
|
-
#else
|
168
|
-
|
169
|
-
/* default method, safe and standard.
|
170
|
-
can sometimes prove slower */
|
171
|
-
|
172
128
|
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
173
129
|
{
|
174
130
|
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
@@ -189,9 +145,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
|
189
145
|
memcpy(memPtr, &value, sizeof(value));
|
190
146
|
}
|
191
147
|
|
192
|
-
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
193
|
-
|
194
|
-
|
195
148
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
196
149
|
{
|
197
150
|
if (MEM_isLittleEndian())
|
@@ -268,7 +221,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
|
268
221
|
header file (to include)
|
269
222
|
Copyright (C) 2013-2015, Yann Collet.
|
270
223
|
|
271
|
-
BSD 2-Clause License (
|
224
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
272
225
|
|
273
226
|
Redistribution and use in source and binary forms, with or without
|
274
227
|
modification, are permitted provided that the following conditions are
|
@@ -349,9 +302,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
|
349
302
|
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
350
303
|
{
|
351
304
|
# if defined(_MSC_VER) /* Visual */
|
352
|
-
unsigned long r
|
353
|
-
_BitScanReverse
|
354
|
-
return (unsigned) r;
|
305
|
+
unsigned long r;
|
306
|
+
return _BitScanReverse(&r, val) ? (unsigned)r : 0;
|
355
307
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
356
308
|
return __builtin_clz (val) ^ 31;
|
357
309
|
# else /* Software version */
|
@@ -433,7 +385,7 @@ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
433
385
|
}
|
434
386
|
|
435
387
|
/*! BIT_lookBitsFast :
|
436
|
-
* unsafe version; only works
|
388
|
+
* unsafe version; only works if nbBits >= 1 */
|
437
389
|
MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
438
390
|
{
|
439
391
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
@@ -453,7 +405,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
453
405
|
}
|
454
406
|
|
455
407
|
/*!BIT_readBitsFast :
|
456
|
-
* unsafe version; only works
|
408
|
+
* unsafe version; only works if nbBits >= 1 */
|
457
409
|
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
458
410
|
{
|
459
411
|
size_t value = BIT_lookBitsFast(bitD, nbBits);
|
@@ -510,7 +462,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
|
510
462
|
Error codes and messages
|
511
463
|
Copyright (C) 2013-2015, Yann Collet
|
512
464
|
|
513
|
-
BSD 2-Clause License (
|
465
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
514
466
|
|
515
467
|
Redistribution and use in source and binary forms, with or without
|
516
468
|
modification, are permitted provided that the following conditions are
|
@@ -609,7 +561,7 @@ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be mor
|
|
609
561
|
header file for static linking (only)
|
610
562
|
Copyright (C) 2013-2015, Yann Collet
|
611
563
|
|
612
|
-
BSD 2-Clause License (
|
564
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
613
565
|
|
614
566
|
Redistribution and use in source and binary forms, with or without
|
615
567
|
modification, are permitted provided that the following conditions are
|
@@ -753,7 +705,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
|
753
705
|
header file for static linking (only)
|
754
706
|
Copyright (C) 2013-2015, Yann Collet
|
755
707
|
|
756
|
-
BSD 2-Clause License (
|
708
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
757
709
|
|
758
710
|
Redistribution and use in source and binary forms, with or without
|
759
711
|
modification, are permitted provided that the following conditions are
|
@@ -822,7 +774,7 @@ static size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, si
|
|
822
774
|
Header File
|
823
775
|
Copyright (C) 2014-2015, Yann Collet.
|
824
776
|
|
825
|
-
BSD 2-Clause License (
|
777
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
826
778
|
|
827
779
|
Redistribution and use in source and binary forms, with or without
|
828
780
|
modification, are permitted provided that the following conditions are
|
@@ -882,7 +834,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */
|
|
882
834
|
Header File for static linking only
|
883
835
|
Copyright (C) 2014-2015, Yann Collet.
|
884
836
|
|
885
|
-
BSD 2-Clause License (
|
837
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
886
838
|
|
887
839
|
Redistribution and use in source and binary forms, with or without
|
888
840
|
modification, are permitted provided that the following conditions are
|
@@ -946,7 +898,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
|
946
898
|
FSE : Finite State Entropy coder
|
947
899
|
Copyright (C) 2013-2015, Yann Collet.
|
948
900
|
|
949
|
-
BSD 2-Clause License (
|
901
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
950
902
|
|
951
903
|
Redistribution and use in source and binary forms, with or without
|
952
904
|
modification, are permitted provided that the following conditions are
|
@@ -1450,7 +1402,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
|
|
1450
1402
|
Huff0 : Huffman coder, part of New Generation Entropy library
|
1451
1403
|
Copyright (C) 2013-2015, Yann Collet.
|
1452
1404
|
|
1453
|
-
BSD 2-Clause License (
|
1405
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
1454
1406
|
|
1455
1407
|
Redistribution and use in source and binary forms, with or without
|
1456
1408
|
modification, are permitted provided that the following conditions are
|
@@ -2609,7 +2561,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
2609
2561
|
zstd - standard compression library
|
2610
2562
|
Copyright (C) 2014-2015, Yann Collet.
|
2611
2563
|
|
2612
|
-
BSD 2-Clause License (
|
2564
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
2613
2565
|
|
2614
2566
|
Redistribution and use in source and binary forms, with or without
|
2615
2567
|
modification, are permitted provided that the following conditions are
|
@@ -3114,12 +3066,19 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
3114
3066
|
const BYTE* const litEnd = *litPtr + sequence.litLength;
|
3115
3067
|
|
3116
3068
|
/* checks */
|
3117
|
-
|
3069
|
+
size_t const seqLength = sequence.litLength + sequence.matchLength;
|
3070
|
+
|
3071
|
+
if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
|
3072
|
+
if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
|
3073
|
+
/* Now we know there are no overflow in literal nor match lengths, can use the pointer check */
|
3074
|
+
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
|
3075
|
+
if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
|
3076
|
+
|
3118
3077
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
3119
3078
|
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
3120
3079
|
|
3121
3080
|
/* copy Literals */
|
3122
|
-
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
3081
|
+
ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
3123
3082
|
op = oLitEnd;
|
3124
3083
|
*litPtr = litEnd; /* update for next sequence */
|
3125
3084
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -29,7 +29,7 @@
|
|
29
29
|
low-level memory access routines
|
30
30
|
Copyright (C) 2013-2015, Yann Collet.
|
31
31
|
|
32
|
-
BSD 2-Clause License (
|
32
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
33
33
|
|
34
34
|
Redistribution and use in source and binary forms, with or without
|
35
35
|
modification, are permitted provided that the following conditions are
|
@@ -90,7 +90,11 @@ extern "C" {
|
|
90
90
|
* Basic Types
|
91
91
|
*****************************************************************/
|
92
92
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
93
|
-
#
|
93
|
+
# if defined(_AIX)
|
94
|
+
# include <inttypes.h>
|
95
|
+
# else
|
96
|
+
# include <stdint.h> /* intptr_t */
|
97
|
+
# endif
|
94
98
|
typedef uint8_t BYTE;
|
95
99
|
typedef uint16_t U16;
|
96
100
|
typedef int16_t S16;
|
@@ -112,27 +116,6 @@ extern "C" {
|
|
112
116
|
/****************************************************************
|
113
117
|
* Memory I/O
|
114
118
|
*****************************************************************/
|
115
|
-
/* MEM_FORCE_MEMORY_ACCESS
|
116
|
-
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
117
|
-
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
118
|
-
* The below switch allow to select different access method for improved performance.
|
119
|
-
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
120
|
-
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
121
|
-
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
122
|
-
* Method 2 : direct access. This method is portable but violate C standard.
|
123
|
-
* It can generate buggy code on targets generating assembly depending on alignment.
|
124
|
-
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
125
|
-
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
126
|
-
* Prefer these methods in priority order (0 > 1 > 2)
|
127
|
-
*/
|
128
|
-
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
129
|
-
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
130
|
-
# define MEM_FORCE_MEMORY_ACCESS 2
|
131
|
-
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
132
|
-
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
133
|
-
# define MEM_FORCE_MEMORY_ACCESS 1
|
134
|
-
# endif
|
135
|
-
#endif
|
136
119
|
|
137
120
|
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
|
138
121
|
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
|
@@ -143,33 +126,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
|
|
143
126
|
return one.c[0];
|
144
127
|
}
|
145
128
|
|
146
|
-
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
147
|
-
|
148
|
-
/* violates C standard on structure alignment.
|
149
|
-
Only use if no other choice to achieve best performance on target platform */
|
150
|
-
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
151
|
-
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
152
|
-
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
153
|
-
|
154
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
155
|
-
|
156
|
-
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
157
|
-
|
158
|
-
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
159
|
-
/* currently only defined for gcc and icc */
|
160
|
-
typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
|
161
|
-
|
162
|
-
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
163
|
-
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
164
|
-
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
165
|
-
|
166
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
167
|
-
|
168
|
-
#else
|
169
|
-
|
170
|
-
/* default method, safe and standard.
|
171
|
-
can sometimes prove slower */
|
172
|
-
|
173
129
|
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
174
130
|
{
|
175
131
|
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
@@ -190,10 +146,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
|
190
146
|
memcpy(memPtr, &value, sizeof(value));
|
191
147
|
}
|
192
148
|
|
193
|
-
|
194
|
-
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
195
|
-
|
196
|
-
|
197
149
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
198
150
|
{
|
199
151
|
if (MEM_isLittleEndian())
|
@@ -270,7 +222,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
|
270
222
|
header file (to include)
|
271
223
|
Copyright (C) 2013-2015, Yann Collet.
|
272
224
|
|
273
|
-
BSD 2-Clause License (
|
225
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
274
226
|
|
275
227
|
Redistribution and use in source and binary forms, with or without
|
276
228
|
modification, are permitted provided that the following conditions are
|
@@ -352,9 +304,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
|
352
304
|
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
353
305
|
{
|
354
306
|
# if defined(_MSC_VER) /* Visual */
|
355
|
-
unsigned long r
|
356
|
-
_BitScanReverse
|
357
|
-
return (unsigned) r;
|
307
|
+
unsigned long r;
|
308
|
+
return _BitScanReverse(&r, val) ? (unsigned)r : 0;
|
358
309
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
359
310
|
return __builtin_clz (val) ^ 31;
|
360
311
|
# else /* Software version */
|
@@ -435,7 +386,7 @@ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
435
386
|
}
|
436
387
|
|
437
388
|
/*! BIT_lookBitsFast :
|
438
|
-
* unsafe version; only works
|
389
|
+
* unsafe version; only works if nbBits >= 1 */
|
439
390
|
MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
440
391
|
{
|
441
392
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
@@ -455,7 +406,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
455
406
|
}
|
456
407
|
|
457
408
|
/*!BIT_readBitsFast :
|
458
|
-
* unsafe version; only works
|
409
|
+
* unsafe version; only works if nbBits >= 1 */
|
459
410
|
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
460
411
|
{
|
461
412
|
size_t value = BIT_lookBitsFast(bitD, nbBits);
|
@@ -512,7 +463,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
|
512
463
|
Error codes and messages
|
513
464
|
Copyright (C) 2013-2015, Yann Collet
|
514
465
|
|
515
|
-
BSD 2-Clause License (
|
466
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
516
467
|
|
517
468
|
Redistribution and use in source and binary forms, with or without
|
518
469
|
modification, are permitted provided that the following conditions are
|
@@ -611,7 +562,7 @@ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be mor
|
|
611
562
|
header file for static linking (only)
|
612
563
|
Copyright (C) 2013-2015, Yann Collet
|
613
564
|
|
614
|
-
BSD 2-Clause License (
|
565
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
615
566
|
|
616
567
|
Redistribution and use in source and binary forms, with or without
|
617
568
|
modification, are permitted provided that the following conditions are
|
@@ -755,7 +706,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
|
755
706
|
header file for static linking (only)
|
756
707
|
Copyright (C) 2013-2015, Yann Collet
|
757
708
|
|
758
|
-
BSD 2-Clause License (
|
709
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
759
710
|
|
760
711
|
Redistribution and use in source and binary forms, with or without
|
761
712
|
modification, are permitted provided that the following conditions are
|
@@ -823,7 +774,7 @@ static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, si
|
|
823
774
|
Header File
|
824
775
|
Copyright (C) 2014-2015, Yann Collet.
|
825
776
|
|
826
|
-
BSD 2-Clause License (
|
777
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
827
778
|
|
828
779
|
Redistribution and use in source and binary forms, with or without
|
829
780
|
modification, are permitted provided that the following conditions are
|
@@ -883,7 +834,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */
|
|
883
834
|
Header File for static linking only
|
884
835
|
Copyright (C) 2014-2015, Yann Collet.
|
885
836
|
|
886
|
-
BSD 2-Clause License (
|
837
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
887
838
|
|
888
839
|
Redistribution and use in source and binary forms, with or without
|
889
840
|
modification, are permitted provided that the following conditions are
|
@@ -947,7 +898,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
|
947
898
|
FSE : Finite State Entropy coder
|
948
899
|
Copyright (C) 2013-2015, Yann Collet.
|
949
900
|
|
950
|
-
BSD 2-Clause License (
|
901
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
951
902
|
|
952
903
|
Redistribution and use in source and binary forms, with or without
|
953
904
|
modification, are permitted provided that the following conditions are
|
@@ -1451,7 +1402,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
|
|
1451
1402
|
Huff0 : Huffman coder, part of New Generation Entropy library
|
1452
1403
|
Copyright (C) 2013-2015, Yann Collet.
|
1453
1404
|
|
1454
|
-
BSD 2-Clause License (
|
1405
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
1455
1406
|
|
1456
1407
|
Redistribution and use in source and binary forms, with or without
|
1457
1408
|
modification, are permitted provided that the following conditions are
|
@@ -2248,7 +2199,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
2248
2199
|
zstd - standard compression library
|
2249
2200
|
Copyright (C) 2014-2015, Yann Collet.
|
2250
2201
|
|
2251
|
-
BSD 2-Clause License (
|
2202
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
2252
2203
|
|
2253
2204
|
Redistribution and use in source and binary forms, with or without
|
2254
2205
|
modification, are permitted provided that the following conditions are
|
@@ -2755,18 +2706,24 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
2755
2706
|
const BYTE* const litEnd = *litPtr + sequence.litLength;
|
2756
2707
|
|
2757
2708
|
/* checks */
|
2758
|
-
|
2709
|
+
size_t const seqLength = sequence.litLength + sequence.matchLength;
|
2710
|
+
|
2711
|
+
if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
|
2712
|
+
if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
|
2713
|
+
/* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
|
2714
|
+
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
|
2715
|
+
if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
|
2716
|
+
|
2759
2717
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
2760
2718
|
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
2761
2719
|
|
2762
2720
|
/* copy Literals */
|
2763
|
-
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
2721
|
+
ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
2764
2722
|
op = oLitEnd;
|
2765
2723
|
*litPtr = litEnd; /* update for next sequence */
|
2766
2724
|
|
2767
2725
|
/* copy Match */
|
2768
|
-
{
|
2769
|
-
const BYTE* match = op - sequence.offset;
|
2726
|
+
{ const BYTE* match = op - sequence.offset;
|
2770
2727
|
|
2771
2728
|
/* check */
|
2772
2729
|
if (sequence.offset > (size_t)op) return ERROR(corruption_detected); /* address space overflow test (this test seems kept by clang optimizer) */
|