extzstd 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +28 -14
- data/contrib/zstd/CHANGELOG +301 -56
- data/contrib/zstd/CONTRIBUTING.md +169 -72
- data/contrib/zstd/LICENSE +4 -4
- data/contrib/zstd/Makefile +116 -87
- data/contrib/zstd/Package.swift +36 -0
- data/contrib/zstd/README.md +62 -32
- data/contrib/zstd/TESTING.md +2 -3
- data/contrib/zstd/appveyor.yml +52 -136
- data/contrib/zstd/lib/BUCK +5 -7
- data/contrib/zstd/lib/Makefile +225 -222
- data/contrib/zstd/lib/README.md +51 -6
- data/contrib/zstd/lib/common/allocations.h +55 -0
- data/contrib/zstd/lib/common/bits.h +200 -0
- data/contrib/zstd/lib/common/bitstream.h +45 -62
- data/contrib/zstd/lib/common/compiler.h +205 -22
- data/contrib/zstd/lib/common/cpu.h +1 -3
- data/contrib/zstd/lib/common/debug.c +1 -1
- data/contrib/zstd/lib/common/debug.h +12 -19
- data/contrib/zstd/lib/common/entropy_common.c +172 -48
- data/contrib/zstd/lib/common/error_private.c +10 -2
- data/contrib/zstd/lib/common/error_private.h +82 -3
- data/contrib/zstd/lib/common/fse.h +37 -86
- data/contrib/zstd/lib/common/fse_decompress.c +117 -92
- data/contrib/zstd/lib/common/huf.h +99 -166
- data/contrib/zstd/lib/common/mem.h +124 -142
- data/contrib/zstd/lib/common/pool.c +54 -27
- data/contrib/zstd/lib/common/pool.h +10 -4
- data/contrib/zstd/lib/common/portability_macros.h +156 -0
- data/contrib/zstd/lib/common/threading.c +74 -19
- data/contrib/zstd/lib/common/threading.h +5 -10
- data/contrib/zstd/lib/common/xxhash.c +7 -847
- data/contrib/zstd/lib/common/xxhash.h +5568 -167
- data/contrib/zstd/lib/common/zstd_common.c +2 -37
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_internal.h +132 -187
- data/contrib/zstd/lib/common/zstd_trace.h +163 -0
- data/contrib/zstd/lib/compress/clevels.h +134 -0
- data/contrib/zstd/lib/compress/fse_compress.c +83 -157
- data/contrib/zstd/lib/compress/hist.c +27 -29
- data/contrib/zstd/lib/compress/hist.h +2 -2
- data/contrib/zstd/lib/compress/huf_compress.c +916 -279
- data/contrib/zstd/lib/compress/zstd_compress.c +3773 -1019
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +610 -203
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +119 -42
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +16 -6
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +42 -19
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +1 -1
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +49 -317
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +1 -1
- data/contrib/zstd/lib/compress/zstd_cwksp.h +320 -103
- data/contrib/zstd/lib/compress/zstd_double_fast.c +388 -151
- data/contrib/zstd/lib/compress/zstd_double_fast.h +3 -2
- data/contrib/zstd/lib/compress/zstd_fast.c +729 -265
- data/contrib/zstd/lib/compress/zstd_fast.h +3 -2
- data/contrib/zstd/lib/compress/zstd_lazy.c +1270 -251
- data/contrib/zstd/lib/compress/zstd_lazy.h +61 -1
- data/contrib/zstd/lib/compress/zstd_ldm.c +324 -219
- data/contrib/zstd/lib/compress/zstd_ldm.h +9 -2
- data/contrib/zstd/lib/compress/zstd_ldm_geartab.h +106 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +481 -209
- data/contrib/zstd/lib/compress/zstd_opt.h +1 -1
- data/contrib/zstd/lib/compress/zstdmt_compress.c +181 -457
- data/contrib/zstd/lib/compress/zstdmt_compress.h +34 -113
- data/contrib/zstd/lib/decompress/huf_decompress.c +1199 -565
- data/contrib/zstd/lib/decompress/huf_decompress_amd64.S +576 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +12 -12
- data/contrib/zstd/lib/decompress/zstd_ddict.h +2 -2
- data/contrib/zstd/lib/decompress/zstd_decompress.c +627 -157
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1086 -326
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +19 -5
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +62 -13
- data/contrib/zstd/lib/deprecated/zbuff.h +1 -1
- data/contrib/zstd/lib/deprecated/zbuff_common.c +1 -1
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +24 -4
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +3 -1
- data/contrib/zstd/lib/dictBuilder/cover.c +73 -52
- data/contrib/zstd/lib/dictBuilder/cover.h +7 -6
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +1 -1
- data/contrib/zstd/lib/dictBuilder/fastcover.c +44 -35
- data/contrib/zstd/lib/dictBuilder/zdict.c +103 -111
- data/contrib/zstd/lib/legacy/zstd_legacy.h +8 -1
- data/contrib/zstd/lib/legacy/zstd_v01.c +21 -54
- data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v02.c +29 -70
- data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v03.c +30 -73
- data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v04.c +29 -71
- data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v05.c +40 -86
- data/contrib/zstd/lib/legacy/zstd_v05.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v06.c +47 -88
- data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v07.c +40 -83
- data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
- data/contrib/zstd/lib/libzstd.mk +214 -0
- data/contrib/zstd/lib/libzstd.pc.in +7 -6
- data/contrib/zstd/lib/module.modulemap +35 -0
- data/contrib/zstd/lib/{dictBuilder/zdict.h → zdict.h} +203 -34
- data/contrib/zstd/lib/zstd.h +1217 -287
- data/contrib/zstd/lib/{common/zstd_errors.h → zstd_errors.h} +28 -8
- data/ext/extconf.rb +7 -6
- data/ext/extzstd.c +19 -10
- data/ext/extzstd.h +6 -0
- data/ext/libzstd_conf.h +0 -1
- data/ext/zstd_decompress_asm.S +1 -0
- data/gemstub.rb +3 -21
- data/lib/extzstd/version.rb +6 -1
- data/lib/extzstd.rb +0 -2
- data/test/test_basic.rb +0 -5
- metadata +18 -6
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c)
|
|
2
|
+
* Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
low-level memory access routines
|
|
29
29
|
Copyright (C) 2013-2015, Yann Collet.
|
|
30
30
|
|
|
31
|
-
BSD 2-Clause License (
|
|
31
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
32
32
|
|
|
33
33
|
Redistribution and use in source and binary forms, with or without
|
|
34
34
|
modification, are permitted provided that the following conditions are
|
|
@@ -89,7 +89,11 @@ extern "C" {
|
|
|
89
89
|
* Basic Types
|
|
90
90
|
*****************************************************************/
|
|
91
91
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
|
92
|
-
#
|
|
92
|
+
# if defined(_AIX)
|
|
93
|
+
# include <inttypes.h>
|
|
94
|
+
# else
|
|
95
|
+
# include <stdint.h> /* intptr_t */
|
|
96
|
+
# endif
|
|
93
97
|
typedef uint8_t BYTE;
|
|
94
98
|
typedef uint16_t U16;
|
|
95
99
|
typedef int16_t S16;
|
|
@@ -111,27 +115,6 @@ extern "C" {
|
|
|
111
115
|
/****************************************************************
|
|
112
116
|
* Memory I/O
|
|
113
117
|
*****************************************************************/
|
|
114
|
-
/* MEM_FORCE_MEMORY_ACCESS
|
|
115
|
-
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
|
116
|
-
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
|
117
|
-
* The below switch allow to select different access method for improved performance.
|
|
118
|
-
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
|
119
|
-
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
|
120
|
-
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
|
121
|
-
* Method 2 : direct access. This method is portable but violate C standard.
|
|
122
|
-
* It can generate buggy code on targets generating assembly depending on alignment.
|
|
123
|
-
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
|
124
|
-
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
|
125
|
-
* Prefer these methods in priority order (0 > 1 > 2)
|
|
126
|
-
*/
|
|
127
|
-
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
|
128
|
-
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
|
129
|
-
# define MEM_FORCE_MEMORY_ACCESS 2
|
|
130
|
-
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
|
131
|
-
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
|
132
|
-
# define MEM_FORCE_MEMORY_ACCESS 1
|
|
133
|
-
# endif
|
|
134
|
-
#endif
|
|
135
118
|
|
|
136
119
|
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
|
|
137
120
|
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
|
|
@@ -142,33 +125,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
|
|
|
142
125
|
return one.c[0];
|
|
143
126
|
}
|
|
144
127
|
|
|
145
|
-
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
|
146
|
-
|
|
147
|
-
/* violates C standard on structure alignment.
|
|
148
|
-
Only use if no other choice to achieve best performance on target platform */
|
|
149
|
-
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
|
150
|
-
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
|
151
|
-
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
|
152
|
-
|
|
153
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
|
154
|
-
|
|
155
|
-
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
|
156
|
-
|
|
157
|
-
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
|
158
|
-
/* currently only defined for gcc and icc */
|
|
159
|
-
typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
|
|
160
|
-
|
|
161
|
-
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
|
162
|
-
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
|
163
|
-
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
|
164
|
-
|
|
165
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
|
166
|
-
|
|
167
|
-
#else
|
|
168
|
-
|
|
169
|
-
/* default method, safe and standard.
|
|
170
|
-
can sometimes prove slower */
|
|
171
|
-
|
|
172
128
|
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
|
173
129
|
{
|
|
174
130
|
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
|
@@ -189,9 +145,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
|
|
189
145
|
memcpy(memPtr, &value, sizeof(value));
|
|
190
146
|
}
|
|
191
147
|
|
|
192
|
-
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
|
193
|
-
|
|
194
|
-
|
|
195
148
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
|
196
149
|
{
|
|
197
150
|
if (MEM_isLittleEndian())
|
|
@@ -268,7 +221,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
|
|
268
221
|
header file (to include)
|
|
269
222
|
Copyright (C) 2013-2015, Yann Collet.
|
|
270
223
|
|
|
271
|
-
BSD 2-Clause License (
|
|
224
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
272
225
|
|
|
273
226
|
Redistribution and use in source and binary forms, with or without
|
|
274
227
|
modification, are permitted provided that the following conditions are
|
|
@@ -349,9 +302,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
|
|
349
302
|
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
|
350
303
|
{
|
|
351
304
|
# if defined(_MSC_VER) /* Visual */
|
|
352
|
-
unsigned long r
|
|
353
|
-
_BitScanReverse
|
|
354
|
-
return (unsigned) r;
|
|
305
|
+
unsigned long r;
|
|
306
|
+
return _BitScanReverse(&r, val) ? (unsigned)r : 0;
|
|
355
307
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
|
356
308
|
return __builtin_clz (val) ^ 31;
|
|
357
309
|
# else /* Software version */
|
|
@@ -433,7 +385,7 @@ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
|
433
385
|
}
|
|
434
386
|
|
|
435
387
|
/*! BIT_lookBitsFast :
|
|
436
|
-
* unsafe version; only works
|
|
388
|
+
* unsafe version; only works if nbBits >= 1 */
|
|
437
389
|
MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
|
438
390
|
{
|
|
439
391
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
|
@@ -453,7 +405,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
|
453
405
|
}
|
|
454
406
|
|
|
455
407
|
/*!BIT_readBitsFast :
|
|
456
|
-
* unsafe version; only works
|
|
408
|
+
* unsafe version; only works if nbBits >= 1 */
|
|
457
409
|
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
|
458
410
|
{
|
|
459
411
|
size_t value = BIT_lookBitsFast(bitD, nbBits);
|
|
@@ -510,7 +462,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
|
|
510
462
|
Error codes and messages
|
|
511
463
|
Copyright (C) 2013-2015, Yann Collet
|
|
512
464
|
|
|
513
|
-
BSD 2-Clause License (
|
|
465
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
514
466
|
|
|
515
467
|
Redistribution and use in source and binary forms, with or without
|
|
516
468
|
modification, are permitted provided that the following conditions are
|
|
@@ -609,7 +561,7 @@ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be mor
|
|
|
609
561
|
header file for static linking (only)
|
|
610
562
|
Copyright (C) 2013-2015, Yann Collet
|
|
611
563
|
|
|
612
|
-
BSD 2-Clause License (
|
|
564
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
613
565
|
|
|
614
566
|
Redistribution and use in source and binary forms, with or without
|
|
615
567
|
modification, are permitted provided that the following conditions are
|
|
@@ -753,7 +705,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
|
|
753
705
|
header file for static linking (only)
|
|
754
706
|
Copyright (C) 2013-2015, Yann Collet
|
|
755
707
|
|
|
756
|
-
BSD 2-Clause License (
|
|
708
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
757
709
|
|
|
758
710
|
Redistribution and use in source and binary forms, with or without
|
|
759
711
|
modification, are permitted provided that the following conditions are
|
|
@@ -822,7 +774,7 @@ static size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, si
|
|
|
822
774
|
Header File
|
|
823
775
|
Copyright (C) 2014-2015, Yann Collet.
|
|
824
776
|
|
|
825
|
-
BSD 2-Clause License (
|
|
777
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
826
778
|
|
|
827
779
|
Redistribution and use in source and binary forms, with or without
|
|
828
780
|
modification, are permitted provided that the following conditions are
|
|
@@ -882,7 +834,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */
|
|
|
882
834
|
Header File for static linking only
|
|
883
835
|
Copyright (C) 2014-2015, Yann Collet.
|
|
884
836
|
|
|
885
|
-
BSD 2-Clause License (
|
|
837
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
886
838
|
|
|
887
839
|
Redistribution and use in source and binary forms, with or without
|
|
888
840
|
modification, are permitted provided that the following conditions are
|
|
@@ -946,7 +898,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
|
|
946
898
|
FSE : Finite State Entropy coder
|
|
947
899
|
Copyright (C) 2013-2015, Yann Collet.
|
|
948
900
|
|
|
949
|
-
BSD 2-Clause License (
|
|
901
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
950
902
|
|
|
951
903
|
Redistribution and use in source and binary forms, with or without
|
|
952
904
|
modification, are permitted provided that the following conditions are
|
|
@@ -1450,7 +1402,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
|
|
|
1450
1402
|
Huff0 : Huffman coder, part of New Generation Entropy library
|
|
1451
1403
|
Copyright (C) 2013-2015, Yann Collet.
|
|
1452
1404
|
|
|
1453
|
-
BSD 2-Clause License (
|
|
1405
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
1454
1406
|
|
|
1455
1407
|
Redistribution and use in source and binary forms, with or without
|
|
1456
1408
|
modification, are permitted provided that the following conditions are
|
|
@@ -2609,7 +2561,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
|
2609
2561
|
zstd - standard compression library
|
|
2610
2562
|
Copyright (C) 2014-2015, Yann Collet.
|
|
2611
2563
|
|
|
2612
|
-
BSD 2-Clause License (
|
|
2564
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
2613
2565
|
|
|
2614
2566
|
Redistribution and use in source and binary forms, with or without
|
|
2615
2567
|
modification, are permitted provided that the following conditions are
|
|
@@ -3114,12 +3066,19 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
3114
3066
|
const BYTE* const litEnd = *litPtr + sequence.litLength;
|
|
3115
3067
|
|
|
3116
3068
|
/* checks */
|
|
3117
|
-
|
|
3069
|
+
size_t const seqLength = sequence.litLength + sequence.matchLength;
|
|
3070
|
+
|
|
3071
|
+
if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
|
|
3072
|
+
if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
|
|
3073
|
+
/* Now we know there are no overflow in literal nor match lengths, can use the pointer check */
|
|
3074
|
+
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
|
|
3075
|
+
if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
|
|
3076
|
+
|
|
3118
3077
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
|
3119
3078
|
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
|
3120
3079
|
|
|
3121
3080
|
/* copy Literals */
|
|
3122
|
-
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
|
3081
|
+
ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
|
3123
3082
|
op = oLitEnd;
|
|
3124
3083
|
*litPtr = litEnd; /* update for next sequence */
|
|
3125
3084
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c)
|
|
2
|
+
* Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
low-level memory access routines
|
|
30
30
|
Copyright (C) 2013-2015, Yann Collet.
|
|
31
31
|
|
|
32
|
-
BSD 2-Clause License (
|
|
32
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
33
33
|
|
|
34
34
|
Redistribution and use in source and binary forms, with or without
|
|
35
35
|
modification, are permitted provided that the following conditions are
|
|
@@ -90,7 +90,11 @@ extern "C" {
|
|
|
90
90
|
* Basic Types
|
|
91
91
|
*****************************************************************/
|
|
92
92
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
|
93
|
-
#
|
|
93
|
+
# if defined(_AIX)
|
|
94
|
+
# include <inttypes.h>
|
|
95
|
+
# else
|
|
96
|
+
# include <stdint.h> /* intptr_t */
|
|
97
|
+
# endif
|
|
94
98
|
typedef uint8_t BYTE;
|
|
95
99
|
typedef uint16_t U16;
|
|
96
100
|
typedef int16_t S16;
|
|
@@ -112,27 +116,6 @@ extern "C" {
|
|
|
112
116
|
/****************************************************************
|
|
113
117
|
* Memory I/O
|
|
114
118
|
*****************************************************************/
|
|
115
|
-
/* MEM_FORCE_MEMORY_ACCESS
|
|
116
|
-
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
|
117
|
-
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
|
118
|
-
* The below switch allow to select different access method for improved performance.
|
|
119
|
-
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
|
120
|
-
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
|
121
|
-
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
|
122
|
-
* Method 2 : direct access. This method is portable but violate C standard.
|
|
123
|
-
* It can generate buggy code on targets generating assembly depending on alignment.
|
|
124
|
-
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
|
125
|
-
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
|
126
|
-
* Prefer these methods in priority order (0 > 1 > 2)
|
|
127
|
-
*/
|
|
128
|
-
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
|
129
|
-
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
|
130
|
-
# define MEM_FORCE_MEMORY_ACCESS 2
|
|
131
|
-
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
|
132
|
-
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
|
133
|
-
# define MEM_FORCE_MEMORY_ACCESS 1
|
|
134
|
-
# endif
|
|
135
|
-
#endif
|
|
136
119
|
|
|
137
120
|
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
|
|
138
121
|
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
|
|
@@ -143,33 +126,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
|
|
|
143
126
|
return one.c[0];
|
|
144
127
|
}
|
|
145
128
|
|
|
146
|
-
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
|
147
|
-
|
|
148
|
-
/* violates C standard on structure alignment.
|
|
149
|
-
Only use if no other choice to achieve best performance on target platform */
|
|
150
|
-
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
|
151
|
-
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
|
152
|
-
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
|
153
|
-
|
|
154
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
|
155
|
-
|
|
156
|
-
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
|
157
|
-
|
|
158
|
-
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
|
159
|
-
/* currently only defined for gcc and icc */
|
|
160
|
-
typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
|
|
161
|
-
|
|
162
|
-
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
|
163
|
-
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
|
164
|
-
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
|
165
|
-
|
|
166
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
|
167
|
-
|
|
168
|
-
#else
|
|
169
|
-
|
|
170
|
-
/* default method, safe and standard.
|
|
171
|
-
can sometimes prove slower */
|
|
172
|
-
|
|
173
129
|
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
|
174
130
|
{
|
|
175
131
|
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
|
@@ -190,10 +146,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
|
|
190
146
|
memcpy(memPtr, &value, sizeof(value));
|
|
191
147
|
}
|
|
192
148
|
|
|
193
|
-
|
|
194
|
-
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
|
195
|
-
|
|
196
|
-
|
|
197
149
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
|
198
150
|
{
|
|
199
151
|
if (MEM_isLittleEndian())
|
|
@@ -270,7 +222,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
|
|
270
222
|
header file (to include)
|
|
271
223
|
Copyright (C) 2013-2015, Yann Collet.
|
|
272
224
|
|
|
273
|
-
BSD 2-Clause License (
|
|
225
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
274
226
|
|
|
275
227
|
Redistribution and use in source and binary forms, with or without
|
|
276
228
|
modification, are permitted provided that the following conditions are
|
|
@@ -352,9 +304,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
|
|
352
304
|
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
|
353
305
|
{
|
|
354
306
|
# if defined(_MSC_VER) /* Visual */
|
|
355
|
-
unsigned long r
|
|
356
|
-
_BitScanReverse
|
|
357
|
-
return (unsigned) r;
|
|
307
|
+
unsigned long r;
|
|
308
|
+
return _BitScanReverse(&r, val) ? (unsigned)r : 0;
|
|
358
309
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
|
359
310
|
return __builtin_clz (val) ^ 31;
|
|
360
311
|
# else /* Software version */
|
|
@@ -435,7 +386,7 @@ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
|
435
386
|
}
|
|
436
387
|
|
|
437
388
|
/*! BIT_lookBitsFast :
|
|
438
|
-
* unsafe version; only works
|
|
389
|
+
* unsafe version; only works if nbBits >= 1 */
|
|
439
390
|
MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
|
440
391
|
{
|
|
441
392
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
|
@@ -455,7 +406,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
|
455
406
|
}
|
|
456
407
|
|
|
457
408
|
/*!BIT_readBitsFast :
|
|
458
|
-
* unsafe version; only works
|
|
409
|
+
* unsafe version; only works if nbBits >= 1 */
|
|
459
410
|
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
|
460
411
|
{
|
|
461
412
|
size_t value = BIT_lookBitsFast(bitD, nbBits);
|
|
@@ -512,7 +463,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
|
|
512
463
|
Error codes and messages
|
|
513
464
|
Copyright (C) 2013-2015, Yann Collet
|
|
514
465
|
|
|
515
|
-
BSD 2-Clause License (
|
|
466
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
516
467
|
|
|
517
468
|
Redistribution and use in source and binary forms, with or without
|
|
518
469
|
modification, are permitted provided that the following conditions are
|
|
@@ -611,7 +562,7 @@ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be mor
|
|
|
611
562
|
header file for static linking (only)
|
|
612
563
|
Copyright (C) 2013-2015, Yann Collet
|
|
613
564
|
|
|
614
|
-
BSD 2-Clause License (
|
|
565
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
615
566
|
|
|
616
567
|
Redistribution and use in source and binary forms, with or without
|
|
617
568
|
modification, are permitted provided that the following conditions are
|
|
@@ -755,7 +706,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
|
|
755
706
|
header file for static linking (only)
|
|
756
707
|
Copyright (C) 2013-2015, Yann Collet
|
|
757
708
|
|
|
758
|
-
BSD 2-Clause License (
|
|
709
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
759
710
|
|
|
760
711
|
Redistribution and use in source and binary forms, with or without
|
|
761
712
|
modification, are permitted provided that the following conditions are
|
|
@@ -823,7 +774,7 @@ static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, si
|
|
|
823
774
|
Header File
|
|
824
775
|
Copyright (C) 2014-2015, Yann Collet.
|
|
825
776
|
|
|
826
|
-
BSD 2-Clause License (
|
|
777
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
827
778
|
|
|
828
779
|
Redistribution and use in source and binary forms, with or without
|
|
829
780
|
modification, are permitted provided that the following conditions are
|
|
@@ -883,7 +834,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */
|
|
|
883
834
|
Header File for static linking only
|
|
884
835
|
Copyright (C) 2014-2015, Yann Collet.
|
|
885
836
|
|
|
886
|
-
BSD 2-Clause License (
|
|
837
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
887
838
|
|
|
888
839
|
Redistribution and use in source and binary forms, with or without
|
|
889
840
|
modification, are permitted provided that the following conditions are
|
|
@@ -947,7 +898,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
|
|
|
947
898
|
FSE : Finite State Entropy coder
|
|
948
899
|
Copyright (C) 2013-2015, Yann Collet.
|
|
949
900
|
|
|
950
|
-
BSD 2-Clause License (
|
|
901
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
951
902
|
|
|
952
903
|
Redistribution and use in source and binary forms, with or without
|
|
953
904
|
modification, are permitted provided that the following conditions are
|
|
@@ -1451,7 +1402,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
|
|
|
1451
1402
|
Huff0 : Huffman coder, part of New Generation Entropy library
|
|
1452
1403
|
Copyright (C) 2013-2015, Yann Collet.
|
|
1453
1404
|
|
|
1454
|
-
BSD 2-Clause License (
|
|
1405
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
1455
1406
|
|
|
1456
1407
|
Redistribution and use in source and binary forms, with or without
|
|
1457
1408
|
modification, are permitted provided that the following conditions are
|
|
@@ -2248,7 +2199,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
|
2248
2199
|
zstd - standard compression library
|
|
2249
2200
|
Copyright (C) 2014-2015, Yann Collet.
|
|
2250
2201
|
|
|
2251
|
-
BSD 2-Clause License (
|
|
2202
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
|
2252
2203
|
|
|
2253
2204
|
Redistribution and use in source and binary forms, with or without
|
|
2254
2205
|
modification, are permitted provided that the following conditions are
|
|
@@ -2755,18 +2706,24 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
|
2755
2706
|
const BYTE* const litEnd = *litPtr + sequence.litLength;
|
|
2756
2707
|
|
|
2757
2708
|
/* checks */
|
|
2758
|
-
|
|
2709
|
+
size_t const seqLength = sequence.litLength + sequence.matchLength;
|
|
2710
|
+
|
|
2711
|
+
if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
|
|
2712
|
+
if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
|
|
2713
|
+
/* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
|
|
2714
|
+
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
|
|
2715
|
+
if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected);
|
|
2716
|
+
|
|
2759
2717
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
|
2760
2718
|
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
|
2761
2719
|
|
|
2762
2720
|
/* copy Literals */
|
|
2763
|
-
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
|
2721
|
+
ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
|
2764
2722
|
op = oLitEnd;
|
|
2765
2723
|
*litPtr = litEnd; /* update for next sequence */
|
|
2766
2724
|
|
|
2767
2725
|
/* copy Match */
|
|
2768
|
-
{
|
|
2769
|
-
const BYTE* match = op - sequence.offset;
|
|
2726
|
+
{ const BYTE* match = op - sequence.offset;
|
|
2770
2727
|
|
|
2771
2728
|
/* check */
|
|
2772
2729
|
if (sequence.offset > (size_t)op) return ERROR(corruption_detected); /* address space overflow test (this test seems kept by clang optimizer) */
|