extzstd 0.3.1 → 0.3.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +28 -14
- data/contrib/zstd/CHANGELOG +301 -56
- data/contrib/zstd/CONTRIBUTING.md +169 -72
- data/contrib/zstd/LICENSE +4 -4
- data/contrib/zstd/Makefile +116 -87
- data/contrib/zstd/Package.swift +36 -0
- data/contrib/zstd/README.md +62 -32
- data/contrib/zstd/TESTING.md +2 -3
- data/contrib/zstd/appveyor.yml +52 -136
- data/contrib/zstd/lib/BUCK +5 -7
- data/contrib/zstd/lib/Makefile +225 -222
- data/contrib/zstd/lib/README.md +51 -6
- data/contrib/zstd/lib/common/allocations.h +55 -0
- data/contrib/zstd/lib/common/bits.h +200 -0
- data/contrib/zstd/lib/common/bitstream.h +45 -62
- data/contrib/zstd/lib/common/compiler.h +205 -22
- data/contrib/zstd/lib/common/cpu.h +1 -3
- data/contrib/zstd/lib/common/debug.c +1 -1
- data/contrib/zstd/lib/common/debug.h +12 -19
- data/contrib/zstd/lib/common/entropy_common.c +172 -48
- data/contrib/zstd/lib/common/error_private.c +10 -2
- data/contrib/zstd/lib/common/error_private.h +82 -3
- data/contrib/zstd/lib/common/fse.h +37 -86
- data/contrib/zstd/lib/common/fse_decompress.c +117 -92
- data/contrib/zstd/lib/common/huf.h +99 -166
- data/contrib/zstd/lib/common/mem.h +124 -142
- data/contrib/zstd/lib/common/pool.c +54 -27
- data/contrib/zstd/lib/common/pool.h +10 -4
- data/contrib/zstd/lib/common/portability_macros.h +156 -0
- data/contrib/zstd/lib/common/threading.c +74 -19
- data/contrib/zstd/lib/common/threading.h +5 -10
- data/contrib/zstd/lib/common/xxhash.c +7 -847
- data/contrib/zstd/lib/common/xxhash.h +5568 -167
- data/contrib/zstd/lib/common/zstd_common.c +2 -37
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_internal.h +132 -187
- data/contrib/zstd/lib/common/zstd_trace.h +163 -0
- data/contrib/zstd/lib/compress/clevels.h +134 -0
- data/contrib/zstd/lib/compress/fse_compress.c +83 -157
- data/contrib/zstd/lib/compress/hist.c +27 -29
- data/contrib/zstd/lib/compress/hist.h +2 -2
- data/contrib/zstd/lib/compress/huf_compress.c +916 -279
- data/contrib/zstd/lib/compress/zstd_compress.c +3773 -1019
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +610 -203
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +119 -42
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +16 -6
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +42 -19
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +1 -1
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +49 -317
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +1 -1
- data/contrib/zstd/lib/compress/zstd_cwksp.h +320 -103
- data/contrib/zstd/lib/compress/zstd_double_fast.c +388 -151
- data/contrib/zstd/lib/compress/zstd_double_fast.h +3 -2
- data/contrib/zstd/lib/compress/zstd_fast.c +729 -265
- data/contrib/zstd/lib/compress/zstd_fast.h +3 -2
- data/contrib/zstd/lib/compress/zstd_lazy.c +1270 -251
- data/contrib/zstd/lib/compress/zstd_lazy.h +61 -1
- data/contrib/zstd/lib/compress/zstd_ldm.c +324 -219
- data/contrib/zstd/lib/compress/zstd_ldm.h +9 -2
- data/contrib/zstd/lib/compress/zstd_ldm_geartab.h +106 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +481 -209
- data/contrib/zstd/lib/compress/zstd_opt.h +1 -1
- data/contrib/zstd/lib/compress/zstdmt_compress.c +181 -457
- data/contrib/zstd/lib/compress/zstdmt_compress.h +34 -113
- data/contrib/zstd/lib/decompress/huf_decompress.c +1199 -565
- data/contrib/zstd/lib/decompress/huf_decompress_amd64.S +576 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +12 -12
- data/contrib/zstd/lib/decompress/zstd_ddict.h +2 -2
- data/contrib/zstd/lib/decompress/zstd_decompress.c +627 -157
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1086 -326
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +19 -5
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +62 -13
- data/contrib/zstd/lib/deprecated/zbuff.h +1 -1
- data/contrib/zstd/lib/deprecated/zbuff_common.c +1 -1
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +24 -4
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +3 -1
- data/contrib/zstd/lib/dictBuilder/cover.c +73 -52
- data/contrib/zstd/lib/dictBuilder/cover.h +7 -6
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +1 -1
- data/contrib/zstd/lib/dictBuilder/fastcover.c +44 -35
- data/contrib/zstd/lib/dictBuilder/zdict.c +103 -111
- data/contrib/zstd/lib/legacy/zstd_legacy.h +8 -1
- data/contrib/zstd/lib/legacy/zstd_v01.c +21 -54
- data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v02.c +29 -70
- data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v03.c +30 -73
- data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v04.c +29 -71
- data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v05.c +40 -86
- data/contrib/zstd/lib/legacy/zstd_v05.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v06.c +47 -88
- data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v07.c +40 -83
- data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
- data/contrib/zstd/lib/libzstd.mk +214 -0
- data/contrib/zstd/lib/libzstd.pc.in +7 -6
- data/contrib/zstd/lib/module.modulemap +35 -0
- data/contrib/zstd/lib/{dictBuilder/zdict.h → zdict.h} +203 -34
- data/contrib/zstd/lib/zstd.h +1217 -287
- data/contrib/zstd/lib/{common/zstd_errors.h → zstd_errors.h} +28 -8
- data/ext/extconf.rb +7 -6
- data/ext/extzstd.c +19 -10
- data/ext/extzstd.h +6 -0
- data/ext/libzstd_conf.h +0 -1
- data/ext/zstd_decompress_asm.S +1 -0
- data/gemstub.rb +3 -21
- data/lib/extzstd/version.rb +6 -1
- data/lib/extzstd.rb +0 -2
- data/test/test_basic.rb +0 -5
- metadata +18 -6
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -52,7 +52,11 @@ extern "C" {
|
|
52
52
|
* Basic Types
|
53
53
|
*****************************************************************/
|
54
54
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
55
|
-
#
|
55
|
+
# if defined(_AIX)
|
56
|
+
# include <inttypes.h>
|
57
|
+
# else
|
58
|
+
# include <stdint.h> /* intptr_t */
|
59
|
+
# endif
|
56
60
|
typedef uint8_t BYTE;
|
57
61
|
typedef uint16_t U16;
|
58
62
|
typedef int16_t S16;
|
@@ -74,7 +78,7 @@ extern "C" {
|
|
74
78
|
/*-*************************************
|
75
79
|
* Debug
|
76
80
|
***************************************/
|
77
|
-
#include "debug.h"
|
81
|
+
#include "../common/debug.h"
|
78
82
|
#ifndef assert
|
79
83
|
# define assert(condition) ((void)0)
|
80
84
|
#endif
|
@@ -83,27 +87,6 @@ extern "C" {
|
|
83
87
|
/****************************************************************
|
84
88
|
* Memory I/O
|
85
89
|
*****************************************************************/
|
86
|
-
/* MEM_FORCE_MEMORY_ACCESS
|
87
|
-
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
88
|
-
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
89
|
-
* The below switch allow to select different access method for improved performance.
|
90
|
-
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
91
|
-
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
92
|
-
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
93
|
-
* Method 2 : direct access. This method is portable but violate C standard.
|
94
|
-
* It can generate buggy code on targets generating assembly depending on alignment.
|
95
|
-
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
96
|
-
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
97
|
-
* Prefer these methods in priority order (0 > 1 > 2)
|
98
|
-
*/
|
99
|
-
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
100
|
-
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
101
|
-
# define MEM_FORCE_MEMORY_ACCESS 2
|
102
|
-
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
103
|
-
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
104
|
-
# define MEM_FORCE_MEMORY_ACCESS 1
|
105
|
-
# endif
|
106
|
-
#endif
|
107
90
|
|
108
91
|
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
|
109
92
|
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
|
@@ -114,33 +97,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
|
|
114
97
|
return one.c[0];
|
115
98
|
}
|
116
99
|
|
117
|
-
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
118
|
-
|
119
|
-
/* violates C standard on structure alignment.
|
120
|
-
Only use if no other choice to achieve best performance on target platform */
|
121
|
-
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
122
|
-
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
123
|
-
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
124
|
-
|
125
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
126
|
-
|
127
|
-
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
128
|
-
|
129
|
-
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
130
|
-
/* currently only defined for gcc and icc */
|
131
|
-
typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
|
132
|
-
|
133
|
-
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
134
|
-
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
135
|
-
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
136
|
-
|
137
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
138
|
-
|
139
|
-
#else
|
140
|
-
|
141
|
-
/* default method, safe and standard.
|
142
|
-
can sometimes prove slower */
|
143
|
-
|
144
100
|
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
145
101
|
{
|
146
102
|
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
@@ -161,9 +117,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
|
161
117
|
memcpy(memPtr, &value, sizeof(value));
|
162
118
|
}
|
163
119
|
|
164
|
-
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
165
|
-
|
166
|
-
|
167
120
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
168
121
|
{
|
169
122
|
if (MEM_isLittleEndian())
|
@@ -541,7 +494,7 @@ If there is an error, the function will return an error code, which can be teste
|
|
541
494
|
header file (to include)
|
542
495
|
Copyright (C) 2013-2015, Yann Collet.
|
543
496
|
|
544
|
-
BSD 2-Clause License (
|
497
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
545
498
|
|
546
499
|
Redistribution and use in source and binary forms, with or without
|
547
500
|
modification, are permitted provided that the following conditions are
|
@@ -623,9 +576,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
|
623
576
|
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
624
577
|
{
|
625
578
|
# if defined(_MSC_VER) /* Visual */
|
626
|
-
unsigned long r
|
627
|
-
_BitScanReverse
|
628
|
-
return (unsigned) r;
|
579
|
+
unsigned long r;
|
580
|
+
return _BitScanReverse(&r, val) ? (unsigned)r : 0;
|
629
581
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
630
582
|
return __builtin_clz (val) ^ 31;
|
631
583
|
# else /* Software version */
|
@@ -700,7 +652,7 @@ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
700
652
|
}
|
701
653
|
|
702
654
|
/*! BIT_lookBitsFast :
|
703
|
-
* unsafe version; only works
|
655
|
+
* unsafe version; only works if nbBits >= 1 */
|
704
656
|
MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
705
657
|
{
|
706
658
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
@@ -720,7 +672,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
720
672
|
}
|
721
673
|
|
722
674
|
/*!BIT_readBitsFast :
|
723
|
-
* unsafe version; only works
|
675
|
+
* unsafe version; only works if nbBits >= 1 */
|
724
676
|
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
725
677
|
{
|
726
678
|
size_t value = BIT_lookBitsFast(bitD, nbBits);
|
@@ -781,7 +733,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
|
781
733
|
header file for static linking (only)
|
782
734
|
Copyright (C) 2013-2015, Yann Collet
|
783
735
|
|
784
|
-
BSD 2-Clause License (
|
736
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
785
737
|
|
786
738
|
Redistribution and use in source and binary forms, with or without
|
787
739
|
modification, are permitted provided that the following conditions are
|
@@ -930,7 +882,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
|
930
882
|
FSE : Finite State Entropy coder
|
931
883
|
Copyright (C) 2013-2015, Yann Collet.
|
932
884
|
|
933
|
-
BSD 2-Clause License (
|
885
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
934
886
|
|
935
887
|
Redistribution and use in source and binary forms, with or without
|
936
888
|
modification, are permitted provided that the following conditions are
|
@@ -1436,7 +1388,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
|
|
1436
1388
|
header file
|
1437
1389
|
Copyright (C) 2013-2015, Yann Collet.
|
1438
1390
|
|
1439
|
-
BSD 2-Clause License (
|
1391
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
1440
1392
|
|
1441
1393
|
Redistribution and use in source and binary forms, with or without
|
1442
1394
|
modification, are permitted provided that the following conditions are
|
@@ -1514,7 +1466,7 @@ static unsigned HUF_isError(size_t code); /* tells if a return value i
|
|
1514
1466
|
header file for static linking (only)
|
1515
1467
|
Copyright (C) 2013-2015, Yann Collet
|
1516
1468
|
|
1517
|
-
BSD 2-Clause License (
|
1469
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
1518
1470
|
|
1519
1471
|
Redistribution and use in source and binary forms, with or without
|
1520
1472
|
modification, are permitted provided that the following conditions are
|
@@ -1601,7 +1553,7 @@ static size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const
|
|
1601
1553
|
Huff0 : Huffman coder, part of New Generation Entropy library
|
1602
1554
|
Copyright (C) 2013-2015, Yann Collet.
|
1603
1555
|
|
1604
|
-
BSD 2-Clause License (
|
1556
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
1605
1557
|
|
1606
1558
|
Redistribution and use in source and binary forms, with or without
|
1607
1559
|
modification, are permitted provided that the following conditions are
|
@@ -2401,7 +2353,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
|
|
2401
2353
|
zstd - decompression module fo v0.4 legacy format
|
2402
2354
|
Copyright (C) 2015-2016, Yann Collet.
|
2403
2355
|
|
2404
|
-
BSD 2-Clause License (
|
2356
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
2405
2357
|
|
2406
2358
|
Redistribution and use in source and binary forms, with or without
|
2407
2359
|
modification, are permitted provided that the following conditions are
|
@@ -2876,13 +2828,19 @@ static size_t ZSTD_execSequence(BYTE* op,
|
|
2876
2828
|
const BYTE* const litEnd = *litPtr + sequence.litLength;
|
2877
2829
|
const BYTE* match = oLitEnd - sequence.offset;
|
2878
2830
|
|
2879
|
-
/*
|
2880
|
-
|
2831
|
+
/* checks */
|
2832
|
+
size_t const seqLength = sequence.litLength + sequence.matchLength;
|
2833
|
+
|
2834
|
+
if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
|
2835
|
+
if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
|
2836
|
+
/* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
|
2837
|
+
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
|
2838
|
+
|
2881
2839
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
2882
|
-
if (litEnd > litLimit) return ERROR(corruption_detected); /*
|
2840
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
2883
2841
|
|
2884
2842
|
/* copy Literals */
|
2885
|
-
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
2843
|
+
ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
2886
2844
|
op = oLitEnd;
|
2887
2845
|
*litPtr = litEnd; /* update for next sequence */
|
2888
2846
|
|
@@ -3283,7 +3241,7 @@ static void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* dict, s
|
|
3283
3241
|
Buffered version of Zstd compression library
|
3284
3242
|
Copyright (C) 2015, Yann Collet.
|
3285
3243
|
|
3286
|
-
BSD 2-Clause License (
|
3244
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
3287
3245
|
|
3288
3246
|
Redistribution and use in source and binary forms, with or without
|
3289
3247
|
modification, are permitted provided that the following conditions are
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -19,7 +19,7 @@
|
|
19
19
|
low-level memory access routines
|
20
20
|
Copyright (C) 2013-2015, Yann Collet.
|
21
21
|
|
22
|
-
BSD 2-Clause License (
|
22
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
23
23
|
|
24
24
|
Redistribution and use in source and binary forms, with or without
|
25
25
|
modification, are permitted provided that the following conditions are
|
@@ -80,7 +80,11 @@ extern "C" {
|
|
80
80
|
* Basic Types
|
81
81
|
*****************************************************************/
|
82
82
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
83
|
-
#
|
83
|
+
# if defined(_AIX)
|
84
|
+
# include <inttypes.h>
|
85
|
+
# else
|
86
|
+
# include <stdint.h> /* intptr_t */
|
87
|
+
# endif
|
84
88
|
typedef uint8_t BYTE;
|
85
89
|
typedef uint16_t U16;
|
86
90
|
typedef int16_t S16;
|
@@ -102,27 +106,6 @@ extern "C" {
|
|
102
106
|
/*-**************************************************************
|
103
107
|
* Memory I/O
|
104
108
|
*****************************************************************/
|
105
|
-
/* MEM_FORCE_MEMORY_ACCESS :
|
106
|
-
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
107
|
-
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
108
|
-
* The below switch allow to select different access method for improved performance.
|
109
|
-
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
110
|
-
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
111
|
-
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
112
|
-
* Method 2 : direct access. This method is portable but violate C standard.
|
113
|
-
* It can generate buggy code on targets depending on alignment.
|
114
|
-
* In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
115
|
-
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
116
|
-
* Prefer these methods in priority order (0 > 1 > 2)
|
117
|
-
*/
|
118
|
-
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
119
|
-
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
120
|
-
# define MEM_FORCE_MEMORY_ACCESS 2
|
121
|
-
# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
|
122
|
-
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
123
|
-
# define MEM_FORCE_MEMORY_ACCESS 1
|
124
|
-
# endif
|
125
|
-
#endif
|
126
109
|
|
127
110
|
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
|
128
111
|
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
|
@@ -133,37 +116,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
|
|
133
116
|
return one.c[0];
|
134
117
|
}
|
135
118
|
|
136
|
-
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
137
|
-
|
138
|
-
/* violates C standard, by lying on structure alignment.
|
139
|
-
Only use if no other choice to achieve best performance on target platform */
|
140
|
-
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
141
|
-
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
142
|
-
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
143
|
-
|
144
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
145
|
-
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
|
146
|
-
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
|
147
|
-
|
148
|
-
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
149
|
-
|
150
|
-
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
151
|
-
/* currently only defined for gcc and icc */
|
152
|
-
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
|
153
|
-
|
154
|
-
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
155
|
-
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
156
|
-
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
157
|
-
|
158
|
-
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
159
|
-
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
|
160
|
-
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
|
161
|
-
|
162
|
-
#else
|
163
|
-
|
164
|
-
/* default method, safe and standard.
|
165
|
-
can sometimes prove slower */
|
166
|
-
|
167
119
|
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
168
120
|
{
|
169
121
|
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
@@ -194,9 +146,6 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value)
|
|
194
146
|
memcpy(memPtr, &value, sizeof(value));
|
195
147
|
}
|
196
148
|
|
197
|
-
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
198
|
-
|
199
|
-
|
200
149
|
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
201
150
|
{
|
202
151
|
if (MEM_isLittleEndian())
|
@@ -261,7 +210,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
|
261
210
|
Header File for static linking only
|
262
211
|
Copyright (C) 2014-2016, Yann Collet.
|
263
212
|
|
264
|
-
BSD 2-Clause License (
|
213
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
265
214
|
|
266
215
|
Redistribution and use in source and binary forms, with or without
|
267
216
|
modification, are permitted provided that the following conditions are
|
@@ -285,7 +234,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
|
285
234
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
286
235
|
|
287
236
|
You can contact the author at :
|
288
|
-
- zstd homepage :
|
237
|
+
- zstd homepage : https://facebook.github.io/zstd
|
289
238
|
*/
|
290
239
|
#ifndef ZSTD_STATIC_H
|
291
240
|
#define ZSTD_STATIC_H
|
@@ -397,7 +346,7 @@ size_t ZSTDv05_decompressBlock(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity
|
|
397
346
|
Header File for include
|
398
347
|
Copyright (C) 2014-2016, Yann Collet.
|
399
348
|
|
400
|
-
BSD 2-Clause License (
|
349
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
401
350
|
|
402
351
|
Redistribution and use in source and binary forms, with or without
|
403
352
|
modification, are permitted provided that the following conditions are
|
@@ -484,7 +433,7 @@ static const size_t ZSTDv05_frameHeaderSize_min = 5;
|
|
484
433
|
#define FSEv05_ENCODING_DYNAMIC 3
|
485
434
|
|
486
435
|
|
487
|
-
#define
|
436
|
+
#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12
|
488
437
|
|
489
438
|
#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
|
490
439
|
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
|
@@ -552,7 +501,7 @@ typedef struct {
|
|
552
501
|
header file
|
553
502
|
Copyright (C) 2013-2015, Yann Collet.
|
554
503
|
|
555
|
-
BSD 2-Clause License (
|
504
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
556
505
|
|
557
506
|
Redistribution and use in source and binary forms, with or without
|
558
507
|
modification, are permitted provided that the following conditions are
|
@@ -671,7 +620,7 @@ size_t FSEv05_decompress_usingDTable(void* dst, size_t dstCapacity, const void*
|
|
671
620
|
header file (to include)
|
672
621
|
Copyright (C) 2013-2016, Yann Collet.
|
673
622
|
|
674
|
-
BSD 2-Clause License (
|
623
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
675
624
|
|
676
625
|
Redistribution and use in source and binary forms, with or without
|
677
626
|
modification, are permitted provided that the following conditions are
|
@@ -752,9 +701,8 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits);
|
|
752
701
|
MEM_STATIC unsigned BITv05_highbit32 (U32 val)
|
753
702
|
{
|
754
703
|
# if defined(_MSC_VER) /* Visual */
|
755
|
-
unsigned long r
|
756
|
-
_BitScanReverse
|
757
|
-
return (unsigned) r;
|
704
|
+
unsigned long r;
|
705
|
+
return _BitScanReverse(&r, val) ? (unsigned)r : 0;
|
758
706
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
759
707
|
return __builtin_clz (val) ^ 31;
|
760
708
|
# else /* Software version */
|
@@ -826,7 +774,7 @@ MEM_STATIC size_t BITv05_lookBits(BITv05_DStream_t* bitD, U32 nbBits)
|
|
826
774
|
}
|
827
775
|
|
828
776
|
/*! BITv05_lookBitsFast :
|
829
|
-
* unsafe version; only works
|
777
|
+
* unsafe version; only works if nbBits >= 1 */
|
830
778
|
MEM_STATIC size_t BITv05_lookBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
|
831
779
|
{
|
832
780
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
@@ -846,7 +794,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits)
|
|
846
794
|
}
|
847
795
|
|
848
796
|
/*!BITv05_readBitsFast :
|
849
|
-
* unsafe version; only works
|
797
|
+
* unsafe version; only works if nbBits >= 1 */
|
850
798
|
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits)
|
851
799
|
{
|
852
800
|
size_t value = BITv05_lookBitsFast(bitD, nbBits);
|
@@ -901,7 +849,7 @@ MEM_STATIC unsigned BITv05_endOfDStream(const BITv05_DStream_t* DStream)
|
|
901
849
|
header file for static linking (only)
|
902
850
|
Copyright (C) 2013-2015, Yann Collet
|
903
851
|
|
904
|
-
BSD 2-Clause License (
|
852
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
905
853
|
|
906
854
|
Redistribution and use in source and binary forms, with or without
|
907
855
|
modification, are permitted provided that the following conditions are
|
@@ -1051,7 +999,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
|
|
1051
999
|
FSEv05 : Finite State Entropy coder
|
1052
1000
|
Copyright (C) 2013-2015, Yann Collet.
|
1053
1001
|
|
1054
|
-
BSD 2-Clause License (
|
1002
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
1055
1003
|
|
1056
1004
|
Redistribution and use in source and binary forms, with or without
|
1057
1005
|
modification, are permitted provided that the following conditions are
|
@@ -1537,7 +1485,7 @@ size_t FSEv05_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t
|
|
1537
1485
|
header file
|
1538
1486
|
Copyright (C) 2013-2016, Yann Collet.
|
1539
1487
|
|
1540
|
-
BSD 2-Clause License (
|
1488
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
1541
1489
|
|
1542
1490
|
Redistribution and use in source and binary forms, with or without
|
1543
1491
|
modification, are permitted provided that the following conditions are
|
@@ -1610,7 +1558,7 @@ const char* HUFv05_getErrorName(size_t code); /* provides error code string (u
|
|
1610
1558
|
header file, for static linking only
|
1611
1559
|
Copyright (C) 2013-2016, Yann Collet
|
1612
1560
|
|
1613
|
-
BSD 2-Clause License (
|
1561
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
1614
1562
|
|
1615
1563
|
Redistribution and use in source and binary forms, with or without
|
1616
1564
|
modification, are permitted provided that the following conditions are
|
@@ -1702,7 +1650,7 @@ size_t HUFv05_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void
|
|
1702
1650
|
Huff0 : Huffman coder, part of New Generation Entropy library
|
1703
1651
|
Copyright (C) 2013-2015, Yann Collet.
|
1704
1652
|
|
1705
|
-
BSD 2-Clause License (
|
1653
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
1706
1654
|
|
1707
1655
|
Redistribution and use in source and binary forms, with or without
|
1708
1656
|
modification, are permitted provided that the following conditions are
|
@@ -2547,7 +2495,7 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
|
2547
2495
|
zstd - standard compression library
|
2548
2496
|
Copyright (C) 2014-2016, Yann Collet.
|
2549
2497
|
|
2550
|
-
BSD 2-Clause License (
|
2498
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
2551
2499
|
|
2552
2500
|
Redistribution and use in source and binary forms, with or without
|
2553
2501
|
modification, are permitted provided that the following conditions are
|
@@ -2645,7 +2593,7 @@ struct ZSTDv05_DCtx_s
|
|
2645
2593
|
FSEv05_DTable LLTable[FSEv05_DTABLE_SIZE_U32(LLFSEv05Log)];
|
2646
2594
|
FSEv05_DTable OffTable[FSEv05_DTABLE_SIZE_U32(OffFSEv05Log)];
|
2647
2595
|
FSEv05_DTable MLTable[FSEv05_DTABLE_SIZE_U32(MLFSEv05Log)];
|
2648
|
-
unsigned hufTableX4[HUFv05_DTABLE_SIZE(
|
2596
|
+
unsigned hufTableX4[HUFv05_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)];
|
2649
2597
|
const void* previousDstEnd;
|
2650
2598
|
const void* base;
|
2651
2599
|
const void* vBase;
|
@@ -2673,7 +2621,7 @@ size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx)
|
|
2673
2621
|
dctx->base = NULL;
|
2674
2622
|
dctx->vBase = NULL;
|
2675
2623
|
dctx->dictEnd = NULL;
|
2676
|
-
dctx->hufTableX4[0] =
|
2624
|
+
dctx->hufTableX4[0] = ZSTD_HUFFDTABLE_CAPACITY_LOG;
|
2677
2625
|
dctx->flagStaticTables = 0;
|
2678
2626
|
return 0;
|
2679
2627
|
}
|
@@ -2829,7 +2777,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
|
|
2829
2777
|
|
2830
2778
|
static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
2831
2779
|
{
|
2832
|
-
const BYTE* const in = (const BYTE*
|
2780
|
+
const BYTE* const in = (const BYTE*)src;
|
2833
2781
|
BYTE headerFlags;
|
2834
2782
|
U32 cSize;
|
2835
2783
|
|
@@ -2998,7 +2946,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
|
|
2998
2946
|
FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
|
2999
2947
|
const void* src, size_t srcSize, U32 flagStaticTable)
|
3000
2948
|
{
|
3001
|
-
const BYTE* const istart = (const BYTE*
|
2949
|
+
const BYTE* const istart = (const BYTE*)src;
|
3002
2950
|
const BYTE* ip = istart;
|
3003
2951
|
const BYTE* const iend = istart + srcSize;
|
3004
2952
|
U32 LLtype, Offtype, MLtype;
|
@@ -3234,13 +3182,19 @@ static size_t ZSTDv05_execSequence(BYTE* op,
|
|
3234
3182
|
const BYTE* const litEnd = *litPtr + sequence.litLength;
|
3235
3183
|
const BYTE* match = oLitEnd - sequence.offset;
|
3236
3184
|
|
3237
|
-
/*
|
3238
|
-
|
3185
|
+
/* checks */
|
3186
|
+
size_t const seqLength = sequence.litLength + sequence.matchLength;
|
3187
|
+
|
3188
|
+
if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
|
3189
|
+
if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
|
3190
|
+
/* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
|
3191
|
+
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
|
3192
|
+
|
3239
3193
|
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
3240
|
-
if (litEnd > litLimit) return ERROR(corruption_detected); /*
|
3194
|
+
if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
|
3241
3195
|
|
3242
3196
|
/* copy Literals */
|
3243
|
-
ZSTDv05_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
3197
|
+
ZSTDv05_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
3244
3198
|
op = oLitEnd;
|
3245
3199
|
*litPtr = litEnd; /* update for next sequence */
|
3246
3200
|
|
@@ -3306,7 +3260,7 @@ static size_t ZSTDv05_decompressSequences(
|
|
3306
3260
|
{
|
3307
3261
|
const BYTE* ip = (const BYTE*)seqStart;
|
3308
3262
|
const BYTE* const iend = ip + seqSize;
|
3309
|
-
BYTE* const ostart = (BYTE*
|
3263
|
+
BYTE* const ostart = (BYTE*)dst;
|
3310
3264
|
BYTE* op = ostart;
|
3311
3265
|
BYTE* const oend = ostart + maxDstSize;
|
3312
3266
|
size_t errorCode, dumpsLength=0;
|
@@ -3419,7 +3373,7 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
|
|
3419
3373
|
{
|
3420
3374
|
const BYTE* ip = (const BYTE*)src;
|
3421
3375
|
const BYTE* iend = ip + srcSize;
|
3422
|
-
BYTE* const ostart = (BYTE*
|
3376
|
+
BYTE* const ostart = (BYTE*)dst;
|
3423
3377
|
BYTE* op = ostart;
|
3424
3378
|
BYTE* const oend = ostart + maxDstSize;
|
3425
3379
|
size_t remainingSize = srcSize;
|
@@ -3746,7 +3700,7 @@ size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, s
|
|
3746
3700
|
Buffered version of Zstd compression library
|
3747
3701
|
Copyright (C) 2015-2016, Yann Collet.
|
3748
3702
|
|
3749
|
-
BSD 2-Clause License (
|
3703
|
+
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
|
3750
3704
|
|
3751
3705
|
Redistribution and use in source and binary forms, with or without
|
3752
3706
|
modification, are permitted provided that the following conditions are
|