extzstd 0.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.ja.md +8 -0
- data/README.md +1 -1
- data/contrib/zstd/CHANGELOG +94 -0
- data/contrib/zstd/CONTRIBUTING.md +351 -1
- data/contrib/zstd/Makefile +32 -10
- data/contrib/zstd/README.md +33 -10
- data/contrib/zstd/TESTING.md +2 -2
- data/contrib/zstd/appveyor.yml +42 -4
- data/contrib/zstd/lib/Makefile +128 -60
- data/contrib/zstd/lib/README.md +47 -16
- data/contrib/zstd/lib/common/bitstream.h +38 -39
- data/contrib/zstd/lib/common/compiler.h +40 -5
- data/contrib/zstd/lib/common/cpu.h +1 -1
- data/contrib/zstd/lib/common/debug.c +11 -31
- data/contrib/zstd/lib/common/debug.h +11 -31
- data/contrib/zstd/lib/common/entropy_common.c +13 -33
- data/contrib/zstd/lib/common/error_private.c +2 -1
- data/contrib/zstd/lib/common/error_private.h +6 -2
- data/contrib/zstd/lib/common/fse.h +12 -32
- data/contrib/zstd/lib/common/fse_decompress.c +12 -35
- data/contrib/zstd/lib/common/huf.h +15 -33
- data/contrib/zstd/lib/common/mem.h +75 -2
- data/contrib/zstd/lib/common/pool.c +8 -4
- data/contrib/zstd/lib/common/pool.h +2 -2
- data/contrib/zstd/lib/common/threading.c +50 -4
- data/contrib/zstd/lib/common/threading.h +36 -4
- data/contrib/zstd/lib/common/xxhash.c +23 -35
- data/contrib/zstd/lib/common/xxhash.h +11 -31
- data/contrib/zstd/lib/common/zstd_common.c +1 -1
- data/contrib/zstd/lib/common/zstd_errors.h +2 -1
- data/contrib/zstd/lib/common/zstd_internal.h +154 -26
- data/contrib/zstd/lib/compress/fse_compress.c +17 -40
- data/contrib/zstd/lib/compress/hist.c +15 -35
- data/contrib/zstd/lib/compress/hist.h +12 -32
- data/contrib/zstd/lib/compress/huf_compress.c +92 -92
- data/contrib/zstd/lib/compress/zstd_compress.c +1191 -1330
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +317 -55
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +419 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +845 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +525 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +65 -43
- data/contrib/zstd/lib/compress/zstd_double_fast.h +2 -2
- data/contrib/zstd/lib/compress/zstd_fast.c +92 -66
- data/contrib/zstd/lib/compress/zstd_fast.h +2 -2
- data/contrib/zstd/lib/compress/zstd_lazy.c +74 -42
- data/contrib/zstd/lib/compress/zstd_lazy.h +1 -1
- data/contrib/zstd/lib/compress/zstd_ldm.c +32 -10
- data/contrib/zstd/lib/compress/zstd_ldm.h +7 -2
- data/contrib/zstd/lib/compress/zstd_opt.c +81 -114
- data/contrib/zstd/lib/compress/zstd_opt.h +1 -1
- data/contrib/zstd/lib/compress/zstdmt_compress.c +95 -51
- data/contrib/zstd/lib/compress/zstdmt_compress.h +3 -2
- data/contrib/zstd/lib/decompress/huf_decompress.c +76 -60
- data/contrib/zstd/lib/decompress/zstd_ddict.c +12 -8
- data/contrib/zstd/lib/decompress/zstd_ddict.h +2 -2
- data/contrib/zstd/lib/decompress/zstd_decompress.c +292 -172
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +459 -338
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +3 -3
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +18 -4
- data/contrib/zstd/lib/deprecated/zbuff.h +9 -8
- data/contrib/zstd/lib/deprecated/zbuff_common.c +2 -2
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +1 -1
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +1 -1
- data/contrib/zstd/lib/dictBuilder/cover.c +164 -54
- data/contrib/zstd/lib/dictBuilder/cover.h +52 -7
- data/contrib/zstd/lib/dictBuilder/fastcover.c +60 -43
- data/contrib/zstd/lib/dictBuilder/zdict.c +43 -19
- data/contrib/zstd/lib/dictBuilder/zdict.h +56 -28
- data/contrib/zstd/lib/legacy/zstd_legacy.h +8 -4
- data/contrib/zstd/lib/legacy/zstd_v01.c +110 -110
- data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v02.c +23 -13
- data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v03.c +23 -13
- data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v04.c +30 -17
- data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v05.c +113 -102
- data/contrib/zstd/lib/legacy/zstd_v05.h +2 -2
- data/contrib/zstd/lib/legacy/zstd_v06.c +20 -18
- data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v07.c +25 -19
- data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
- data/contrib/zstd/lib/libzstd.pc.in +3 -2
- data/contrib/zstd/lib/zstd.h +265 -88
- data/ext/extzstd.h +1 -1
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +1 -3
- data/ext/zstd_compress.c +3 -3
- data/ext/zstd_decompress.c +1 -5
- data/ext/zstd_dictbuilder.c +2 -3
- data/ext/zstd_dictbuilder_fastcover.c +1 -3
- data/ext/zstd_legacy_v01.c +2 -0
- data/ext/zstd_legacy_v02.c +2 -0
- data/ext/zstd_legacy_v03.c +2 -0
- data/ext/zstd_legacy_v04.c +2 -0
- data/ext/zstd_legacy_v05.c +2 -0
- data/ext/zstd_legacy_v06.c +2 -0
- data/ext/zstd_legacy_v07.c +2 -0
- data/lib/extzstd.rb +18 -10
- data/lib/extzstd/version.rb +1 -1
- metadata +15 -6
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -61,9 +61,57 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
|
|
|
61
61
|
const void* samplesBuffer,
|
|
62
62
|
const size_t* samplesSizes, unsigned nbSamples);
|
|
63
63
|
|
|
64
|
+
typedef struct {
|
|
65
|
+
int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
|
|
66
|
+
unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
|
67
|
+
unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */
|
|
68
|
+
} ZDICT_params_t;
|
|
69
|
+
|
|
70
|
+
/*! ZDICT_finalizeDictionary():
|
|
71
|
+
* Given a custom content as a basis for dictionary, and a set of samples,
|
|
72
|
+
* finalize dictionary by adding headers and statistics according to the zstd
|
|
73
|
+
* dictionary format.
|
|
74
|
+
*
|
|
75
|
+
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
|
76
|
+
* supplied with an array of sizes `samplesSizes`, providing the size of each
|
|
77
|
+
* sample in order. The samples are used to construct the statistics, so they
|
|
78
|
+
* should be representative of what you will compress with this dictionary.
|
|
79
|
+
*
|
|
80
|
+
* The compression level can be set in `parameters`. You should pass the
|
|
81
|
+
* compression level you expect to use in production. The statistics for each
|
|
82
|
+
* compression level differ, so tuning the dictionary for the compression level
|
|
83
|
+
* can help quite a bit.
|
|
84
|
+
*
|
|
85
|
+
* You can set an explicit dictionary ID in `parameters`, or allow us to pick
|
|
86
|
+
* a random dictionary ID for you, but we can't guarantee no collisions.
|
|
87
|
+
*
|
|
88
|
+
* The dstDictBuffer and the dictContent may overlap, and the content will be
|
|
89
|
+
* appended to the end of the header. If the header + the content doesn't fit in
|
|
90
|
+
* maxDictSize the beginning of the content is truncated to make room, since it
|
|
91
|
+
* is presumed that the most profitable content is at the end of the dictionary,
|
|
92
|
+
* since that is the cheapest to reference.
|
|
93
|
+
*
|
|
94
|
+
* `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
|
|
95
|
+
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
|
|
96
|
+
*
|
|
97
|
+
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
|
|
98
|
+
* or an error code, which can be tested by ZDICT_isError().
|
|
99
|
+
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if
|
|
100
|
+
* instructed to, using notificationLevel>0.
|
|
101
|
+
* NOTE: This function currently may fail in several edge cases including:
|
|
102
|
+
* * Not enough samples
|
|
103
|
+
* * Samples are uncompressible
|
|
104
|
+
* * Samples are all exactly the same
|
|
105
|
+
*/
|
|
106
|
+
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
|
|
107
|
+
const void* dictContent, size_t dictContentSize,
|
|
108
|
+
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
|
109
|
+
ZDICT_params_t parameters);
|
|
110
|
+
|
|
64
111
|
|
|
65
112
|
/*====== Helper functions ======*/
|
|
66
113
|
ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
|
|
114
|
+
ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
|
|
67
115
|
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
|
|
68
116
|
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
|
69
117
|
|
|
@@ -78,11 +126,8 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
|
|
78
126
|
* Use them only in association with static linking.
|
|
79
127
|
* ==================================================================================== */
|
|
80
128
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
unsigned notificationLevel; /* Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
|
84
|
-
unsigned dictID; /* force dictID value; 0 means auto mode (32-bits random value) */
|
|
85
|
-
} ZDICT_params_t;
|
|
129
|
+
#define ZDICT_CONTENTSIZE_MIN 128
|
|
130
|
+
#define ZDICT_DICTSIZE_MIN 256
|
|
86
131
|
|
|
87
132
|
/*! ZDICT_cover_params_t:
|
|
88
133
|
* k and d are the only required parameters.
|
|
@@ -94,6 +139,8 @@ typedef struct {
|
|
|
94
139
|
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
|
|
95
140
|
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
|
96
141
|
double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
|
|
142
|
+
unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
|
|
143
|
+
unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
|
|
97
144
|
ZDICT_params_t zParams;
|
|
98
145
|
} ZDICT_cover_params_t;
|
|
99
146
|
|
|
@@ -105,6 +152,9 @@ typedef struct {
|
|
|
105
152
|
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
|
106
153
|
double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
|
|
107
154
|
unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
|
|
155
|
+
unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
|
|
156
|
+
unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
|
|
157
|
+
|
|
108
158
|
ZDICT_params_t zParams;
|
|
109
159
|
} ZDICT_fastCover_params_t;
|
|
110
160
|
|
|
@@ -193,28 +243,6 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
|
|
193
243
|
const size_t* samplesSizes, unsigned nbSamples,
|
|
194
244
|
ZDICT_fastCover_params_t* parameters);
|
|
195
245
|
|
|
196
|
-
/*! ZDICT_finalizeDictionary():
|
|
197
|
-
* Given a custom content as a basis for dictionary, and a set of samples,
|
|
198
|
-
* finalize dictionary by adding headers and statistics.
|
|
199
|
-
*
|
|
200
|
-
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
|
201
|
-
* supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
|
|
202
|
-
*
|
|
203
|
-
* dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes.
|
|
204
|
-
* maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes.
|
|
205
|
-
*
|
|
206
|
-
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
|
|
207
|
-
* or an error code, which can be tested by ZDICT_isError().
|
|
208
|
-
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
|
|
209
|
-
* Note 2: dictBuffer and dictContent can overlap
|
|
210
|
-
*/
|
|
211
|
-
#define ZDICT_CONTENTSIZE_MIN 128
|
|
212
|
-
#define ZDICT_DICTSIZE_MIN 256
|
|
213
|
-
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
|
214
|
-
const void* dictContent, size_t dictContentSize,
|
|
215
|
-
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
|
216
|
-
ZDICT_params_t parameters);
|
|
217
|
-
|
|
218
246
|
typedef struct {
|
|
219
247
|
unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
|
|
220
248
|
ZDICT_params_t zParams;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -18,9 +18,9 @@ extern "C" {
|
|
|
18
18
|
/* *************************************
|
|
19
19
|
* Includes
|
|
20
20
|
***************************************/
|
|
21
|
-
#include "mem.h" /* MEM_STATIC */
|
|
22
|
-
#include "error_private.h" /* ERROR */
|
|
23
|
-
#include "zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
|
|
21
|
+
#include "../common/mem.h" /* MEM_STATIC */
|
|
22
|
+
#include "../common/error_private.h" /* ERROR */
|
|
23
|
+
#include "../common/zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
|
|
24
24
|
|
|
25
25
|
#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
|
|
26
26
|
# undef ZSTD_LEGACY_SUPPORT
|
|
@@ -238,6 +238,10 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size
|
|
|
238
238
|
frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
|
|
239
239
|
break;
|
|
240
240
|
}
|
|
241
|
+
if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) {
|
|
242
|
+
frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
|
|
243
|
+
frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
|
|
244
|
+
}
|
|
241
245
|
return frameSizeInfo;
|
|
242
246
|
}
|
|
243
247
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
******************************************/
|
|
15
15
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
|
16
16
|
#include "zstd_v01.h"
|
|
17
|
-
#include "error_private.h"
|
|
17
|
+
#include "../common/error_private.h"
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
/******************************************
|
|
@@ -257,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
|
|
|
257
257
|
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
|
258
258
|
}
|
|
259
259
|
|
|
260
|
-
#endif
|
|
260
|
+
#endif /* FSE_FORCE_MEMORY_ACCESS */
|
|
261
261
|
|
|
262
262
|
static U16 FSE_readLE16(const void* memPtr)
|
|
263
263
|
{
|
|
@@ -346,7 +346,7 @@ FORCE_INLINE unsigned FSE_highbit32 (U32 val)
|
|
|
346
346
|
_BitScanReverse ( &r, val );
|
|
347
347
|
return (unsigned) r;
|
|
348
348
|
# elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */
|
|
349
|
-
return
|
|
349
|
+
return __builtin_clz (val) ^ 31;
|
|
350
350
|
# else /* Software version */
|
|
351
351
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
|
352
352
|
U32 v = val;
|
|
@@ -1073,99 +1073,102 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
|
1073
1073
|
const void* cSrc, size_t cSrcSize,
|
|
1074
1074
|
const U16* DTable)
|
|
1075
1075
|
{
|
|
1076
|
-
|
|
1077
|
-
BYTE* op = ostart;
|
|
1078
|
-
BYTE* const omax = op + maxDstSize;
|
|
1079
|
-
BYTE* const olimit = omax-15;
|
|
1080
|
-
|
|
1081
|
-
const void* ptr = DTable;
|
|
1082
|
-
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
|
1083
|
-
const U32 dtLog = DTable[0];
|
|
1084
|
-
size_t errorCode;
|
|
1085
|
-
U32 reloadStatus;
|
|
1086
|
-
|
|
1087
|
-
/* Init */
|
|
1088
|
-
|
|
1089
|
-
const U16* jumpTable = (const U16*)cSrc;
|
|
1090
|
-
const size_t length1 = FSE_readLE16(jumpTable);
|
|
1091
|
-
const size_t length2 = FSE_readLE16(jumpTable+1);
|
|
1092
|
-
const size_t length3 = FSE_readLE16(jumpTable+2);
|
|
1093
|
-
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
|
|
1094
|
-
const char* const start1 = (const char*)(cSrc) + 6;
|
|
1095
|
-
const char* const start2 = start1 + length1;
|
|
1096
|
-
const char* const start3 = start2 + length2;
|
|
1097
|
-
const char* const start4 = start3 + length3;
|
|
1098
|
-
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
|
|
1099
|
-
|
|
1100
|
-
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
|
1101
|
-
|
|
1102
|
-
errorCode = FSE_initDStream(&bitD1, start1, length1);
|
|
1103
|
-
if (FSE_isError(errorCode)) return errorCode;
|
|
1104
|
-
errorCode = FSE_initDStream(&bitD2, start2, length2);
|
|
1105
|
-
if (FSE_isError(errorCode)) return errorCode;
|
|
1106
|
-
errorCode = FSE_initDStream(&bitD3, start3, length3);
|
|
1107
|
-
if (FSE_isError(errorCode)) return errorCode;
|
|
1108
|
-
errorCode = FSE_initDStream(&bitD4, start4, length4);
|
|
1109
|
-
if (FSE_isError(errorCode)) return errorCode;
|
|
1110
|
-
|
|
1111
|
-
reloadStatus=FSE_reloadDStream(&bitD2);
|
|
1112
|
-
|
|
1113
|
-
/* 16 symbols per loop */
|
|
1114
|
-
for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
|
|
1115
|
-
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
|
|
1076
|
+
if (cSrcSize < 6) return (size_t)-FSE_ERROR_srcSize_wrong;
|
|
1116
1077
|
{
|
|
1117
|
-
|
|
1118
|
-
op
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1078
|
+
BYTE* const ostart = (BYTE*) dst;
|
|
1079
|
+
BYTE* op = ostart;
|
|
1080
|
+
BYTE* const omax = op + maxDstSize;
|
|
1081
|
+
BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
|
|
1082
|
+
|
|
1083
|
+
const void* ptr = DTable;
|
|
1084
|
+
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
|
1085
|
+
const U32 dtLog = DTable[0];
|
|
1086
|
+
size_t errorCode;
|
|
1087
|
+
U32 reloadStatus;
|
|
1088
|
+
|
|
1089
|
+
/* Init */
|
|
1090
|
+
|
|
1091
|
+
const U16* jumpTable = (const U16*)cSrc;
|
|
1092
|
+
const size_t length1 = FSE_readLE16(jumpTable);
|
|
1093
|
+
const size_t length2 = FSE_readLE16(jumpTable+1);
|
|
1094
|
+
const size_t length3 = FSE_readLE16(jumpTable+2);
|
|
1095
|
+
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
|
|
1096
|
+
const char* const start1 = (const char*)(cSrc) + 6;
|
|
1097
|
+
const char* const start2 = start1 + length1;
|
|
1098
|
+
const char* const start3 = start2 + length2;
|
|
1099
|
+
const char* const start4 = start3 + length3;
|
|
1100
|
+
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
|
|
1101
|
+
|
|
1102
|
+
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
|
1103
|
+
|
|
1104
|
+
errorCode = FSE_initDStream(&bitD1, start1, length1);
|
|
1105
|
+
if (FSE_isError(errorCode)) return errorCode;
|
|
1106
|
+
errorCode = FSE_initDStream(&bitD2, start2, length2);
|
|
1107
|
+
if (FSE_isError(errorCode)) return errorCode;
|
|
1108
|
+
errorCode = FSE_initDStream(&bitD3, start3, length3);
|
|
1109
|
+
if (FSE_isError(errorCode)) return errorCode;
|
|
1110
|
+
errorCode = FSE_initDStream(&bitD4, start4, length4);
|
|
1111
|
+
if (FSE_isError(errorCode)) return errorCode;
|
|
1112
|
+
|
|
1113
|
+
reloadStatus=FSE_reloadDStream(&bitD2);
|
|
1114
|
+
|
|
1115
|
+
/* 16 symbols per loop */
|
|
1116
|
+
for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
|
|
1117
|
+
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
|
|
1118
|
+
{
|
|
1119
|
+
#define HUF_DECODE_SYMBOL_0(n, Dstream) \
|
|
1120
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
|
|
1121
|
+
|
|
1122
|
+
#define HUF_DECODE_SYMBOL_1(n, Dstream) \
|
|
1123
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
|
|
1124
|
+
if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
|
|
1125
|
+
|
|
1126
|
+
#define HUF_DECODE_SYMBOL_2(n, Dstream) \
|
|
1127
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
|
|
1128
|
+
if (FSE_32bits()) FSE_reloadDStream(&Dstream)
|
|
1129
|
+
|
|
1130
|
+
HUF_DECODE_SYMBOL_1( 0, bitD1);
|
|
1131
|
+
HUF_DECODE_SYMBOL_1( 1, bitD2);
|
|
1132
|
+
HUF_DECODE_SYMBOL_1( 2, bitD3);
|
|
1133
|
+
HUF_DECODE_SYMBOL_1( 3, bitD4);
|
|
1134
|
+
HUF_DECODE_SYMBOL_2( 4, bitD1);
|
|
1135
|
+
HUF_DECODE_SYMBOL_2( 5, bitD2);
|
|
1136
|
+
HUF_DECODE_SYMBOL_2( 6, bitD3);
|
|
1137
|
+
HUF_DECODE_SYMBOL_2( 7, bitD4);
|
|
1138
|
+
HUF_DECODE_SYMBOL_1( 8, bitD1);
|
|
1139
|
+
HUF_DECODE_SYMBOL_1( 9, bitD2);
|
|
1140
|
+
HUF_DECODE_SYMBOL_1(10, bitD3);
|
|
1141
|
+
HUF_DECODE_SYMBOL_1(11, bitD4);
|
|
1142
|
+
HUF_DECODE_SYMBOL_0(12, bitD1);
|
|
1143
|
+
HUF_DECODE_SYMBOL_0(13, bitD2);
|
|
1144
|
+
HUF_DECODE_SYMBOL_0(14, bitD3);
|
|
1145
|
+
HUF_DECODE_SYMBOL_0(15, bitD4);
|
|
1146
|
+
}
|
|
1145
1147
|
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
+
if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
|
|
1149
|
+
return (size_t)-FSE_ERROR_corruptionDetected;
|
|
1148
1150
|
|
|
1149
|
-
|
|
1150
|
-
{
|
|
1151
|
-
// bitTail = bitD1; // *much* slower : -20% !??!
|
|
1152
|
-
FSE_DStream_t bitTail;
|
|
1153
|
-
bitTail.ptr = bitD1.ptr;
|
|
1154
|
-
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
|
1155
|
-
bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
|
|
1156
|
-
bitTail.start = start1;
|
|
1157
|
-
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
|
1151
|
+
/* tail */
|
|
1158
1152
|
{
|
|
1159
|
-
|
|
1160
|
-
|
|
1153
|
+
/* bitTail = bitD1; */ /* *much* slower : -20% !??! */
|
|
1154
|
+
FSE_DStream_t bitTail;
|
|
1155
|
+
bitTail.ptr = bitD1.ptr;
|
|
1156
|
+
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
|
1157
|
+
bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
|
|
1158
|
+
bitTail.start = start1;
|
|
1159
|
+
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
|
1160
|
+
{
|
|
1161
|
+
HUF_DECODE_SYMBOL_0(0, bitTail);
|
|
1162
|
+
}
|
|
1161
1163
|
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1164
|
+
if (FSE_endOfDStream(&bitTail))
|
|
1165
|
+
return op-ostart;
|
|
1166
|
+
}
|
|
1165
1167
|
|
|
1166
|
-
|
|
1168
|
+
if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
|
|
1167
1169
|
|
|
1168
|
-
|
|
1170
|
+
return (size_t)-FSE_ERROR_corruptionDetected;
|
|
1171
|
+
}
|
|
1169
1172
|
}
|
|
1170
1173
|
|
|
1171
1174
|
|
|
@@ -1355,8 +1358,6 @@ static unsigned ZSTD_isLittleEndian(void)
|
|
|
1355
1358
|
|
|
1356
1359
|
static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); return r; }
|
|
1357
1360
|
|
|
1358
|
-
static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; }
|
|
1359
|
-
|
|
1360
1361
|
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
|
1361
1362
|
|
|
1362
1363
|
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
|
@@ -1381,16 +1382,9 @@ static U16 ZSTD_readLE16(const void* memPtr)
|
|
|
1381
1382
|
}
|
|
1382
1383
|
}
|
|
1383
1384
|
|
|
1384
|
-
|
|
1385
|
-
static U32 ZSTD_readLE32(const void* memPtr)
|
|
1385
|
+
static U32 ZSTD_readLE24(const void* memPtr)
|
|
1386
1386
|
{
|
|
1387
|
-
|
|
1388
|
-
return ZSTD_read32(memPtr);
|
|
1389
|
-
else
|
|
1390
|
-
{
|
|
1391
|
-
const BYTE* p = (const BYTE*)memPtr;
|
|
1392
|
-
return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
|
|
1393
|
-
}
|
|
1387
|
+
return ZSTD_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
|
1394
1388
|
}
|
|
1395
1389
|
|
|
1396
1390
|
static U32 ZSTD_readBE32(const void* memPtr)
|
|
@@ -1489,7 +1483,9 @@ static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProper
|
|
|
1489
1483
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
|
1490
1484
|
{
|
|
1491
1485
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
|
1492
|
-
|
|
1486
|
+
if (srcSize > 0) {
|
|
1487
|
+
memcpy(dst, src, srcSize);
|
|
1488
|
+
}
|
|
1493
1489
|
return srcSize;
|
|
1494
1490
|
}
|
|
1495
1491
|
|
|
@@ -1508,7 +1504,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
|
|
1508
1504
|
if (srcSize <= 3) return ERROR(corruption_detected);
|
|
1509
1505
|
|
|
1510
1506
|
litSize = ip[1] + (ip[0]<<8);
|
|
1511
|
-
litSize += ((ip[-3] >> 3) & 7) << 16;
|
|
1507
|
+
litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
|
|
1512
1508
|
op = oend - litSize;
|
|
1513
1509
|
|
|
1514
1510
|
(void)ctx;
|
|
@@ -1547,7 +1543,9 @@ static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
|
1547
1543
|
size_t rleSize = litbp.origSize;
|
|
1548
1544
|
if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
|
|
1549
1545
|
if (!srcSize) return ERROR(srcSize_wrong);
|
|
1550
|
-
|
|
1546
|
+
if (rleSize > 0) {
|
|
1547
|
+
memset(oend - rleSize, *ip, rleSize);
|
|
1548
|
+
}
|
|
1551
1549
|
*litStart = oend - rleSize;
|
|
1552
1550
|
*litSize = rleSize;
|
|
1553
1551
|
ip++;
|
|
@@ -1704,13 +1702,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
1704
1702
|
seqState->prevOffset = seq->offset;
|
|
1705
1703
|
if (litLength == MaxLL)
|
|
1706
1704
|
{
|
|
1707
|
-
U32 add = dumps<de ? *dumps++ : 0;
|
|
1705
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
|
1708
1706
|
if (add < 255) litLength += add;
|
|
1709
1707
|
else
|
|
1710
1708
|
{
|
|
1711
1709
|
if (dumps<=(de-3))
|
|
1712
1710
|
{
|
|
1713
|
-
litLength =
|
|
1711
|
+
litLength = ZSTD_readLE24(dumps);
|
|
1714
1712
|
dumps += 3;
|
|
1715
1713
|
}
|
|
1716
1714
|
}
|
|
@@ -1732,13 +1730,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
|
1732
1730
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
|
1733
1731
|
if (matchLength == MaxML)
|
|
1734
1732
|
{
|
|
1735
|
-
U32 add = dumps<de ? *dumps++ : 0;
|
|
1733
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
|
1736
1734
|
if (add < 255) matchLength += add;
|
|
1737
1735
|
else
|
|
1738
1736
|
{
|
|
1739
1737
|
if (dumps<=(de-3))
|
|
1740
1738
|
{
|
|
1741
|
-
matchLength =
|
|
1739
|
+
matchLength = ZSTD_readLE24(dumps);
|
|
1742
1740
|
dumps += 3;
|
|
1743
1741
|
}
|
|
1744
1742
|
}
|
|
@@ -1907,8 +1905,10 @@ static size_t ZSTD_decompressSequences(
|
|
|
1907
1905
|
{
|
|
1908
1906
|
size_t lastLLSize = litEnd - litPtr;
|
|
1909
1907
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
|
1910
|
-
if (
|
|
1911
|
-
|
|
1908
|
+
if (lastLLSize > 0) {
|
|
1909
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
|
1910
|
+
op += lastLLSize;
|
|
1911
|
+
}
|
|
1912
1912
|
}
|
|
1913
1913
|
}
|
|
1914
1914
|
|