extzstd 0.3 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/HISTORY.ja.md +8 -0
- data/README.md +1 -1
- data/contrib/zstd/CHANGELOG +94 -0
- data/contrib/zstd/CONTRIBUTING.md +351 -1
- data/contrib/zstd/Makefile +32 -10
- data/contrib/zstd/README.md +33 -10
- data/contrib/zstd/TESTING.md +2 -2
- data/contrib/zstd/appveyor.yml +42 -4
- data/contrib/zstd/lib/Makefile +128 -60
- data/contrib/zstd/lib/README.md +47 -16
- data/contrib/zstd/lib/common/bitstream.h +38 -39
- data/contrib/zstd/lib/common/compiler.h +40 -5
- data/contrib/zstd/lib/common/cpu.h +1 -1
- data/contrib/zstd/lib/common/debug.c +11 -31
- data/contrib/zstd/lib/common/debug.h +11 -31
- data/contrib/zstd/lib/common/entropy_common.c +13 -33
- data/contrib/zstd/lib/common/error_private.c +2 -1
- data/contrib/zstd/lib/common/error_private.h +6 -2
- data/contrib/zstd/lib/common/fse.h +12 -32
- data/contrib/zstd/lib/common/fse_decompress.c +12 -35
- data/contrib/zstd/lib/common/huf.h +15 -33
- data/contrib/zstd/lib/common/mem.h +75 -2
- data/contrib/zstd/lib/common/pool.c +8 -4
- data/contrib/zstd/lib/common/pool.h +2 -2
- data/contrib/zstd/lib/common/threading.c +50 -4
- data/contrib/zstd/lib/common/threading.h +36 -4
- data/contrib/zstd/lib/common/xxhash.c +23 -35
- data/contrib/zstd/lib/common/xxhash.h +11 -31
- data/contrib/zstd/lib/common/zstd_common.c +1 -1
- data/contrib/zstd/lib/common/zstd_errors.h +2 -1
- data/contrib/zstd/lib/common/zstd_internal.h +154 -26
- data/contrib/zstd/lib/compress/fse_compress.c +17 -40
- data/contrib/zstd/lib/compress/hist.c +15 -35
- data/contrib/zstd/lib/compress/hist.h +12 -32
- data/contrib/zstd/lib/compress/huf_compress.c +92 -92
- data/contrib/zstd/lib/compress/zstd_compress.c +1191 -1330
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +317 -55
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +419 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +845 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +525 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +65 -43
- data/contrib/zstd/lib/compress/zstd_double_fast.h +2 -2
- data/contrib/zstd/lib/compress/zstd_fast.c +92 -66
- data/contrib/zstd/lib/compress/zstd_fast.h +2 -2
- data/contrib/zstd/lib/compress/zstd_lazy.c +74 -42
- data/contrib/zstd/lib/compress/zstd_lazy.h +1 -1
- data/contrib/zstd/lib/compress/zstd_ldm.c +32 -10
- data/contrib/zstd/lib/compress/zstd_ldm.h +7 -2
- data/contrib/zstd/lib/compress/zstd_opt.c +81 -114
- data/contrib/zstd/lib/compress/zstd_opt.h +1 -1
- data/contrib/zstd/lib/compress/zstdmt_compress.c +95 -51
- data/contrib/zstd/lib/compress/zstdmt_compress.h +3 -2
- data/contrib/zstd/lib/decompress/huf_decompress.c +76 -60
- data/contrib/zstd/lib/decompress/zstd_ddict.c +12 -8
- data/contrib/zstd/lib/decompress/zstd_ddict.h +2 -2
- data/contrib/zstd/lib/decompress/zstd_decompress.c +292 -172
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +459 -338
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +3 -3
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +18 -4
- data/contrib/zstd/lib/deprecated/zbuff.h +9 -8
- data/contrib/zstd/lib/deprecated/zbuff_common.c +2 -2
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +1 -1
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +1 -1
- data/contrib/zstd/lib/dictBuilder/cover.c +164 -54
- data/contrib/zstd/lib/dictBuilder/cover.h +52 -7
- data/contrib/zstd/lib/dictBuilder/fastcover.c +60 -43
- data/contrib/zstd/lib/dictBuilder/zdict.c +43 -19
- data/contrib/zstd/lib/dictBuilder/zdict.h +56 -28
- data/contrib/zstd/lib/legacy/zstd_legacy.h +8 -4
- data/contrib/zstd/lib/legacy/zstd_v01.c +110 -110
- data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v02.c +23 -13
- data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v03.c +23 -13
- data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v04.c +30 -17
- data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v05.c +113 -102
- data/contrib/zstd/lib/legacy/zstd_v05.h +2 -2
- data/contrib/zstd/lib/legacy/zstd_v06.c +20 -18
- data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v07.c +25 -19
- data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
- data/contrib/zstd/lib/libzstd.pc.in +3 -2
- data/contrib/zstd/lib/zstd.h +265 -88
- data/ext/extzstd.h +1 -1
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +1 -3
- data/ext/zstd_compress.c +3 -3
- data/ext/zstd_decompress.c +1 -5
- data/ext/zstd_dictbuilder.c +2 -3
- data/ext/zstd_dictbuilder_fastcover.c +1 -3
- data/ext/zstd_legacy_v01.c +2 -0
- data/ext/zstd_legacy_v02.c +2 -0
- data/ext/zstd_legacy_v03.c +2 -0
- data/ext/zstd_legacy_v04.c +2 -0
- data/ext/zstd_legacy_v05.c +2 -0
- data/ext/zstd_legacy_v06.c +2 -0
- data/ext/zstd_legacy_v07.c +2 -0
- data/lib/extzstd.rb +18 -10
- data/lib/extzstd/version.rb +1 -1
- metadata +15 -6
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -61,9 +61,57 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
|
|
61
61
|
const void* samplesBuffer,
|
62
62
|
const size_t* samplesSizes, unsigned nbSamples);
|
63
63
|
|
64
|
+
typedef struct {
|
65
|
+
int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
|
66
|
+
unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
67
|
+
unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */
|
68
|
+
} ZDICT_params_t;
|
69
|
+
|
70
|
+
/*! ZDICT_finalizeDictionary():
|
71
|
+
* Given a custom content as a basis for dictionary, and a set of samples,
|
72
|
+
* finalize dictionary by adding headers and statistics according to the zstd
|
73
|
+
* dictionary format.
|
74
|
+
*
|
75
|
+
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
76
|
+
* supplied with an array of sizes `samplesSizes`, providing the size of each
|
77
|
+
* sample in order. The samples are used to construct the statistics, so they
|
78
|
+
* should be representative of what you will compress with this dictionary.
|
79
|
+
*
|
80
|
+
* The compression level can be set in `parameters`. You should pass the
|
81
|
+
* compression level you expect to use in production. The statistics for each
|
82
|
+
* compression level differ, so tuning the dictionary for the compression level
|
83
|
+
* can help quite a bit.
|
84
|
+
*
|
85
|
+
* You can set an explicit dictionary ID in `parameters`, or allow us to pick
|
86
|
+
* a random dictionary ID for you, but we can't guarantee no collisions.
|
87
|
+
*
|
88
|
+
* The dstDictBuffer and the dictContent may overlap, and the content will be
|
89
|
+
* appended to the end of the header. If the header + the content doesn't fit in
|
90
|
+
* maxDictSize the beginning of the content is truncated to make room, since it
|
91
|
+
* is presumed that the most profitable content is at the end of the dictionary,
|
92
|
+
* since that is the cheapest to reference.
|
93
|
+
*
|
94
|
+
* `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
|
95
|
+
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
|
96
|
+
*
|
97
|
+
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
|
98
|
+
* or an error code, which can be tested by ZDICT_isError().
|
99
|
+
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if
|
100
|
+
* instructed to, using notificationLevel>0.
|
101
|
+
* NOTE: This function currently may fail in several edge cases including:
|
102
|
+
* * Not enough samples
|
103
|
+
* * Samples are uncompressible
|
104
|
+
* * Samples are all exactly the same
|
105
|
+
*/
|
106
|
+
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
|
107
|
+
const void* dictContent, size_t dictContentSize,
|
108
|
+
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
109
|
+
ZDICT_params_t parameters);
|
110
|
+
|
64
111
|
|
65
112
|
/*====== Helper functions ======*/
|
66
113
|
ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
|
114
|
+
ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
|
67
115
|
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
|
68
116
|
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
69
117
|
|
@@ -78,11 +126,8 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
|
78
126
|
* Use them only in association with static linking.
|
79
127
|
* ==================================================================================== */
|
80
128
|
|
81
|
-
|
82
|
-
|
83
|
-
unsigned notificationLevel; /* Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
84
|
-
unsigned dictID; /* force dictID value; 0 means auto mode (32-bits random value) */
|
85
|
-
} ZDICT_params_t;
|
129
|
+
#define ZDICT_CONTENTSIZE_MIN 128
|
130
|
+
#define ZDICT_DICTSIZE_MIN 256
|
86
131
|
|
87
132
|
/*! ZDICT_cover_params_t:
|
88
133
|
* k and d are the only required parameters.
|
@@ -94,6 +139,8 @@ typedef struct {
|
|
94
139
|
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
|
95
140
|
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
96
141
|
double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
|
142
|
+
unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
|
143
|
+
unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
|
97
144
|
ZDICT_params_t zParams;
|
98
145
|
} ZDICT_cover_params_t;
|
99
146
|
|
@@ -105,6 +152,9 @@ typedef struct {
|
|
105
152
|
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
106
153
|
double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
|
107
154
|
unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
|
155
|
+
unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
|
156
|
+
unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
|
157
|
+
|
108
158
|
ZDICT_params_t zParams;
|
109
159
|
} ZDICT_fastCover_params_t;
|
110
160
|
|
@@ -193,28 +243,6 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
|
193
243
|
const size_t* samplesSizes, unsigned nbSamples,
|
194
244
|
ZDICT_fastCover_params_t* parameters);
|
195
245
|
|
196
|
-
/*! ZDICT_finalizeDictionary():
|
197
|
-
* Given a custom content as a basis for dictionary, and a set of samples,
|
198
|
-
* finalize dictionary by adding headers and statistics.
|
199
|
-
*
|
200
|
-
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
201
|
-
* supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
|
202
|
-
*
|
203
|
-
* dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes.
|
204
|
-
* maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes.
|
205
|
-
*
|
206
|
-
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
|
207
|
-
* or an error code, which can be tested by ZDICT_isError().
|
208
|
-
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
|
209
|
-
* Note 2: dictBuffer and dictContent can overlap
|
210
|
-
*/
|
211
|
-
#define ZDICT_CONTENTSIZE_MIN 128
|
212
|
-
#define ZDICT_DICTSIZE_MIN 256
|
213
|
-
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
214
|
-
const void* dictContent, size_t dictContentSize,
|
215
|
-
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
216
|
-
ZDICT_params_t parameters);
|
217
|
-
|
218
246
|
typedef struct {
|
219
247
|
unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
|
220
248
|
ZDICT_params_t zParams;
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -18,9 +18,9 @@ extern "C" {
|
|
18
18
|
/* *************************************
|
19
19
|
* Includes
|
20
20
|
***************************************/
|
21
|
-
#include "mem.h" /* MEM_STATIC */
|
22
|
-
#include "error_private.h" /* ERROR */
|
23
|
-
#include "zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
|
21
|
+
#include "../common/mem.h" /* MEM_STATIC */
|
22
|
+
#include "../common/error_private.h" /* ERROR */
|
23
|
+
#include "../common/zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
|
24
24
|
|
25
25
|
#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
|
26
26
|
# undef ZSTD_LEGACY_SUPPORT
|
@@ -238,6 +238,10 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size
|
|
238
238
|
frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
|
239
239
|
break;
|
240
240
|
}
|
241
|
+
if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) {
|
242
|
+
frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
|
243
|
+
frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
|
244
|
+
}
|
241
245
|
return frameSizeInfo;
|
242
246
|
}
|
243
247
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -14,7 +14,7 @@
|
|
14
14
|
******************************************/
|
15
15
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
16
16
|
#include "zstd_v01.h"
|
17
|
-
#include "error_private.h"
|
17
|
+
#include "../common/error_private.h"
|
18
18
|
|
19
19
|
|
20
20
|
/******************************************
|
@@ -257,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
|
|
257
257
|
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
258
258
|
}
|
259
259
|
|
260
|
-
#endif
|
260
|
+
#endif /* FSE_FORCE_MEMORY_ACCESS */
|
261
261
|
|
262
262
|
static U16 FSE_readLE16(const void* memPtr)
|
263
263
|
{
|
@@ -346,7 +346,7 @@ FORCE_INLINE unsigned FSE_highbit32 (U32 val)
|
|
346
346
|
_BitScanReverse ( &r, val );
|
347
347
|
return (unsigned) r;
|
348
348
|
# elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */
|
349
|
-
return
|
349
|
+
return __builtin_clz (val) ^ 31;
|
350
350
|
# else /* Software version */
|
351
351
|
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
352
352
|
U32 v = val;
|
@@ -1073,99 +1073,102 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
1073
1073
|
const void* cSrc, size_t cSrcSize,
|
1074
1074
|
const U16* DTable)
|
1075
1075
|
{
|
1076
|
-
|
1077
|
-
BYTE* op = ostart;
|
1078
|
-
BYTE* const omax = op + maxDstSize;
|
1079
|
-
BYTE* const olimit = omax-15;
|
1080
|
-
|
1081
|
-
const void* ptr = DTable;
|
1082
|
-
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
1083
|
-
const U32 dtLog = DTable[0];
|
1084
|
-
size_t errorCode;
|
1085
|
-
U32 reloadStatus;
|
1086
|
-
|
1087
|
-
/* Init */
|
1088
|
-
|
1089
|
-
const U16* jumpTable = (const U16*)cSrc;
|
1090
|
-
const size_t length1 = FSE_readLE16(jumpTable);
|
1091
|
-
const size_t length2 = FSE_readLE16(jumpTable+1);
|
1092
|
-
const size_t length3 = FSE_readLE16(jumpTable+2);
|
1093
|
-
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
|
1094
|
-
const char* const start1 = (const char*)(cSrc) + 6;
|
1095
|
-
const char* const start2 = start1 + length1;
|
1096
|
-
const char* const start3 = start2 + length2;
|
1097
|
-
const char* const start4 = start3 + length3;
|
1098
|
-
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
|
1099
|
-
|
1100
|
-
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
1101
|
-
|
1102
|
-
errorCode = FSE_initDStream(&bitD1, start1, length1);
|
1103
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1104
|
-
errorCode = FSE_initDStream(&bitD2, start2, length2);
|
1105
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1106
|
-
errorCode = FSE_initDStream(&bitD3, start3, length3);
|
1107
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1108
|
-
errorCode = FSE_initDStream(&bitD4, start4, length4);
|
1109
|
-
if (FSE_isError(errorCode)) return errorCode;
|
1110
|
-
|
1111
|
-
reloadStatus=FSE_reloadDStream(&bitD2);
|
1112
|
-
|
1113
|
-
/* 16 symbols per loop */
|
1114
|
-
for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
|
1115
|
-
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
|
1076
|
+
if (cSrcSize < 6) return (size_t)-FSE_ERROR_srcSize_wrong;
|
1116
1077
|
{
|
1117
|
-
|
1118
|
-
op
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
1126
|
-
|
1127
|
-
|
1128
|
-
|
1129
|
-
|
1130
|
-
|
1131
|
-
|
1132
|
-
|
1133
|
-
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1078
|
+
BYTE* const ostart = (BYTE*) dst;
|
1079
|
+
BYTE* op = ostart;
|
1080
|
+
BYTE* const omax = op + maxDstSize;
|
1081
|
+
BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
|
1082
|
+
|
1083
|
+
const void* ptr = DTable;
|
1084
|
+
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
1085
|
+
const U32 dtLog = DTable[0];
|
1086
|
+
size_t errorCode;
|
1087
|
+
U32 reloadStatus;
|
1088
|
+
|
1089
|
+
/* Init */
|
1090
|
+
|
1091
|
+
const U16* jumpTable = (const U16*)cSrc;
|
1092
|
+
const size_t length1 = FSE_readLE16(jumpTable);
|
1093
|
+
const size_t length2 = FSE_readLE16(jumpTable+1);
|
1094
|
+
const size_t length3 = FSE_readLE16(jumpTable+2);
|
1095
|
+
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
|
1096
|
+
const char* const start1 = (const char*)(cSrc) + 6;
|
1097
|
+
const char* const start2 = start1 + length1;
|
1098
|
+
const char* const start3 = start2 + length2;
|
1099
|
+
const char* const start4 = start3 + length3;
|
1100
|
+
FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
|
1101
|
+
|
1102
|
+
if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
|
1103
|
+
|
1104
|
+
errorCode = FSE_initDStream(&bitD1, start1, length1);
|
1105
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1106
|
+
errorCode = FSE_initDStream(&bitD2, start2, length2);
|
1107
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1108
|
+
errorCode = FSE_initDStream(&bitD3, start3, length3);
|
1109
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1110
|
+
errorCode = FSE_initDStream(&bitD4, start4, length4);
|
1111
|
+
if (FSE_isError(errorCode)) return errorCode;
|
1112
|
+
|
1113
|
+
reloadStatus=FSE_reloadDStream(&bitD2);
|
1114
|
+
|
1115
|
+
/* 16 symbols per loop */
|
1116
|
+
for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
|
1117
|
+
op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
|
1118
|
+
{
|
1119
|
+
#define HUF_DECODE_SYMBOL_0(n, Dstream) \
|
1120
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
|
1121
|
+
|
1122
|
+
#define HUF_DECODE_SYMBOL_1(n, Dstream) \
|
1123
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
|
1124
|
+
if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
|
1125
|
+
|
1126
|
+
#define HUF_DECODE_SYMBOL_2(n, Dstream) \
|
1127
|
+
op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
|
1128
|
+
if (FSE_32bits()) FSE_reloadDStream(&Dstream)
|
1129
|
+
|
1130
|
+
HUF_DECODE_SYMBOL_1( 0, bitD1);
|
1131
|
+
HUF_DECODE_SYMBOL_1( 1, bitD2);
|
1132
|
+
HUF_DECODE_SYMBOL_1( 2, bitD3);
|
1133
|
+
HUF_DECODE_SYMBOL_1( 3, bitD4);
|
1134
|
+
HUF_DECODE_SYMBOL_2( 4, bitD1);
|
1135
|
+
HUF_DECODE_SYMBOL_2( 5, bitD2);
|
1136
|
+
HUF_DECODE_SYMBOL_2( 6, bitD3);
|
1137
|
+
HUF_DECODE_SYMBOL_2( 7, bitD4);
|
1138
|
+
HUF_DECODE_SYMBOL_1( 8, bitD1);
|
1139
|
+
HUF_DECODE_SYMBOL_1( 9, bitD2);
|
1140
|
+
HUF_DECODE_SYMBOL_1(10, bitD3);
|
1141
|
+
HUF_DECODE_SYMBOL_1(11, bitD4);
|
1142
|
+
HUF_DECODE_SYMBOL_0(12, bitD1);
|
1143
|
+
HUF_DECODE_SYMBOL_0(13, bitD2);
|
1144
|
+
HUF_DECODE_SYMBOL_0(14, bitD3);
|
1145
|
+
HUF_DECODE_SYMBOL_0(15, bitD4);
|
1146
|
+
}
|
1145
1147
|
|
1146
|
-
|
1147
|
-
|
1148
|
+
if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
|
1149
|
+
return (size_t)-FSE_ERROR_corruptionDetected;
|
1148
1150
|
|
1149
|
-
|
1150
|
-
{
|
1151
|
-
// bitTail = bitD1; // *much* slower : -20% !??!
|
1152
|
-
FSE_DStream_t bitTail;
|
1153
|
-
bitTail.ptr = bitD1.ptr;
|
1154
|
-
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
1155
|
-
bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
|
1156
|
-
bitTail.start = start1;
|
1157
|
-
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
1151
|
+
/* tail */
|
1158
1152
|
{
|
1159
|
-
|
1160
|
-
|
1153
|
+
/* bitTail = bitD1; */ /* *much* slower : -20% !??! */
|
1154
|
+
FSE_DStream_t bitTail;
|
1155
|
+
bitTail.ptr = bitD1.ptr;
|
1156
|
+
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
1157
|
+
bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
|
1158
|
+
bitTail.start = start1;
|
1159
|
+
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
1160
|
+
{
|
1161
|
+
HUF_DECODE_SYMBOL_0(0, bitTail);
|
1162
|
+
}
|
1161
1163
|
|
1162
|
-
|
1163
|
-
|
1164
|
-
|
1164
|
+
if (FSE_endOfDStream(&bitTail))
|
1165
|
+
return op-ostart;
|
1166
|
+
}
|
1165
1167
|
|
1166
|
-
|
1168
|
+
if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
|
1167
1169
|
|
1168
|
-
|
1170
|
+
return (size_t)-FSE_ERROR_corruptionDetected;
|
1171
|
+
}
|
1169
1172
|
}
|
1170
1173
|
|
1171
1174
|
|
@@ -1355,8 +1358,6 @@ static unsigned ZSTD_isLittleEndian(void)
|
|
1355
1358
|
|
1356
1359
|
static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); return r; }
|
1357
1360
|
|
1358
|
-
static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; }
|
1359
|
-
|
1360
1361
|
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
1361
1362
|
|
1362
1363
|
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
@@ -1381,16 +1382,9 @@ static U16 ZSTD_readLE16(const void* memPtr)
|
|
1381
1382
|
}
|
1382
1383
|
}
|
1383
1384
|
|
1384
|
-
|
1385
|
-
static U32 ZSTD_readLE32(const void* memPtr)
|
1385
|
+
static U32 ZSTD_readLE24(const void* memPtr)
|
1386
1386
|
{
|
1387
|
-
|
1388
|
-
return ZSTD_read32(memPtr);
|
1389
|
-
else
|
1390
|
-
{
|
1391
|
-
const BYTE* p = (const BYTE*)memPtr;
|
1392
|
-
return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
|
1393
|
-
}
|
1387
|
+
return ZSTD_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
1394
1388
|
}
|
1395
1389
|
|
1396
1390
|
static U32 ZSTD_readBE32(const void* memPtr)
|
@@ -1489,7 +1483,9 @@ static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProper
|
|
1489
1483
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
1490
1484
|
{
|
1491
1485
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
1492
|
-
|
1486
|
+
if (srcSize > 0) {
|
1487
|
+
memcpy(dst, src, srcSize);
|
1488
|
+
}
|
1493
1489
|
return srcSize;
|
1494
1490
|
}
|
1495
1491
|
|
@@ -1508,7 +1504,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
|
1508
1504
|
if (srcSize <= 3) return ERROR(corruption_detected);
|
1509
1505
|
|
1510
1506
|
litSize = ip[1] + (ip[0]<<8);
|
1511
|
-
litSize += ((ip[-3] >> 3) & 7) << 16;
|
1507
|
+
litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
|
1512
1508
|
op = oend - litSize;
|
1513
1509
|
|
1514
1510
|
(void)ctx;
|
@@ -1547,7 +1543,9 @@ static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
1547
1543
|
size_t rleSize = litbp.origSize;
|
1548
1544
|
if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
|
1549
1545
|
if (!srcSize) return ERROR(srcSize_wrong);
|
1550
|
-
|
1546
|
+
if (rleSize > 0) {
|
1547
|
+
memset(oend - rleSize, *ip, rleSize);
|
1548
|
+
}
|
1551
1549
|
*litStart = oend - rleSize;
|
1552
1550
|
*litSize = rleSize;
|
1553
1551
|
ip++;
|
@@ -1704,13 +1702,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
1704
1702
|
seqState->prevOffset = seq->offset;
|
1705
1703
|
if (litLength == MaxLL)
|
1706
1704
|
{
|
1707
|
-
U32 add = dumps<de ? *dumps++ : 0;
|
1705
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
1708
1706
|
if (add < 255) litLength += add;
|
1709
1707
|
else
|
1710
1708
|
{
|
1711
1709
|
if (dumps<=(de-3))
|
1712
1710
|
{
|
1713
|
-
litLength =
|
1711
|
+
litLength = ZSTD_readLE24(dumps);
|
1714
1712
|
dumps += 3;
|
1715
1713
|
}
|
1716
1714
|
}
|
@@ -1732,13 +1730,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|
1732
1730
|
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
1733
1731
|
if (matchLength == MaxML)
|
1734
1732
|
{
|
1735
|
-
U32 add = dumps<de ? *dumps++ : 0;
|
1733
|
+
const U32 add = dumps<de ? *dumps++ : 0;
|
1736
1734
|
if (add < 255) matchLength += add;
|
1737
1735
|
else
|
1738
1736
|
{
|
1739
1737
|
if (dumps<=(de-3))
|
1740
1738
|
{
|
1741
|
-
matchLength =
|
1739
|
+
matchLength = ZSTD_readLE24(dumps);
|
1742
1740
|
dumps += 3;
|
1743
1741
|
}
|
1744
1742
|
}
|
@@ -1907,8 +1905,10 @@ static size_t ZSTD_decompressSequences(
|
|
1907
1905
|
{
|
1908
1906
|
size_t lastLLSize = litEnd - litPtr;
|
1909
1907
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
1910
|
-
if (
|
1911
|
-
|
1908
|
+
if (lastLLSize > 0) {
|
1909
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
1910
|
+
op += lastLLSize;
|
1911
|
+
}
|
1912
1912
|
}
|
1913
1913
|
}
|
1914
1914
|
|