extzstd 0.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +39 -0
- data/README.md +38 -56
- data/contrib/zstd/CHANGELOG +613 -0
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/CONTRIBUTING.md +406 -0
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/Makefile +420 -0
- data/contrib/zstd/README.md +179 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +292 -0
- data/contrib/zstd/lib/BUCK +234 -0
- data/contrib/zstd/lib/Makefile +451 -0
- data/contrib/zstd/lib/README.md +207 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
- data/contrib/zstd/lib/common/compiler.h +288 -0
- data/contrib/zstd/lib/common/cpu.h +213 -0
- data/contrib/zstd/lib/common/debug.c +24 -0
- data/contrib/zstd/lib/common/debug.h +107 -0
- data/contrib/zstd/lib/common/entropy_common.c +362 -0
- data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
- data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
- data/contrib/zstd/{common → lib/common}/fse.h +173 -92
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
- data/contrib/zstd/lib/common/huf.h +361 -0
- data/contrib/zstd/{common → lib/common}/mem.h +115 -59
- data/contrib/zstd/lib/common/pool.c +350 -0
- data/contrib/zstd/lib/common/pool.h +84 -0
- data/contrib/zstd/lib/common/threading.c +122 -0
- data/contrib/zstd/lib/common/threading.h +155 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
- data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
- data/contrib/zstd/lib/common/zstd_common.c +83 -0
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_errors.h +95 -0
- data/contrib/zstd/lib/common/zstd_internal.h +478 -0
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
- data/contrib/zstd/lib/compress/hist.c +181 -0
- data/contrib/zstd/lib/compress/hist.h +75 -0
- data/contrib/zstd/lib/compress/huf_compress.c +913 -0
- data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
- data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
- data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
- data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
- data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
- data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
- data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
- data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
- data/contrib/zstd/lib/libzstd.pc.in +15 -0
- data/contrib/zstd/lib/zstd.h +2391 -0
- data/ext/depend +2 -0
- data/ext/extconf.rb +15 -6
- data/ext/extzstd.c +76 -145
- data/ext/extzstd.h +80 -31
- data/ext/extzstd_stream.c +417 -142
- data/ext/libzstd_conf.h +8 -0
- data/ext/zstd_common.c +10 -7
- data/ext/zstd_compress.c +14 -5
- data/ext/zstd_decompress.c +5 -4
- data/ext/zstd_dictbuilder.c +9 -4
- data/ext/zstd_dictbuilder_fastcover.c +3 -0
- data/ext/zstd_legacy_v01.c +3 -1
- data/ext/zstd_legacy_v02.c +3 -1
- data/ext/zstd_legacy_v03.c +3 -1
- data/ext/zstd_legacy_v04.c +3 -1
- data/ext/zstd_legacy_v05.c +3 -1
- data/ext/zstd_legacy_v06.c +3 -1
- data/ext/zstd_legacy_v07.c +3 -1
- data/gemstub.rb +10 -24
- data/lib/extzstd.rb +64 -179
- data/lib/extzstd/version.rb +6 -1
- data/test/test_basic.rb +9 -6
- metadata +113 -57
- data/HISTORY.ja +0 -5
- data/contrib/zstd/common/entropy_common.c +0 -225
- data/contrib/zstd/common/huf.h +0 -228
- data/contrib/zstd/common/zstd_common.c +0 -83
- data/contrib/zstd/common/zstd_errors.h +0 -60
- data/contrib/zstd/common/zstd_internal.h +0 -267
- data/contrib/zstd/compress/huf_compress.c +0 -533
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/compress/zstd_compress.c +0 -3264
- data/contrib/zstd/compress/zstd_opt.h +0 -900
- data/contrib/zstd/decompress/huf_decompress.c +0 -883
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
- data/contrib/zstd/zstd.h +0 -640
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
|
+
* All rights reserved.
|
|
4
|
+
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
#ifndef DICTBUILDER_H_001
|
|
12
|
+
#define DICTBUILDER_H_001
|
|
13
|
+
|
|
14
|
+
#if defined (__cplusplus)
|
|
15
|
+
extern "C" {
|
|
16
|
+
#endif
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
/*====== Dependencies ======*/
|
|
20
|
+
#include <stddef.h> /* size_t */
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
/* ===== ZDICTLIB_API : control library symbols visibility ===== */
|
|
24
|
+
#ifndef ZDICTLIB_VISIBILITY
|
|
25
|
+
# if defined(__GNUC__) && (__GNUC__ >= 4)
|
|
26
|
+
# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
|
|
27
|
+
# else
|
|
28
|
+
# define ZDICTLIB_VISIBILITY
|
|
29
|
+
# endif
|
|
30
|
+
#endif
|
|
31
|
+
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
|
32
|
+
# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
|
|
33
|
+
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
|
|
34
|
+
# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
|
35
|
+
#else
|
|
36
|
+
# define ZDICTLIB_API ZDICTLIB_VISIBILITY
|
|
37
|
+
#endif
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
/*! ZDICT_trainFromBuffer():
|
|
41
|
+
* Train a dictionary from an array of samples.
|
|
42
|
+
* Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4,
|
|
43
|
+
* f=20, and accel=1.
|
|
44
|
+
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
|
45
|
+
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
|
46
|
+
* The resulting dictionary will be saved into `dictBuffer`.
|
|
47
|
+
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
|
48
|
+
* or an error code, which can be tested with ZDICT_isError().
|
|
49
|
+
* Note: Dictionary training will fail if there are not enough samples to construct a
|
|
50
|
+
* dictionary, or if most of the samples are too small (< 8 bytes being the lower limit).
|
|
51
|
+
* If dictionary training fails, you should use zstd without a dictionary, as the dictionary
|
|
52
|
+
* would've been ineffective anyways. If you believe your samples would benefit from a dictionary
|
|
53
|
+
* please open an issue with details, and we can look into it.
|
|
54
|
+
* Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB.
|
|
55
|
+
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
|
|
56
|
+
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
|
|
57
|
+
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
|
58
|
+
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
|
59
|
+
*/
|
|
60
|
+
ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
|
61
|
+
const void* samplesBuffer,
|
|
62
|
+
const size_t* samplesSizes, unsigned nbSamples);
|
|
63
|
+
|
|
64
|
+
typedef struct {
|
|
65
|
+
int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
|
|
66
|
+
unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
|
67
|
+
unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */
|
|
68
|
+
} ZDICT_params_t;
|
|
69
|
+
|
|
70
|
+
/*! ZDICT_finalizeDictionary():
|
|
71
|
+
* Given a custom content as a basis for dictionary, and a set of samples,
|
|
72
|
+
* finalize dictionary by adding headers and statistics according to the zstd
|
|
73
|
+
* dictionary format.
|
|
74
|
+
*
|
|
75
|
+
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
|
76
|
+
* supplied with an array of sizes `samplesSizes`, providing the size of each
|
|
77
|
+
* sample in order. The samples are used to construct the statistics, so they
|
|
78
|
+
* should be representative of what you will compress with this dictionary.
|
|
79
|
+
*
|
|
80
|
+
* The compression level can be set in `parameters`. You should pass the
|
|
81
|
+
* compression level you expect to use in production. The statistics for each
|
|
82
|
+
* compression level differ, so tuning the dictionary for the compression level
|
|
83
|
+
* can help quite a bit.
|
|
84
|
+
*
|
|
85
|
+
* You can set an explicit dictionary ID in `parameters`, or allow us to pick
|
|
86
|
+
* a random dictionary ID for you, but we can't guarantee no collisions.
|
|
87
|
+
*
|
|
88
|
+
* The dstDictBuffer and the dictContent may overlap, and the content will be
|
|
89
|
+
* appended to the end of the header. If the header + the content doesn't fit in
|
|
90
|
+
* maxDictSize the beginning of the content is truncated to make room, since it
|
|
91
|
+
* is presumed that the most profitable content is at the end of the dictionary,
|
|
92
|
+
* since that is the cheapest to reference.
|
|
93
|
+
*
|
|
94
|
+
* `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
|
|
95
|
+
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
|
|
96
|
+
*
|
|
97
|
+
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
|
|
98
|
+
* or an error code, which can be tested by ZDICT_isError().
|
|
99
|
+
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if
|
|
100
|
+
* instructed to, using notificationLevel>0.
|
|
101
|
+
* NOTE: This function currently may fail in several edge cases including:
|
|
102
|
+
* * Not enough samples
|
|
103
|
+
* * Samples are uncompressible
|
|
104
|
+
* * Samples are all exactly the same
|
|
105
|
+
*/
|
|
106
|
+
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
|
|
107
|
+
const void* dictContent, size_t dictContentSize,
|
|
108
|
+
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
|
109
|
+
ZDICT_params_t parameters);
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
/*====== Helper functions ======*/
|
|
113
|
+
ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
|
|
114
|
+
ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
|
|
115
|
+
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
|
|
116
|
+
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
#ifdef ZDICT_STATIC_LINKING_ONLY
|
|
121
|
+
|
|
122
|
+
/* ====================================================================================
|
|
123
|
+
* The definitions in this section are considered experimental.
|
|
124
|
+
* They should never be used with a dynamic library, as they may change in the future.
|
|
125
|
+
* They are provided for advanced usages.
|
|
126
|
+
* Use them only in association with static linking.
|
|
127
|
+
* ==================================================================================== */
|
|
128
|
+
|
|
129
|
+
#define ZDICT_CONTENTSIZE_MIN 128
|
|
130
|
+
#define ZDICT_DICTSIZE_MIN 256
|
|
131
|
+
|
|
132
|
+
/*! ZDICT_cover_params_t:
|
|
133
|
+
* k and d are the only required parameters.
|
|
134
|
+
* For others, value 0 means default.
|
|
135
|
+
*/
|
|
136
|
+
typedef struct {
|
|
137
|
+
unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
|
|
138
|
+
unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
|
|
139
|
+
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
|
|
140
|
+
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
|
141
|
+
double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
|
|
142
|
+
unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
|
|
143
|
+
unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
|
|
144
|
+
ZDICT_params_t zParams;
|
|
145
|
+
} ZDICT_cover_params_t;
|
|
146
|
+
|
|
147
|
+
typedef struct {
|
|
148
|
+
unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
|
|
149
|
+
unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
|
|
150
|
+
unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/
|
|
151
|
+
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
|
|
152
|
+
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
|
153
|
+
double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
|
|
154
|
+
unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
|
|
155
|
+
unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
|
|
156
|
+
unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
|
|
157
|
+
|
|
158
|
+
ZDICT_params_t zParams;
|
|
159
|
+
} ZDICT_fastCover_params_t;
|
|
160
|
+
|
|
161
|
+
/*! ZDICT_trainFromBuffer_cover():
|
|
162
|
+
* Train a dictionary from an array of samples using the COVER algorithm.
|
|
163
|
+
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
|
164
|
+
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
|
165
|
+
* The resulting dictionary will be saved into `dictBuffer`.
|
|
166
|
+
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
|
167
|
+
* or an error code, which can be tested with ZDICT_isError().
|
|
168
|
+
* See ZDICT_trainFromBuffer() for details on failure modes.
|
|
169
|
+
* Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte.
|
|
170
|
+
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
|
|
171
|
+
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
|
|
172
|
+
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
|
173
|
+
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
|
174
|
+
*/
|
|
175
|
+
ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
|
176
|
+
void *dictBuffer, size_t dictBufferCapacity,
|
|
177
|
+
const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
|
|
178
|
+
ZDICT_cover_params_t parameters);
|
|
179
|
+
|
|
180
|
+
/*! ZDICT_optimizeTrainFromBuffer_cover():
|
|
181
|
+
* The same requirements as above hold for all the parameters except `parameters`.
|
|
182
|
+
* This function tries many parameter combinations and picks the best parameters.
|
|
183
|
+
* `*parameters` is filled with the best parameters found,
|
|
184
|
+
* dictionary constructed with those parameters is stored in `dictBuffer`.
|
|
185
|
+
*
|
|
186
|
+
* All of the parameters d, k, steps are optional.
|
|
187
|
+
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
|
|
188
|
+
* if steps is zero it defaults to its default value.
|
|
189
|
+
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
|
|
190
|
+
*
|
|
191
|
+
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
|
192
|
+
* or an error code, which can be tested with ZDICT_isError().
|
|
193
|
+
* On success `*parameters` contains the parameters selected.
|
|
194
|
+
* See ZDICT_trainFromBuffer() for details on failure modes.
|
|
195
|
+
* Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
|
|
196
|
+
*/
|
|
197
|
+
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
|
198
|
+
void* dictBuffer, size_t dictBufferCapacity,
|
|
199
|
+
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
|
200
|
+
ZDICT_cover_params_t* parameters);
|
|
201
|
+
|
|
202
|
+
/*! ZDICT_trainFromBuffer_fastCover():
|
|
203
|
+
* Train a dictionary from an array of samples using a modified version of COVER algorithm.
|
|
204
|
+
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
|
205
|
+
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
|
206
|
+
* d and k are required.
|
|
207
|
+
* All other parameters are optional, will use default values if not provided
|
|
208
|
+
* The resulting dictionary will be saved into `dictBuffer`.
|
|
209
|
+
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
|
210
|
+
* or an error code, which can be tested with ZDICT_isError().
|
|
211
|
+
* See ZDICT_trainFromBuffer() for details on failure modes.
|
|
212
|
+
* Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory.
|
|
213
|
+
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
|
|
214
|
+
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
|
|
215
|
+
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
|
216
|
+
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
|
217
|
+
*/
|
|
218
|
+
ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
|
|
219
|
+
size_t dictBufferCapacity, const void *samplesBuffer,
|
|
220
|
+
const size_t *samplesSizes, unsigned nbSamples,
|
|
221
|
+
ZDICT_fastCover_params_t parameters);
|
|
222
|
+
|
|
223
|
+
/*! ZDICT_optimizeTrainFromBuffer_fastCover():
|
|
224
|
+
* The same requirements as above hold for all the parameters except `parameters`.
|
|
225
|
+
* This function tries many parameter combinations (specifically, k and d combinations)
|
|
226
|
+
* and picks the best parameters. `*parameters` is filled with the best parameters found,
|
|
227
|
+
* dictionary constructed with those parameters is stored in `dictBuffer`.
|
|
228
|
+
* All of the parameters d, k, steps, f, and accel are optional.
|
|
229
|
+
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
|
|
230
|
+
* if steps is zero it defaults to its default value.
|
|
231
|
+
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
|
|
232
|
+
* If f is zero, default value of 20 is used.
|
|
233
|
+
* If accel is zero, default value of 1 is used.
|
|
234
|
+
*
|
|
235
|
+
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
|
236
|
+
* or an error code, which can be tested with ZDICT_isError().
|
|
237
|
+
* On success `*parameters` contains the parameters selected.
|
|
238
|
+
* See ZDICT_trainFromBuffer() for details on failure modes.
|
|
239
|
+
* Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
|
|
240
|
+
*/
|
|
241
|
+
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
|
242
|
+
size_t dictBufferCapacity, const void* samplesBuffer,
|
|
243
|
+
const size_t* samplesSizes, unsigned nbSamples,
|
|
244
|
+
ZDICT_fastCover_params_t* parameters);
|
|
245
|
+
|
|
246
|
+
typedef struct {
|
|
247
|
+
unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
|
|
248
|
+
ZDICT_params_t zParams;
|
|
249
|
+
} ZDICT_legacy_params_t;
|
|
250
|
+
|
|
251
|
+
/*! ZDICT_trainFromBuffer_legacy():
|
|
252
|
+
* Train a dictionary from an array of samples.
|
|
253
|
+
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
|
254
|
+
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
|
255
|
+
* The resulting dictionary will be saved into `dictBuffer`.
|
|
256
|
+
* `parameters` is optional and can be provided with values set to 0 to mean "default".
|
|
257
|
+
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
|
258
|
+
* or an error code, which can be tested with ZDICT_isError().
|
|
259
|
+
* See ZDICT_trainFromBuffer() for details on failure modes.
|
|
260
|
+
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
|
|
261
|
+
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
|
|
262
|
+
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
|
263
|
+
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
|
264
|
+
* Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
|
|
265
|
+
*/
|
|
266
|
+
ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
|
|
267
|
+
void *dictBuffer, size_t dictBufferCapacity,
|
|
268
|
+
const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
|
|
269
|
+
ZDICT_legacy_params_t parameters);
|
|
270
|
+
|
|
271
|
+
/* Deprecation warnings */
|
|
272
|
+
/* It is generally possible to disable deprecation warnings from compiler,
|
|
273
|
+
for example with -Wno-deprecated-declarations for gcc
|
|
274
|
+
or _CRT_SECURE_NO_WARNINGS in Visual.
|
|
275
|
+
Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
|
|
276
|
+
#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
|
|
277
|
+
# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */
|
|
278
|
+
#else
|
|
279
|
+
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
|
280
|
+
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
|
281
|
+
# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
|
|
282
|
+
# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
|
|
283
|
+
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
|
|
284
|
+
# elif (ZDICT_GCC_VERSION >= 301)
|
|
285
|
+
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
|
|
286
|
+
# elif defined(_MSC_VER)
|
|
287
|
+
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message))
|
|
288
|
+
# else
|
|
289
|
+
# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
|
|
290
|
+
# define ZDICT_DEPRECATED(message) ZDICTLIB_API
|
|
291
|
+
# endif
|
|
292
|
+
#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
|
|
293
|
+
|
|
294
|
+
ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
|
|
295
|
+
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
|
296
|
+
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
#endif /* ZDICT_STATIC_LINKING_ONLY */
|
|
300
|
+
|
|
301
|
+
#if defined (__cplusplus)
|
|
302
|
+
}
|
|
303
|
+
#endif
|
|
304
|
+
|
|
305
|
+
#endif /* DICTBUILDER_H_001 */
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
* Copyright (c) 2016-
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
#ifndef ZSTD_LEGACY_H
|
|
@@ -17,17 +18,36 @@ extern "C" {
|
|
|
17
18
|
/* *************************************
|
|
18
19
|
* Includes
|
|
19
20
|
***************************************/
|
|
20
|
-
#include "mem.h" /* MEM_STATIC */
|
|
21
|
-
#include "error_private.h" /* ERROR */
|
|
22
|
-
#include "
|
|
23
|
-
#include "zstd_v01.h"
|
|
24
|
-
#include "zstd_v02.h"
|
|
25
|
-
#include "zstd_v03.h"
|
|
26
|
-
#include "zstd_v04.h"
|
|
27
|
-
#include "zstd_v05.h"
|
|
28
|
-
#include "zstd_v06.h"
|
|
29
|
-
#include "zstd_v07.h"
|
|
21
|
+
#include "../common/mem.h" /* MEM_STATIC */
|
|
22
|
+
#include "../common/error_private.h" /* ERROR */
|
|
23
|
+
#include "../common/zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
|
|
30
24
|
|
|
25
|
+
#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
|
|
26
|
+
# undef ZSTD_LEGACY_SUPPORT
|
|
27
|
+
# define ZSTD_LEGACY_SUPPORT 8
|
|
28
|
+
#endif
|
|
29
|
+
|
|
30
|
+
#if (ZSTD_LEGACY_SUPPORT <= 1)
|
|
31
|
+
# include "zstd_v01.h"
|
|
32
|
+
#endif
|
|
33
|
+
#if (ZSTD_LEGACY_SUPPORT <= 2)
|
|
34
|
+
# include "zstd_v02.h"
|
|
35
|
+
#endif
|
|
36
|
+
#if (ZSTD_LEGACY_SUPPORT <= 3)
|
|
37
|
+
# include "zstd_v03.h"
|
|
38
|
+
#endif
|
|
39
|
+
#if (ZSTD_LEGACY_SUPPORT <= 4)
|
|
40
|
+
# include "zstd_v04.h"
|
|
41
|
+
#endif
|
|
42
|
+
#if (ZSTD_LEGACY_SUPPORT <= 5)
|
|
43
|
+
# include "zstd_v05.h"
|
|
44
|
+
#endif
|
|
45
|
+
#if (ZSTD_LEGACY_SUPPORT <= 6)
|
|
46
|
+
# include "zstd_v06.h"
|
|
47
|
+
#endif
|
|
48
|
+
#if (ZSTD_LEGACY_SUPPORT <= 7)
|
|
49
|
+
# include "zstd_v07.h"
|
|
50
|
+
#endif
|
|
31
51
|
|
|
32
52
|
/** ZSTD_isLegacy() :
|
|
33
53
|
@return : > 0 if supported by legacy decoder. 0 otherwise.
|
|
@@ -40,13 +60,27 @@ MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize)
|
|
|
40
60
|
magicNumberLE = MEM_readLE32(src);
|
|
41
61
|
switch(magicNumberLE)
|
|
42
62
|
{
|
|
63
|
+
#if (ZSTD_LEGACY_SUPPORT <= 1)
|
|
43
64
|
case ZSTDv01_magicNumberLE:return 1;
|
|
65
|
+
#endif
|
|
66
|
+
#if (ZSTD_LEGACY_SUPPORT <= 2)
|
|
44
67
|
case ZSTDv02_magicNumber : return 2;
|
|
68
|
+
#endif
|
|
69
|
+
#if (ZSTD_LEGACY_SUPPORT <= 3)
|
|
45
70
|
case ZSTDv03_magicNumber : return 3;
|
|
71
|
+
#endif
|
|
72
|
+
#if (ZSTD_LEGACY_SUPPORT <= 4)
|
|
46
73
|
case ZSTDv04_magicNumber : return 4;
|
|
74
|
+
#endif
|
|
75
|
+
#if (ZSTD_LEGACY_SUPPORT <= 5)
|
|
47
76
|
case ZSTDv05_MAGICNUMBER : return 5;
|
|
77
|
+
#endif
|
|
78
|
+
#if (ZSTD_LEGACY_SUPPORT <= 6)
|
|
48
79
|
case ZSTDv06_MAGICNUMBER : return 6;
|
|
80
|
+
#endif
|
|
81
|
+
#if (ZSTD_LEGACY_SUPPORT <= 7)
|
|
49
82
|
case ZSTDv07_MAGICNUMBER : return 7;
|
|
83
|
+
#endif
|
|
50
84
|
default : return 0;
|
|
51
85
|
}
|
|
52
86
|
}
|
|
@@ -56,24 +90,30 @@ MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, s
|
|
|
56
90
|
{
|
|
57
91
|
U32 const version = ZSTD_isLegacy(src, srcSize);
|
|
58
92
|
if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */
|
|
93
|
+
#if (ZSTD_LEGACY_SUPPORT <= 5)
|
|
59
94
|
if (version==5) {
|
|
60
95
|
ZSTDv05_parameters fParams;
|
|
61
96
|
size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize);
|
|
62
97
|
if (frResult != 0) return 0;
|
|
63
98
|
return fParams.srcSize;
|
|
64
99
|
}
|
|
100
|
+
#endif
|
|
101
|
+
#if (ZSTD_LEGACY_SUPPORT <= 6)
|
|
65
102
|
if (version==6) {
|
|
66
103
|
ZSTDv06_frameParams fParams;
|
|
67
104
|
size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize);
|
|
68
105
|
if (frResult != 0) return 0;
|
|
69
106
|
return fParams.frameContentSize;
|
|
70
107
|
}
|
|
108
|
+
#endif
|
|
109
|
+
#if (ZSTD_LEGACY_SUPPORT <= 7)
|
|
71
110
|
if (version==7) {
|
|
72
111
|
ZSTDv07_frameParams fParams;
|
|
73
112
|
size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize);
|
|
74
113
|
if (frResult != 0) return 0;
|
|
75
114
|
return fParams.frameContentSize;
|
|
76
115
|
}
|
|
116
|
+
#endif
|
|
77
117
|
return 0; /* should not be possible */
|
|
78
118
|
}
|
|
79
119
|
|
|
@@ -84,16 +124,26 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
|
|
|
84
124
|
const void* dict,size_t dictSize)
|
|
85
125
|
{
|
|
86
126
|
U32 const version = ZSTD_isLegacy(src, compressedSize);
|
|
127
|
+
(void)dst; (void)dstCapacity; (void)dict; (void)dictSize; /* unused when ZSTD_LEGACY_SUPPORT >= 8 */
|
|
87
128
|
switch(version)
|
|
88
129
|
{
|
|
130
|
+
#if (ZSTD_LEGACY_SUPPORT <= 1)
|
|
89
131
|
case 1 :
|
|
90
132
|
return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize);
|
|
133
|
+
#endif
|
|
134
|
+
#if (ZSTD_LEGACY_SUPPORT <= 2)
|
|
91
135
|
case 2 :
|
|
92
136
|
return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize);
|
|
137
|
+
#endif
|
|
138
|
+
#if (ZSTD_LEGACY_SUPPORT <= 3)
|
|
93
139
|
case 3 :
|
|
94
140
|
return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize);
|
|
141
|
+
#endif
|
|
142
|
+
#if (ZSTD_LEGACY_SUPPORT <= 4)
|
|
95
143
|
case 4 :
|
|
96
144
|
return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);
|
|
145
|
+
#endif
|
|
146
|
+
#if (ZSTD_LEGACY_SUPPORT <= 5)
|
|
97
147
|
case 5 :
|
|
98
148
|
{ size_t result;
|
|
99
149
|
ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx();
|
|
@@ -102,6 +152,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
|
|
|
102
152
|
ZSTDv05_freeDCtx(zd);
|
|
103
153
|
return result;
|
|
104
154
|
}
|
|
155
|
+
#endif
|
|
156
|
+
#if (ZSTD_LEGACY_SUPPORT <= 6)
|
|
105
157
|
case 6 :
|
|
106
158
|
{ size_t result;
|
|
107
159
|
ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx();
|
|
@@ -110,6 +162,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
|
|
|
110
162
|
ZSTDv06_freeDCtx(zd);
|
|
111
163
|
return result;
|
|
112
164
|
}
|
|
165
|
+
#endif
|
|
166
|
+
#if (ZSTD_LEGACY_SUPPORT <= 7)
|
|
113
167
|
case 7 :
|
|
114
168
|
{ size_t result;
|
|
115
169
|
ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx();
|
|
@@ -118,11 +172,84 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
|
|
|
118
172
|
ZSTDv07_freeDCtx(zd);
|
|
119
173
|
return result;
|
|
120
174
|
}
|
|
175
|
+
#endif
|
|
121
176
|
default :
|
|
122
177
|
return ERROR(prefix_unknown);
|
|
123
178
|
}
|
|
124
179
|
}
|
|
125
180
|
|
|
181
|
+
MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize)
|
|
182
|
+
{
|
|
183
|
+
ZSTD_frameSizeInfo frameSizeInfo;
|
|
184
|
+
U32 const version = ZSTD_isLegacy(src, srcSize);
|
|
185
|
+
switch(version)
|
|
186
|
+
{
|
|
187
|
+
#if (ZSTD_LEGACY_SUPPORT <= 1)
|
|
188
|
+
case 1 :
|
|
189
|
+
ZSTDv01_findFrameSizeInfoLegacy(src, srcSize,
|
|
190
|
+
&frameSizeInfo.compressedSize,
|
|
191
|
+
&frameSizeInfo.decompressedBound);
|
|
192
|
+
break;
|
|
193
|
+
#endif
|
|
194
|
+
#if (ZSTD_LEGACY_SUPPORT <= 2)
|
|
195
|
+
case 2 :
|
|
196
|
+
ZSTDv02_findFrameSizeInfoLegacy(src, srcSize,
|
|
197
|
+
&frameSizeInfo.compressedSize,
|
|
198
|
+
&frameSizeInfo.decompressedBound);
|
|
199
|
+
break;
|
|
200
|
+
#endif
|
|
201
|
+
#if (ZSTD_LEGACY_SUPPORT <= 3)
|
|
202
|
+
case 3 :
|
|
203
|
+
ZSTDv03_findFrameSizeInfoLegacy(src, srcSize,
|
|
204
|
+
&frameSizeInfo.compressedSize,
|
|
205
|
+
&frameSizeInfo.decompressedBound);
|
|
206
|
+
break;
|
|
207
|
+
#endif
|
|
208
|
+
#if (ZSTD_LEGACY_SUPPORT <= 4)
|
|
209
|
+
case 4 :
|
|
210
|
+
ZSTDv04_findFrameSizeInfoLegacy(src, srcSize,
|
|
211
|
+
&frameSizeInfo.compressedSize,
|
|
212
|
+
&frameSizeInfo.decompressedBound);
|
|
213
|
+
break;
|
|
214
|
+
#endif
|
|
215
|
+
#if (ZSTD_LEGACY_SUPPORT <= 5)
|
|
216
|
+
case 5 :
|
|
217
|
+
ZSTDv05_findFrameSizeInfoLegacy(src, srcSize,
|
|
218
|
+
&frameSizeInfo.compressedSize,
|
|
219
|
+
&frameSizeInfo.decompressedBound);
|
|
220
|
+
break;
|
|
221
|
+
#endif
|
|
222
|
+
#if (ZSTD_LEGACY_SUPPORT <= 6)
|
|
223
|
+
case 6 :
|
|
224
|
+
ZSTDv06_findFrameSizeInfoLegacy(src, srcSize,
|
|
225
|
+
&frameSizeInfo.compressedSize,
|
|
226
|
+
&frameSizeInfo.decompressedBound);
|
|
227
|
+
break;
|
|
228
|
+
#endif
|
|
229
|
+
#if (ZSTD_LEGACY_SUPPORT <= 7)
|
|
230
|
+
case 7 :
|
|
231
|
+
ZSTDv07_findFrameSizeInfoLegacy(src, srcSize,
|
|
232
|
+
&frameSizeInfo.compressedSize,
|
|
233
|
+
&frameSizeInfo.decompressedBound);
|
|
234
|
+
break;
|
|
235
|
+
#endif
|
|
236
|
+
default :
|
|
237
|
+
frameSizeInfo.compressedSize = ERROR(prefix_unknown);
|
|
238
|
+
frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
|
|
239
|
+
break;
|
|
240
|
+
}
|
|
241
|
+
if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) {
|
|
242
|
+
frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
|
|
243
|
+
frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
|
|
244
|
+
}
|
|
245
|
+
return frameSizeInfo;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src, size_t srcSize)
|
|
249
|
+
{
|
|
250
|
+
ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfoLegacy(src, srcSize);
|
|
251
|
+
return frameSizeInfo.compressedSize;
|
|
252
|
+
}
|
|
126
253
|
|
|
127
254
|
MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
|
|
128
255
|
{
|
|
@@ -132,11 +259,20 @@ MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
|
|
|
132
259
|
case 1 :
|
|
133
260
|
case 2 :
|
|
134
261
|
case 3 :
|
|
262
|
+
(void)legacyContext;
|
|
135
263
|
return ERROR(version_unsupported);
|
|
264
|
+
#if (ZSTD_LEGACY_SUPPORT <= 4)
|
|
136
265
|
case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext);
|
|
266
|
+
#endif
|
|
267
|
+
#if (ZSTD_LEGACY_SUPPORT <= 5)
|
|
137
268
|
case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext);
|
|
269
|
+
#endif
|
|
270
|
+
#if (ZSTD_LEGACY_SUPPORT <= 6)
|
|
138
271
|
case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext);
|
|
272
|
+
#endif
|
|
273
|
+
#if (ZSTD_LEGACY_SUPPORT <= 7)
|
|
139
274
|
case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext);
|
|
275
|
+
#endif
|
|
140
276
|
}
|
|
141
277
|
}
|
|
142
278
|
|
|
@@ -144,6 +280,7 @@ MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
|
|
|
144
280
|
MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion,
|
|
145
281
|
const void* dict, size_t dictSize)
|
|
146
282
|
{
|
|
283
|
+
DEBUGLOG(5, "ZSTD_initLegacyStream for v0.%u", newVersion);
|
|
147
284
|
if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion);
|
|
148
285
|
switch(newVersion)
|
|
149
286
|
{
|
|
@@ -151,7 +288,9 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
|
|
|
151
288
|
case 1 :
|
|
152
289
|
case 2 :
|
|
153
290
|
case 3 :
|
|
291
|
+
(void)dict; (void)dictSize;
|
|
154
292
|
return 0;
|
|
293
|
+
#if (ZSTD_LEGACY_SUPPORT <= 4)
|
|
155
294
|
case 4 :
|
|
156
295
|
{
|
|
157
296
|
ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext;
|
|
@@ -161,6 +300,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
|
|
|
161
300
|
*legacyContext = dctx;
|
|
162
301
|
return 0;
|
|
163
302
|
}
|
|
303
|
+
#endif
|
|
304
|
+
#if (ZSTD_LEGACY_SUPPORT <= 5)
|
|
164
305
|
case 5 :
|
|
165
306
|
{
|
|
166
307
|
ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext;
|
|
@@ -169,6 +310,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
|
|
|
169
310
|
*legacyContext = dctx;
|
|
170
311
|
return 0;
|
|
171
312
|
}
|
|
313
|
+
#endif
|
|
314
|
+
#if (ZSTD_LEGACY_SUPPORT <= 6)
|
|
172
315
|
case 6 :
|
|
173
316
|
{
|
|
174
317
|
ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext;
|
|
@@ -177,6 +320,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
|
|
|
177
320
|
*legacyContext = dctx;
|
|
178
321
|
return 0;
|
|
179
322
|
}
|
|
323
|
+
#endif
|
|
324
|
+
#if (ZSTD_LEGACY_SUPPORT <= 7)
|
|
180
325
|
case 7 :
|
|
181
326
|
{
|
|
182
327
|
ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext;
|
|
@@ -185,6 +330,7 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
|
|
|
185
330
|
*legacyContext = dctx;
|
|
186
331
|
return 0;
|
|
187
332
|
}
|
|
333
|
+
#endif
|
|
188
334
|
}
|
|
189
335
|
}
|
|
190
336
|
|
|
@@ -193,13 +339,16 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
|
|
|
193
339
|
MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
|
|
194
340
|
ZSTD_outBuffer* output, ZSTD_inBuffer* input)
|
|
195
341
|
{
|
|
342
|
+
DEBUGLOG(5, "ZSTD_decompressLegacyStream for v0.%u", version);
|
|
196
343
|
switch(version)
|
|
197
344
|
{
|
|
198
345
|
default :
|
|
199
346
|
case 1 :
|
|
200
347
|
case 2 :
|
|
201
348
|
case 3 :
|
|
349
|
+
(void)legacyContext; (void)output; (void)input;
|
|
202
350
|
return ERROR(version_unsupported);
|
|
351
|
+
#if (ZSTD_LEGACY_SUPPORT <= 4)
|
|
203
352
|
case 4 :
|
|
204
353
|
{
|
|
205
354
|
ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext;
|
|
@@ -212,6 +361,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
|
|
|
212
361
|
input->pos += readSize;
|
|
213
362
|
return hintSize;
|
|
214
363
|
}
|
|
364
|
+
#endif
|
|
365
|
+
#if (ZSTD_LEGACY_SUPPORT <= 5)
|
|
215
366
|
case 5 :
|
|
216
367
|
{
|
|
217
368
|
ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext;
|
|
@@ -224,6 +375,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
|
|
|
224
375
|
input->pos += readSize;
|
|
225
376
|
return hintSize;
|
|
226
377
|
}
|
|
378
|
+
#endif
|
|
379
|
+
#if (ZSTD_LEGACY_SUPPORT <= 6)
|
|
227
380
|
case 6 :
|
|
228
381
|
{
|
|
229
382
|
ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext;
|
|
@@ -236,6 +389,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
|
|
|
236
389
|
input->pos += readSize;
|
|
237
390
|
return hintSize;
|
|
238
391
|
}
|
|
392
|
+
#endif
|
|
393
|
+
#if (ZSTD_LEGACY_SUPPORT <= 7)
|
|
239
394
|
case 7 :
|
|
240
395
|
{
|
|
241
396
|
ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext;
|
|
@@ -248,6 +403,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
|
|
|
248
403
|
input->pos += readSize;
|
|
249
404
|
return hintSize;
|
|
250
405
|
}
|
|
406
|
+
#endif
|
|
251
407
|
}
|
|
252
408
|
}
|
|
253
409
|
|