extzstd 0.3.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +28 -14
- data/contrib/zstd/CHANGELOG +114 -56
- data/contrib/zstd/CONTRIBUTING.md +14 -0
- data/contrib/zstd/Makefile +37 -31
- data/contrib/zstd/README.md +6 -0
- data/contrib/zstd/appveyor.yml +4 -1
- data/contrib/zstd/lib/Makefile +231 -134
- data/contrib/zstd/lib/README.md +28 -0
- data/contrib/zstd/lib/common/bitstream.h +24 -15
- data/contrib/zstd/lib/common/compiler.h +116 -3
- data/contrib/zstd/lib/common/cpu.h +0 -2
- data/contrib/zstd/lib/common/debug.h +11 -18
- data/contrib/zstd/lib/common/entropy_common.c +188 -42
- data/contrib/zstd/lib/common/error_private.c +1 -0
- data/contrib/zstd/lib/common/error_private.h +1 -1
- data/contrib/zstd/lib/common/fse.h +38 -11
- data/contrib/zstd/lib/common/fse_decompress.c +123 -16
- data/contrib/zstd/lib/common/huf.h +26 -5
- data/contrib/zstd/lib/common/mem.h +66 -93
- data/contrib/zstd/lib/common/pool.c +22 -16
- data/contrib/zstd/lib/common/pool.h +1 -1
- data/contrib/zstd/lib/common/threading.c +6 -5
- data/contrib/zstd/lib/common/xxhash.c +18 -56
- data/contrib/zstd/lib/common/xxhash.h +1 -1
- data/contrib/zstd/lib/common/zstd_common.c +9 -9
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_errors.h +1 -0
- data/contrib/zstd/lib/common/zstd_internal.h +89 -58
- data/contrib/zstd/lib/compress/fse_compress.c +30 -23
- data/contrib/zstd/lib/compress/hist.c +26 -28
- data/contrib/zstd/lib/compress/hist.h +1 -1
- data/contrib/zstd/lib/compress/huf_compress.c +210 -95
- data/contrib/zstd/lib/compress/zstd_compress.c +1339 -409
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +119 -41
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +4 -4
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +17 -3
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +23 -19
- data/contrib/zstd/lib/compress/zstd_cwksp.h +60 -24
- data/contrib/zstd/lib/compress/zstd_double_fast.c +22 -22
- data/contrib/zstd/lib/compress/zstd_fast.c +19 -19
- data/contrib/zstd/lib/compress/zstd_lazy.c +351 -77
- data/contrib/zstd/lib/compress/zstd_lazy.h +20 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +59 -18
- data/contrib/zstd/lib/compress/zstd_ldm.h +6 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +190 -45
- data/contrib/zstd/lib/compress/zstdmt_compress.c +74 -406
- data/contrib/zstd/lib/compress/zstdmt_compress.h +26 -108
- data/contrib/zstd/lib/decompress/huf_decompress.c +302 -200
- data/contrib/zstd/lib/decompress/zstd_ddict.c +8 -8
- data/contrib/zstd/lib/decompress/zstd_ddict.h +1 -1
- data/contrib/zstd/lib/decompress/zstd_decompress.c +125 -80
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +145 -37
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +5 -2
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +11 -10
- data/contrib/zstd/lib/dictBuilder/cover.c +29 -20
- data/contrib/zstd/lib/dictBuilder/cover.h +1 -1
- data/contrib/zstd/lib/dictBuilder/fastcover.c +20 -19
- data/contrib/zstd/lib/dictBuilder/zdict.c +15 -16
- data/contrib/zstd/lib/dictBuilder/zdict.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v01.c +5 -1
- data/contrib/zstd/lib/legacy/zstd_v02.c +5 -1
- data/contrib/zstd/lib/legacy/zstd_v03.c +5 -1
- data/contrib/zstd/lib/legacy/zstd_v04.c +6 -2
- data/contrib/zstd/lib/legacy/zstd_v05.c +5 -1
- data/contrib/zstd/lib/legacy/zstd_v06.c +5 -1
- data/contrib/zstd/lib/legacy/zstd_v07.c +5 -1
- data/contrib/zstd/lib/libzstd.pc.in +3 -3
- data/contrib/zstd/lib/zstd.h +348 -47
- data/ext/extzstd.c +6 -0
- data/ext/extzstd.h +6 -0
- data/gemstub.rb +3 -21
- data/lib/extzstd.rb +0 -2
- data/lib/extzstd/version.rb +6 -1
- data/test/test_basic.rb +0 -5
- metadata +5 -4
@@ -40,33 +40,42 @@
|
|
40
40
|
* Constants
|
41
41
|
***************************************/
|
42
42
|
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
|
43
|
-
#define
|
43
|
+
#define COVER_DEFAULT_SPLITPOINT 1.0
|
44
44
|
|
45
45
|
/*-*************************************
|
46
46
|
* Console display
|
47
47
|
***************************************/
|
48
|
+
#ifndef LOCALDISPLAYLEVEL
|
48
49
|
static int g_displayLevel = 2;
|
50
|
+
#endif
|
51
|
+
#undef DISPLAY
|
49
52
|
#define DISPLAY(...) \
|
50
53
|
{ \
|
51
54
|
fprintf(stderr, __VA_ARGS__); \
|
52
55
|
fflush(stderr); \
|
53
56
|
}
|
57
|
+
#undef LOCALDISPLAYLEVEL
|
54
58
|
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
|
55
59
|
if (displayLevel >= l) { \
|
56
60
|
DISPLAY(__VA_ARGS__); \
|
57
61
|
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
62
|
+
#undef DISPLAYLEVEL
|
58
63
|
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
|
59
64
|
|
65
|
+
#ifndef LOCALDISPLAYUPDATE
|
66
|
+
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
67
|
+
static clock_t g_time = 0;
|
68
|
+
#endif
|
69
|
+
#undef LOCALDISPLAYUPDATE
|
60
70
|
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
61
71
|
if (displayLevel >= l) { \
|
62
|
-
if ((clock() - g_time >
|
72
|
+
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
|
63
73
|
g_time = clock(); \
|
64
74
|
DISPLAY(__VA_ARGS__); \
|
65
75
|
} \
|
66
76
|
}
|
77
|
+
#undef DISPLAYUPDATE
|
67
78
|
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
|
68
|
-
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
69
|
-
static clock_t g_time = 0;
|
70
79
|
|
71
80
|
/*-*************************************
|
72
81
|
* Hash table
|
@@ -120,9 +129,9 @@ static int COVER_map_init(COVER_map_t *map, U32 size) {
|
|
120
129
|
/**
|
121
130
|
* Internal hash function
|
122
131
|
*/
|
123
|
-
static const U32
|
132
|
+
static const U32 COVER_prime4bytes = 2654435761U;
|
124
133
|
static U32 COVER_map_hash(COVER_map_t *map, U32 key) {
|
125
|
-
return (key *
|
134
|
+
return (key * COVER_prime4bytes) >> (32 - map->sizeLog);
|
126
135
|
}
|
127
136
|
|
128
137
|
/**
|
@@ -215,7 +224,7 @@ typedef struct {
|
|
215
224
|
} COVER_ctx_t;
|
216
225
|
|
217
226
|
/* We need a global context for qsort... */
|
218
|
-
static COVER_ctx_t *
|
227
|
+
static COVER_ctx_t *g_coverCtx = NULL;
|
219
228
|
|
220
229
|
/*-*************************************
|
221
230
|
* Helper functions
|
@@ -258,11 +267,11 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
|
|
258
267
|
|
259
268
|
/**
|
260
269
|
* Same as COVER_cmp() except ties are broken by pointer value
|
261
|
-
* NOTE:
|
270
|
+
* NOTE: g_coverCtx must be set to call this function. A global is required because
|
262
271
|
* qsort doesn't take an opaque pointer.
|
263
272
|
*/
|
264
|
-
static int COVER_strict_cmp(const void *lp, const void *rp) {
|
265
|
-
int result = COVER_cmp(
|
273
|
+
static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
|
274
|
+
int result = COVER_cmp(g_coverCtx, lp, rp);
|
266
275
|
if (result == 0) {
|
267
276
|
result = lp < rp ? -1 : 1;
|
268
277
|
}
|
@@ -271,8 +280,8 @@ static int COVER_strict_cmp(const void *lp, const void *rp) {
|
|
271
280
|
/**
|
272
281
|
* Faster version for d <= 8.
|
273
282
|
*/
|
274
|
-
static int COVER_strict_cmp8(const void *lp, const void *rp) {
|
275
|
-
int result = COVER_cmp8(
|
283
|
+
static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) {
|
284
|
+
int result = COVER_cmp8(g_coverCtx, lp, rp);
|
276
285
|
if (result == 0) {
|
277
286
|
result = lp < rp ? -1 : 1;
|
278
287
|
}
|
@@ -603,7 +612,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
|
603
612
|
/* qsort doesn't take an opaque pointer, so pass as a global.
|
604
613
|
* On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is.
|
605
614
|
*/
|
606
|
-
|
615
|
+
g_coverCtx = ctx;
|
607
616
|
#if defined(__OpenBSD__)
|
608
617
|
mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
609
618
|
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
|
@@ -946,7 +955,7 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection){
|
|
946
955
|
free(selection.dictContent);
|
947
956
|
}
|
948
957
|
|
949
|
-
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
958
|
+
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
|
950
959
|
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
|
951
960
|
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) {
|
952
961
|
|
@@ -954,8 +963,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
|
954
963
|
size_t largestCompressed = 0;
|
955
964
|
BYTE* customDictContentEnd = customDictContent + dictContentSize;
|
956
965
|
|
957
|
-
BYTE * largestDictbuffer = (BYTE *)malloc(
|
958
|
-
BYTE * candidateDictBuffer = (BYTE *)malloc(
|
966
|
+
BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity);
|
967
|
+
BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity);
|
959
968
|
double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00;
|
960
969
|
|
961
970
|
if (!largestDictbuffer || !candidateDictBuffer) {
|
@@ -967,7 +976,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
|
967
976
|
/* Initial dictionary size and compressed size */
|
968
977
|
memcpy(largestDictbuffer, customDictContent, dictContentSize);
|
969
978
|
dictContentSize = ZDICT_finalizeDictionary(
|
970
|
-
largestDictbuffer,
|
979
|
+
largestDictbuffer, dictBufferCapacity, customDictContent, dictContentSize,
|
971
980
|
samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
|
972
981
|
|
973
982
|
if (ZDICT_isError(dictContentSize)) {
|
@@ -1001,7 +1010,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
|
1001
1010
|
while (dictContentSize < largestDict) {
|
1002
1011
|
memcpy(candidateDictBuffer, largestDictbuffer, largestDict);
|
1003
1012
|
dictContentSize = ZDICT_finalizeDictionary(
|
1004
|
-
candidateDictBuffer,
|
1013
|
+
candidateDictBuffer, dictBufferCapacity, customDictContentEnd - dictContentSize, dictContentSize,
|
1005
1014
|
samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
|
1006
1015
|
|
1007
1016
|
if (ZDICT_isError(dictContentSize)) {
|
@@ -1079,7 +1088,7 @@ static void COVER_tryParameters(void *opaque) {
|
|
1079
1088
|
{
|
1080
1089
|
const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
|
1081
1090
|
dictBufferCapacity, parameters);
|
1082
|
-
selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
|
1091
|
+
selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
|
1083
1092
|
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
|
1084
1093
|
totalCompressedSize);
|
1085
1094
|
|
@@ -1106,7 +1115,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
|
1106
1115
|
/* constants */
|
1107
1116
|
const unsigned nbThreads = parameters->nbThreads;
|
1108
1117
|
const double splitPoint =
|
1109
|
-
parameters->splitPoint <= 0.0 ?
|
1118
|
+
parameters->splitPoint <= 0.0 ? COVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
|
1110
1119
|
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
|
1111
1120
|
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
|
1112
1121
|
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
|
@@ -152,6 +152,6 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection);
|
|
152
152
|
* smallest dictionary within a specified regression of the compressed size
|
153
153
|
* from the largest dictionary.
|
154
154
|
*/
|
155
|
-
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
155
|
+
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
|
156
156
|
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
|
157
157
|
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
|
@@ -21,6 +21,7 @@
|
|
21
21
|
#include "../common/threading.h"
|
22
22
|
#include "cover.h"
|
23
23
|
#include "../common/zstd_internal.h" /* includes zstd.h */
|
24
|
+
#include "../compress/zstd_compress_internal.h" /* ZSTD_hash*() */
|
24
25
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
25
26
|
#define ZDICT_STATIC_LINKING_ONLY
|
26
27
|
#endif
|
@@ -33,7 +34,7 @@
|
|
33
34
|
#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
|
34
35
|
#define FASTCOVER_MAX_F 31
|
35
36
|
#define FASTCOVER_MAX_ACCEL 10
|
36
|
-
#define
|
37
|
+
#define FASTCOVER_DEFAULT_SPLITPOINT 0.75
|
37
38
|
#define DEFAULT_F 20
|
38
39
|
#define DEFAULT_ACCEL 1
|
39
40
|
|
@@ -41,50 +42,50 @@
|
|
41
42
|
/*-*************************************
|
42
43
|
* Console display
|
43
44
|
***************************************/
|
45
|
+
#ifndef LOCALDISPLAYLEVEL
|
44
46
|
static int g_displayLevel = 2;
|
47
|
+
#endif
|
48
|
+
#undef DISPLAY
|
45
49
|
#define DISPLAY(...) \
|
46
50
|
{ \
|
47
51
|
fprintf(stderr, __VA_ARGS__); \
|
48
52
|
fflush(stderr); \
|
49
53
|
}
|
54
|
+
#undef LOCALDISPLAYLEVEL
|
50
55
|
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
|
51
56
|
if (displayLevel >= l) { \
|
52
57
|
DISPLAY(__VA_ARGS__); \
|
53
58
|
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
59
|
+
#undef DISPLAYLEVEL
|
54
60
|
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
|
55
61
|
|
62
|
+
#ifndef LOCALDISPLAYUPDATE
|
63
|
+
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
64
|
+
static clock_t g_time = 0;
|
65
|
+
#endif
|
66
|
+
#undef LOCALDISPLAYUPDATE
|
56
67
|
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
57
68
|
if (displayLevel >= l) { \
|
58
|
-
if ((clock() - g_time >
|
69
|
+
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
|
59
70
|
g_time = clock(); \
|
60
71
|
DISPLAY(__VA_ARGS__); \
|
61
72
|
} \
|
62
73
|
}
|
74
|
+
#undef DISPLAYUPDATE
|
63
75
|
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
|
64
|
-
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
65
|
-
static clock_t g_time = 0;
|
66
76
|
|
67
77
|
|
68
78
|
/*-*************************************
|
69
79
|
* Hash Functions
|
70
80
|
***************************************/
|
71
|
-
static const U64 prime6bytes = 227718039650203ULL;
|
72
|
-
static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
|
73
|
-
static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
|
74
|
-
|
75
|
-
static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
|
76
|
-
static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
|
77
|
-
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
|
78
|
-
|
79
|
-
|
80
81
|
/**
|
81
|
-
* Hash the d-byte value pointed to by p and mod 2^f
|
82
|
+
* Hash the d-byte value pointed to by p and mod 2^f into the frequency vector
|
82
83
|
*/
|
83
|
-
static size_t FASTCOVER_hashPtrToIndex(const void* p, U32
|
84
|
+
static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 f, unsigned d) {
|
84
85
|
if (d == 6) {
|
85
|
-
return ZSTD_hash6Ptr(p,
|
86
|
+
return ZSTD_hash6Ptr(p, f);
|
86
87
|
}
|
87
|
-
return ZSTD_hash8Ptr(p,
|
88
|
+
return ZSTD_hash8Ptr(p, f);
|
88
89
|
}
|
89
90
|
|
90
91
|
|
@@ -486,7 +487,7 @@ static void FASTCOVER_tryParameters(void *opaque)
|
|
486
487
|
parameters, segmentFreqs);
|
487
488
|
|
488
489
|
const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
|
489
|
-
selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
|
490
|
+
selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
|
490
491
|
ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
|
491
492
|
totalCompressedSize);
|
492
493
|
|
@@ -617,7 +618,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
|
|
617
618
|
/* constants */
|
618
619
|
const unsigned nbThreads = parameters->nbThreads;
|
619
620
|
const double splitPoint =
|
620
|
-
parameters->splitPoint <= 0.0 ?
|
621
|
+
parameters->splitPoint <= 0.0 ? FASTCOVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
|
621
622
|
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
|
622
623
|
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
|
623
624
|
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
|
@@ -62,14 +62,15 @@
|
|
62
62
|
|
63
63
|
#define NOISELENGTH 32
|
64
64
|
|
65
|
-
static const int g_compressionLevel_default = 3;
|
66
65
|
static const U32 g_selectivity_default = 9;
|
67
66
|
|
68
67
|
|
69
68
|
/*-*************************************
|
70
69
|
* Console display
|
71
70
|
***************************************/
|
71
|
+
#undef DISPLAY
|
72
72
|
#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
|
73
|
+
#undef DISPLAYLEVEL
|
73
74
|
#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
74
75
|
|
75
76
|
static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
|
@@ -105,20 +106,17 @@ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
|
|
105
106
|
size_t headerSize;
|
106
107
|
if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);
|
107
108
|
|
108
|
-
{
|
109
|
-
ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
|
109
|
+
{ ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
|
110
110
|
U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE);
|
111
|
-
|
112
|
-
if (!bs || !wksp || !offcodeNCount) {
|
111
|
+
if (!bs || !wksp) {
|
113
112
|
headerSize = ERROR(memory_allocation);
|
114
113
|
} else {
|
115
114
|
ZSTD_reset_compressedBlockState(bs);
|
116
|
-
headerSize = ZSTD_loadCEntropy(bs, wksp,
|
115
|
+
headerSize = ZSTD_loadCEntropy(bs, wksp, dictBuffer, dictSize);
|
117
116
|
}
|
118
117
|
|
119
118
|
free(bs);
|
120
119
|
free(wksp);
|
121
|
-
free(offcodeNCount);
|
122
120
|
}
|
123
121
|
|
124
122
|
return headerSize;
|
@@ -532,6 +530,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
|
|
532
530
|
clock_t displayClock = 0;
|
533
531
|
clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
|
534
532
|
|
533
|
+
# undef DISPLAYUPDATE
|
535
534
|
# define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \
|
536
535
|
if (ZDICT_clockSpan(displayClock) > refreshRate) \
|
537
536
|
{ displayClock = clock(); DISPLAY(__VA_ARGS__); \
|
@@ -706,7 +705,7 @@ static void ZDICT_flatLit(unsigned* countLit)
|
|
706
705
|
|
707
706
|
#define OFFCODE_MAX 30 /* only applicable to first block */
|
708
707
|
static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
709
|
-
|
708
|
+
int compressionLevel,
|
710
709
|
const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles,
|
711
710
|
const void* dictBuffer, size_t dictBufferSize,
|
712
711
|
unsigned notificationLevel)
|
@@ -741,7 +740,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
741
740
|
memset(repOffset, 0, sizeof(repOffset));
|
742
741
|
repOffset[1] = repOffset[4] = repOffset[8] = 1;
|
743
742
|
memset(bestRepOffset, 0, sizeof(bestRepOffset));
|
744
|
-
if (compressionLevel==0) compressionLevel =
|
743
|
+
if (compressionLevel==0) compressionLevel = ZSTD_CLEVEL_DEFAULT;
|
745
744
|
params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
|
746
745
|
|
747
746
|
esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem);
|
@@ -786,7 +785,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
786
785
|
/* note : the result of this phase should be used to better appreciate the impact on statistics */
|
787
786
|
|
788
787
|
total=0; for (u=0; u<=offcodeMax; u++) total+=offcodeCount[u];
|
789
|
-
errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax);
|
788
|
+
errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax, /* useLowProbCount */ 1);
|
790
789
|
if (FSE_isError(errorCode)) {
|
791
790
|
eSize = errorCode;
|
792
791
|
DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n");
|
@@ -795,7 +794,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
795
794
|
Offlog = (U32)errorCode;
|
796
795
|
|
797
796
|
total=0; for (u=0; u<=MaxML; u++) total+=matchLengthCount[u];
|
798
|
-
errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML);
|
797
|
+
errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML, /* useLowProbCount */ 1);
|
799
798
|
if (FSE_isError(errorCode)) {
|
800
799
|
eSize = errorCode;
|
801
800
|
DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n");
|
@@ -804,7 +803,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
804
803
|
mlLog = (U32)errorCode;
|
805
804
|
|
806
805
|
total=0; for (u=0; u<=MaxLL; u++) total+=litLengthCount[u];
|
807
|
-
errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL);
|
806
|
+
errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL, /* useLowProbCount */ 1);
|
808
807
|
if (FSE_isError(errorCode)) {
|
809
808
|
eSize = errorCode;
|
810
809
|
DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n");
|
@@ -893,7 +892,7 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
|
893
892
|
size_t hSize;
|
894
893
|
#define HBUFFSIZE 256 /* should prove large enough for all entropy headers */
|
895
894
|
BYTE header[HBUFFSIZE];
|
896
|
-
int const compressionLevel = (params.compressionLevel == 0) ?
|
895
|
+
int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel;
|
897
896
|
U32 const notificationLevel = params.notificationLevel;
|
898
897
|
|
899
898
|
/* check conditions */
|
@@ -939,7 +938,7 @@ static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
|
|
939
938
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
940
939
|
ZDICT_params_t params)
|
941
940
|
{
|
942
|
-
int const compressionLevel = (params.compressionLevel == 0) ?
|
941
|
+
int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel;
|
943
942
|
U32 const notificationLevel = params.notificationLevel;
|
944
943
|
size_t hSize = 8;
|
945
944
|
|
@@ -1114,8 +1113,8 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
|
1114
1113
|
memset(¶ms, 0, sizeof(params));
|
1115
1114
|
params.d = 8;
|
1116
1115
|
params.steps = 4;
|
1117
|
-
/*
|
1118
|
-
params.zParams.compressionLevel =
|
1116
|
+
/* Use default level since no compression level information is available */
|
1117
|
+
params.zParams.compressionLevel = ZSTD_CLEVEL_DEFAULT;
|
1119
1118
|
#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1)
|
1120
1119
|
params.zParams.notificationLevel = DEBUGLEVEL;
|
1121
1120
|
#endif
|
@@ -279,7 +279,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
|
|
279
279
|
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
280
280
|
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
281
281
|
# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
|
282
|
-
# elif (ZDICT_GCC_VERSION >= 405)
|
282
|
+
# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
|
283
283
|
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
|
284
284
|
# elif (ZDICT_GCC_VERSION >= 301)
|
285
285
|
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
|
@@ -1280,7 +1280,11 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
|
|
1280
1280
|
* Basic Types
|
1281
1281
|
*********************************************************/
|
1282
1282
|
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
1283
|
-
#
|
1283
|
+
# if defined(_AIX)
|
1284
|
+
# include <inttypes.h>
|
1285
|
+
# else
|
1286
|
+
# include <stdint.h> /* intptr_t */
|
1287
|
+
# endif
|
1284
1288
|
typedef uint8_t BYTE;
|
1285
1289
|
typedef uint16_t U16;
|
1286
1290
|
typedef int16_t S16;
|
@@ -89,7 +89,11 @@ extern "C" {
|
|
89
89
|
* Basic Types
|
90
90
|
*****************************************************************/
|
91
91
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
92
|
-
#
|
92
|
+
# if defined(_AIX)
|
93
|
+
# include <inttypes.h>
|
94
|
+
# else
|
95
|
+
# include <stdint.h> /* intptr_t */
|
96
|
+
# endif
|
93
97
|
typedef uint8_t BYTE;
|
94
98
|
typedef uint16_t U16;
|
95
99
|
typedef int16_t S16;
|
@@ -90,7 +90,11 @@ extern "C" {
|
|
90
90
|
* Basic Types
|
91
91
|
*****************************************************************/
|
92
92
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
93
|
-
#
|
93
|
+
# if defined(_AIX)
|
94
|
+
# include <inttypes.h>
|
95
|
+
# else
|
96
|
+
# include <stdint.h> /* intptr_t */
|
97
|
+
# endif
|
94
98
|
typedef uint8_t BYTE;
|
95
99
|
typedef uint16_t U16;
|
96
100
|
typedef int16_t S16;
|
@@ -52,7 +52,11 @@ extern "C" {
|
|
52
52
|
* Basic Types
|
53
53
|
*****************************************************************/
|
54
54
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
55
|
-
#
|
55
|
+
# if defined(_AIX)
|
56
|
+
# include <inttypes.h>
|
57
|
+
# else
|
58
|
+
# include <stdint.h> /* intptr_t */
|
59
|
+
# endif
|
56
60
|
typedef uint8_t BYTE;
|
57
61
|
typedef uint16_t U16;
|
58
62
|
typedef int16_t S16;
|
@@ -74,7 +78,7 @@ extern "C" {
|
|
74
78
|
/*-*************************************
|
75
79
|
* Debug
|
76
80
|
***************************************/
|
77
|
-
#include "debug.h"
|
81
|
+
#include "../common/debug.h"
|
78
82
|
#ifndef assert
|
79
83
|
# define assert(condition) ((void)0)
|
80
84
|
#endif
|