extzstd 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +28 -14
- data/contrib/zstd/CHANGELOG +114 -56
- data/contrib/zstd/CONTRIBUTING.md +14 -0
- data/contrib/zstd/Makefile +37 -31
- data/contrib/zstd/README.md +6 -0
- data/contrib/zstd/appveyor.yml +4 -1
- data/contrib/zstd/lib/Makefile +231 -134
- data/contrib/zstd/lib/README.md +28 -0
- data/contrib/zstd/lib/common/bitstream.h +24 -15
- data/contrib/zstd/lib/common/compiler.h +116 -3
- data/contrib/zstd/lib/common/cpu.h +0 -2
- data/contrib/zstd/lib/common/debug.h +11 -18
- data/contrib/zstd/lib/common/entropy_common.c +188 -42
- data/contrib/zstd/lib/common/error_private.c +1 -0
- data/contrib/zstd/lib/common/error_private.h +1 -1
- data/contrib/zstd/lib/common/fse.h +38 -11
- data/contrib/zstd/lib/common/fse_decompress.c +123 -16
- data/contrib/zstd/lib/common/huf.h +26 -5
- data/contrib/zstd/lib/common/mem.h +66 -93
- data/contrib/zstd/lib/common/pool.c +22 -16
- data/contrib/zstd/lib/common/pool.h +1 -1
- data/contrib/zstd/lib/common/threading.c +6 -5
- data/contrib/zstd/lib/common/xxhash.c +18 -56
- data/contrib/zstd/lib/common/xxhash.h +1 -1
- data/contrib/zstd/lib/common/zstd_common.c +9 -9
- data/contrib/zstd/lib/common/zstd_deps.h +111 -0
- data/contrib/zstd/lib/common/zstd_errors.h +1 -0
- data/contrib/zstd/lib/common/zstd_internal.h +89 -58
- data/contrib/zstd/lib/compress/fse_compress.c +30 -23
- data/contrib/zstd/lib/compress/hist.c +26 -28
- data/contrib/zstd/lib/compress/hist.h +1 -1
- data/contrib/zstd/lib/compress/huf_compress.c +210 -95
- data/contrib/zstd/lib/compress/zstd_compress.c +1339 -409
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +119 -41
- data/contrib/zstd/lib/compress/zstd_compress_literals.c +4 -4
- data/contrib/zstd/lib/compress/zstd_compress_sequences.c +17 -3
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +23 -19
- data/contrib/zstd/lib/compress/zstd_cwksp.h +60 -24
- data/contrib/zstd/lib/compress/zstd_double_fast.c +22 -22
- data/contrib/zstd/lib/compress/zstd_fast.c +19 -19
- data/contrib/zstd/lib/compress/zstd_lazy.c +351 -77
- data/contrib/zstd/lib/compress/zstd_lazy.h +20 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +59 -18
- data/contrib/zstd/lib/compress/zstd_ldm.h +6 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +190 -45
- data/contrib/zstd/lib/compress/zstdmt_compress.c +74 -406
- data/contrib/zstd/lib/compress/zstdmt_compress.h +26 -108
- data/contrib/zstd/lib/decompress/huf_decompress.c +302 -200
- data/contrib/zstd/lib/decompress/zstd_ddict.c +8 -8
- data/contrib/zstd/lib/decompress/zstd_ddict.h +1 -1
- data/contrib/zstd/lib/decompress/zstd_decompress.c +125 -80
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +145 -37
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +5 -2
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +11 -10
- data/contrib/zstd/lib/dictBuilder/cover.c +29 -20
- data/contrib/zstd/lib/dictBuilder/cover.h +1 -1
- data/contrib/zstd/lib/dictBuilder/fastcover.c +20 -19
- data/contrib/zstd/lib/dictBuilder/zdict.c +15 -16
- data/contrib/zstd/lib/dictBuilder/zdict.h +1 -1
- data/contrib/zstd/lib/legacy/zstd_v01.c +5 -1
- data/contrib/zstd/lib/legacy/zstd_v02.c +5 -1
- data/contrib/zstd/lib/legacy/zstd_v03.c +5 -1
- data/contrib/zstd/lib/legacy/zstd_v04.c +6 -2
- data/contrib/zstd/lib/legacy/zstd_v05.c +5 -1
- data/contrib/zstd/lib/legacy/zstd_v06.c +5 -1
- data/contrib/zstd/lib/legacy/zstd_v07.c +5 -1
- data/contrib/zstd/lib/libzstd.pc.in +3 -3
- data/contrib/zstd/lib/zstd.h +348 -47
- data/ext/extzstd.c +6 -0
- data/ext/extzstd.h +6 -0
- data/gemstub.rb +3 -21
- data/lib/extzstd.rb +0 -2
- data/lib/extzstd/version.rb +6 -1
- data/test/test_basic.rb +0 -5
- metadata +5 -4
@@ -40,33 +40,42 @@
|
|
40
40
|
* Constants
|
41
41
|
***************************************/
|
42
42
|
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
|
43
|
-
#define
|
43
|
+
#define COVER_DEFAULT_SPLITPOINT 1.0
|
44
44
|
|
45
45
|
/*-*************************************
|
46
46
|
* Console display
|
47
47
|
***************************************/
|
48
|
+
#ifndef LOCALDISPLAYLEVEL
|
48
49
|
static int g_displayLevel = 2;
|
50
|
+
#endif
|
51
|
+
#undef DISPLAY
|
49
52
|
#define DISPLAY(...) \
|
50
53
|
{ \
|
51
54
|
fprintf(stderr, __VA_ARGS__); \
|
52
55
|
fflush(stderr); \
|
53
56
|
}
|
57
|
+
#undef LOCALDISPLAYLEVEL
|
54
58
|
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
|
55
59
|
if (displayLevel >= l) { \
|
56
60
|
DISPLAY(__VA_ARGS__); \
|
57
61
|
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
62
|
+
#undef DISPLAYLEVEL
|
58
63
|
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
|
59
64
|
|
65
|
+
#ifndef LOCALDISPLAYUPDATE
|
66
|
+
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
67
|
+
static clock_t g_time = 0;
|
68
|
+
#endif
|
69
|
+
#undef LOCALDISPLAYUPDATE
|
60
70
|
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
61
71
|
if (displayLevel >= l) { \
|
62
|
-
if ((clock() - g_time >
|
72
|
+
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
|
63
73
|
g_time = clock(); \
|
64
74
|
DISPLAY(__VA_ARGS__); \
|
65
75
|
} \
|
66
76
|
}
|
77
|
+
#undef DISPLAYUPDATE
|
67
78
|
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
|
68
|
-
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
69
|
-
static clock_t g_time = 0;
|
70
79
|
|
71
80
|
/*-*************************************
|
72
81
|
* Hash table
|
@@ -120,9 +129,9 @@ static int COVER_map_init(COVER_map_t *map, U32 size) {
|
|
120
129
|
/**
|
121
130
|
* Internal hash function
|
122
131
|
*/
|
123
|
-
static const U32
|
132
|
+
static const U32 COVER_prime4bytes = 2654435761U;
|
124
133
|
static U32 COVER_map_hash(COVER_map_t *map, U32 key) {
|
125
|
-
return (key *
|
134
|
+
return (key * COVER_prime4bytes) >> (32 - map->sizeLog);
|
126
135
|
}
|
127
136
|
|
128
137
|
/**
|
@@ -215,7 +224,7 @@ typedef struct {
|
|
215
224
|
} COVER_ctx_t;
|
216
225
|
|
217
226
|
/* We need a global context for qsort... */
|
218
|
-
static COVER_ctx_t *
|
227
|
+
static COVER_ctx_t *g_coverCtx = NULL;
|
219
228
|
|
220
229
|
/*-*************************************
|
221
230
|
* Helper functions
|
@@ -258,11 +267,11 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
|
|
258
267
|
|
259
268
|
/**
|
260
269
|
* Same as COVER_cmp() except ties are broken by pointer value
|
261
|
-
* NOTE:
|
270
|
+
* NOTE: g_coverCtx must be set to call this function. A global is required because
|
262
271
|
* qsort doesn't take an opaque pointer.
|
263
272
|
*/
|
264
|
-
static int COVER_strict_cmp(const void *lp, const void *rp) {
|
265
|
-
int result = COVER_cmp(
|
273
|
+
static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
|
274
|
+
int result = COVER_cmp(g_coverCtx, lp, rp);
|
266
275
|
if (result == 0) {
|
267
276
|
result = lp < rp ? -1 : 1;
|
268
277
|
}
|
@@ -271,8 +280,8 @@ static int COVER_strict_cmp(const void *lp, const void *rp) {
|
|
271
280
|
/**
|
272
281
|
* Faster version for d <= 8.
|
273
282
|
*/
|
274
|
-
static int COVER_strict_cmp8(const void *lp, const void *rp) {
|
275
|
-
int result = COVER_cmp8(
|
283
|
+
static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) {
|
284
|
+
int result = COVER_cmp8(g_coverCtx, lp, rp);
|
276
285
|
if (result == 0) {
|
277
286
|
result = lp < rp ? -1 : 1;
|
278
287
|
}
|
@@ -603,7 +612,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
|
603
612
|
/* qsort doesn't take an opaque pointer, so pass as a global.
|
604
613
|
* On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is.
|
605
614
|
*/
|
606
|
-
|
615
|
+
g_coverCtx = ctx;
|
607
616
|
#if defined(__OpenBSD__)
|
608
617
|
mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
|
609
618
|
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
|
@@ -946,7 +955,7 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection){
|
|
946
955
|
free(selection.dictContent);
|
947
956
|
}
|
948
957
|
|
949
|
-
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
958
|
+
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
|
950
959
|
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
|
951
960
|
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) {
|
952
961
|
|
@@ -954,8 +963,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
|
954
963
|
size_t largestCompressed = 0;
|
955
964
|
BYTE* customDictContentEnd = customDictContent + dictContentSize;
|
956
965
|
|
957
|
-
BYTE * largestDictbuffer = (BYTE *)malloc(
|
958
|
-
BYTE * candidateDictBuffer = (BYTE *)malloc(
|
966
|
+
BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity);
|
967
|
+
BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity);
|
959
968
|
double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00;
|
960
969
|
|
961
970
|
if (!largestDictbuffer || !candidateDictBuffer) {
|
@@ -967,7 +976,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
|
967
976
|
/* Initial dictionary size and compressed size */
|
968
977
|
memcpy(largestDictbuffer, customDictContent, dictContentSize);
|
969
978
|
dictContentSize = ZDICT_finalizeDictionary(
|
970
|
-
largestDictbuffer,
|
979
|
+
largestDictbuffer, dictBufferCapacity, customDictContent, dictContentSize,
|
971
980
|
samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
|
972
981
|
|
973
982
|
if (ZDICT_isError(dictContentSize)) {
|
@@ -1001,7 +1010,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
|
1001
1010
|
while (dictContentSize < largestDict) {
|
1002
1011
|
memcpy(candidateDictBuffer, largestDictbuffer, largestDict);
|
1003
1012
|
dictContentSize = ZDICT_finalizeDictionary(
|
1004
|
-
candidateDictBuffer,
|
1013
|
+
candidateDictBuffer, dictBufferCapacity, customDictContentEnd - dictContentSize, dictContentSize,
|
1005
1014
|
samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
|
1006
1015
|
|
1007
1016
|
if (ZDICT_isError(dictContentSize)) {
|
@@ -1079,7 +1088,7 @@ static void COVER_tryParameters(void *opaque) {
|
|
1079
1088
|
{
|
1080
1089
|
const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
|
1081
1090
|
dictBufferCapacity, parameters);
|
1082
|
-
selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
|
1091
|
+
selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
|
1083
1092
|
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
|
1084
1093
|
totalCompressedSize);
|
1085
1094
|
|
@@ -1106,7 +1115,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
|
1106
1115
|
/* constants */
|
1107
1116
|
const unsigned nbThreads = parameters->nbThreads;
|
1108
1117
|
const double splitPoint =
|
1109
|
-
parameters->splitPoint <= 0.0 ?
|
1118
|
+
parameters->splitPoint <= 0.0 ? COVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
|
1110
1119
|
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
|
1111
1120
|
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
|
1112
1121
|
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
|
@@ -152,6 +152,6 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection);
|
|
152
152
|
* smallest dictionary within a specified regression of the compressed size
|
153
153
|
* from the largest dictionary.
|
154
154
|
*/
|
155
|
-
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
|
155
|
+
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
|
156
156
|
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
|
157
157
|
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
|
@@ -21,6 +21,7 @@
|
|
21
21
|
#include "../common/threading.h"
|
22
22
|
#include "cover.h"
|
23
23
|
#include "../common/zstd_internal.h" /* includes zstd.h */
|
24
|
+
#include "../compress/zstd_compress_internal.h" /* ZSTD_hash*() */
|
24
25
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
25
26
|
#define ZDICT_STATIC_LINKING_ONLY
|
26
27
|
#endif
|
@@ -33,7 +34,7 @@
|
|
33
34
|
#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
|
34
35
|
#define FASTCOVER_MAX_F 31
|
35
36
|
#define FASTCOVER_MAX_ACCEL 10
|
36
|
-
#define
|
37
|
+
#define FASTCOVER_DEFAULT_SPLITPOINT 0.75
|
37
38
|
#define DEFAULT_F 20
|
38
39
|
#define DEFAULT_ACCEL 1
|
39
40
|
|
@@ -41,50 +42,50 @@
|
|
41
42
|
/*-*************************************
|
42
43
|
* Console display
|
43
44
|
***************************************/
|
45
|
+
#ifndef LOCALDISPLAYLEVEL
|
44
46
|
static int g_displayLevel = 2;
|
47
|
+
#endif
|
48
|
+
#undef DISPLAY
|
45
49
|
#define DISPLAY(...) \
|
46
50
|
{ \
|
47
51
|
fprintf(stderr, __VA_ARGS__); \
|
48
52
|
fflush(stderr); \
|
49
53
|
}
|
54
|
+
#undef LOCALDISPLAYLEVEL
|
50
55
|
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
|
51
56
|
if (displayLevel >= l) { \
|
52
57
|
DISPLAY(__VA_ARGS__); \
|
53
58
|
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
59
|
+
#undef DISPLAYLEVEL
|
54
60
|
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
|
55
61
|
|
62
|
+
#ifndef LOCALDISPLAYUPDATE
|
63
|
+
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
64
|
+
static clock_t g_time = 0;
|
65
|
+
#endif
|
66
|
+
#undef LOCALDISPLAYUPDATE
|
56
67
|
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
57
68
|
if (displayLevel >= l) { \
|
58
|
-
if ((clock() - g_time >
|
69
|
+
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
|
59
70
|
g_time = clock(); \
|
60
71
|
DISPLAY(__VA_ARGS__); \
|
61
72
|
} \
|
62
73
|
}
|
74
|
+
#undef DISPLAYUPDATE
|
63
75
|
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
|
64
|
-
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
65
|
-
static clock_t g_time = 0;
|
66
76
|
|
67
77
|
|
68
78
|
/*-*************************************
|
69
79
|
* Hash Functions
|
70
80
|
***************************************/
|
71
|
-
static const U64 prime6bytes = 227718039650203ULL;
|
72
|
-
static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
|
73
|
-
static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
|
74
|
-
|
75
|
-
static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
|
76
|
-
static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
|
77
|
-
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
|
78
|
-
|
79
|
-
|
80
81
|
/**
|
81
|
-
* Hash the d-byte value pointed to by p and mod 2^f
|
82
|
+
* Hash the d-byte value pointed to by p and mod 2^f into the frequency vector
|
82
83
|
*/
|
83
|
-
static size_t FASTCOVER_hashPtrToIndex(const void* p, U32
|
84
|
+
static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 f, unsigned d) {
|
84
85
|
if (d == 6) {
|
85
|
-
return ZSTD_hash6Ptr(p,
|
86
|
+
return ZSTD_hash6Ptr(p, f);
|
86
87
|
}
|
87
|
-
return ZSTD_hash8Ptr(p,
|
88
|
+
return ZSTD_hash8Ptr(p, f);
|
88
89
|
}
|
89
90
|
|
90
91
|
|
@@ -486,7 +487,7 @@ static void FASTCOVER_tryParameters(void *opaque)
|
|
486
487
|
parameters, segmentFreqs);
|
487
488
|
|
488
489
|
const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
|
489
|
-
selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
|
490
|
+
selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
|
490
491
|
ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
|
491
492
|
totalCompressedSize);
|
492
493
|
|
@@ -617,7 +618,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
|
|
617
618
|
/* constants */
|
618
619
|
const unsigned nbThreads = parameters->nbThreads;
|
619
620
|
const double splitPoint =
|
620
|
-
parameters->splitPoint <= 0.0 ?
|
621
|
+
parameters->splitPoint <= 0.0 ? FASTCOVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
|
621
622
|
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
|
622
623
|
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
|
623
624
|
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
|
@@ -62,14 +62,15 @@
|
|
62
62
|
|
63
63
|
#define NOISELENGTH 32
|
64
64
|
|
65
|
-
static const int g_compressionLevel_default = 3;
|
66
65
|
static const U32 g_selectivity_default = 9;
|
67
66
|
|
68
67
|
|
69
68
|
/*-*************************************
|
70
69
|
* Console display
|
71
70
|
***************************************/
|
71
|
+
#undef DISPLAY
|
72
72
|
#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
|
73
|
+
#undef DISPLAYLEVEL
|
73
74
|
#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
74
75
|
|
75
76
|
static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
|
@@ -105,20 +106,17 @@ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
|
|
105
106
|
size_t headerSize;
|
106
107
|
if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);
|
107
108
|
|
108
|
-
{
|
109
|
-
ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
|
109
|
+
{ ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
|
110
110
|
U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE);
|
111
|
-
|
112
|
-
if (!bs || !wksp || !offcodeNCount) {
|
111
|
+
if (!bs || !wksp) {
|
113
112
|
headerSize = ERROR(memory_allocation);
|
114
113
|
} else {
|
115
114
|
ZSTD_reset_compressedBlockState(bs);
|
116
|
-
headerSize = ZSTD_loadCEntropy(bs, wksp,
|
115
|
+
headerSize = ZSTD_loadCEntropy(bs, wksp, dictBuffer, dictSize);
|
117
116
|
}
|
118
117
|
|
119
118
|
free(bs);
|
120
119
|
free(wksp);
|
121
|
-
free(offcodeNCount);
|
122
120
|
}
|
123
121
|
|
124
122
|
return headerSize;
|
@@ -532,6 +530,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
|
|
532
530
|
clock_t displayClock = 0;
|
533
531
|
clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
|
534
532
|
|
533
|
+
# undef DISPLAYUPDATE
|
535
534
|
# define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \
|
536
535
|
if (ZDICT_clockSpan(displayClock) > refreshRate) \
|
537
536
|
{ displayClock = clock(); DISPLAY(__VA_ARGS__); \
|
@@ -706,7 +705,7 @@ static void ZDICT_flatLit(unsigned* countLit)
|
|
706
705
|
|
707
706
|
#define OFFCODE_MAX 30 /* only applicable to first block */
|
708
707
|
static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
709
|
-
|
708
|
+
int compressionLevel,
|
710
709
|
const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles,
|
711
710
|
const void* dictBuffer, size_t dictBufferSize,
|
712
711
|
unsigned notificationLevel)
|
@@ -741,7 +740,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
741
740
|
memset(repOffset, 0, sizeof(repOffset));
|
742
741
|
repOffset[1] = repOffset[4] = repOffset[8] = 1;
|
743
742
|
memset(bestRepOffset, 0, sizeof(bestRepOffset));
|
744
|
-
if (compressionLevel==0) compressionLevel =
|
743
|
+
if (compressionLevel==0) compressionLevel = ZSTD_CLEVEL_DEFAULT;
|
745
744
|
params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
|
746
745
|
|
747
746
|
esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem);
|
@@ -786,7 +785,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
786
785
|
/* note : the result of this phase should be used to better appreciate the impact on statistics */
|
787
786
|
|
788
787
|
total=0; for (u=0; u<=offcodeMax; u++) total+=offcodeCount[u];
|
789
|
-
errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax);
|
788
|
+
errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax, /* useLowProbCount */ 1);
|
790
789
|
if (FSE_isError(errorCode)) {
|
791
790
|
eSize = errorCode;
|
792
791
|
DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n");
|
@@ -795,7 +794,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
795
794
|
Offlog = (U32)errorCode;
|
796
795
|
|
797
796
|
total=0; for (u=0; u<=MaxML; u++) total+=matchLengthCount[u];
|
798
|
-
errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML);
|
797
|
+
errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML, /* useLowProbCount */ 1);
|
799
798
|
if (FSE_isError(errorCode)) {
|
800
799
|
eSize = errorCode;
|
801
800
|
DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n");
|
@@ -804,7 +803,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
804
803
|
mlLog = (U32)errorCode;
|
805
804
|
|
806
805
|
total=0; for (u=0; u<=MaxLL; u++) total+=litLengthCount[u];
|
807
|
-
errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL);
|
806
|
+
errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL, /* useLowProbCount */ 1);
|
808
807
|
if (FSE_isError(errorCode)) {
|
809
808
|
eSize = errorCode;
|
810
809
|
DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n");
|
@@ -893,7 +892,7 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
|
893
892
|
size_t hSize;
|
894
893
|
#define HBUFFSIZE 256 /* should prove large enough for all entropy headers */
|
895
894
|
BYTE header[HBUFFSIZE];
|
896
|
-
int const compressionLevel = (params.compressionLevel == 0) ?
|
895
|
+
int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel;
|
897
896
|
U32 const notificationLevel = params.notificationLevel;
|
898
897
|
|
899
898
|
/* check conditions */
|
@@ -939,7 +938,7 @@ static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
|
|
939
938
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
940
939
|
ZDICT_params_t params)
|
941
940
|
{
|
942
|
-
int const compressionLevel = (params.compressionLevel == 0) ?
|
941
|
+
int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel;
|
943
942
|
U32 const notificationLevel = params.notificationLevel;
|
944
943
|
size_t hSize = 8;
|
945
944
|
|
@@ -1114,8 +1113,8 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
|
1114
1113
|
memset(¶ms, 0, sizeof(params));
|
1115
1114
|
params.d = 8;
|
1116
1115
|
params.steps = 4;
|
1117
|
-
/*
|
1118
|
-
params.zParams.compressionLevel =
|
1116
|
+
/* Use default level since no compression level information is available */
|
1117
|
+
params.zParams.compressionLevel = ZSTD_CLEVEL_DEFAULT;
|
1119
1118
|
#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1)
|
1120
1119
|
params.zParams.notificationLevel = DEBUGLEVEL;
|
1121
1120
|
#endif
|
@@ -279,7 +279,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
|
|
279
279
|
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
280
280
|
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
281
281
|
# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
|
282
|
-
# elif (ZDICT_GCC_VERSION >= 405)
|
282
|
+
# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
|
283
283
|
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
|
284
284
|
# elif (ZDICT_GCC_VERSION >= 301)
|
285
285
|
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
|
@@ -1280,7 +1280,11 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
|
|
1280
1280
|
* Basic Types
|
1281
1281
|
*********************************************************/
|
1282
1282
|
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
1283
|
-
#
|
1283
|
+
# if defined(_AIX)
|
1284
|
+
# include <inttypes.h>
|
1285
|
+
# else
|
1286
|
+
# include <stdint.h> /* intptr_t */
|
1287
|
+
# endif
|
1284
1288
|
typedef uint8_t BYTE;
|
1285
1289
|
typedef uint16_t U16;
|
1286
1290
|
typedef int16_t S16;
|
@@ -89,7 +89,11 @@ extern "C" {
|
|
89
89
|
* Basic Types
|
90
90
|
*****************************************************************/
|
91
91
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
92
|
-
#
|
92
|
+
# if defined(_AIX)
|
93
|
+
# include <inttypes.h>
|
94
|
+
# else
|
95
|
+
# include <stdint.h> /* intptr_t */
|
96
|
+
# endif
|
93
97
|
typedef uint8_t BYTE;
|
94
98
|
typedef uint16_t U16;
|
95
99
|
typedef int16_t S16;
|
@@ -90,7 +90,11 @@ extern "C" {
|
|
90
90
|
* Basic Types
|
91
91
|
*****************************************************************/
|
92
92
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
93
|
-
#
|
93
|
+
# if defined(_AIX)
|
94
|
+
# include <inttypes.h>
|
95
|
+
# else
|
96
|
+
# include <stdint.h> /* intptr_t */
|
97
|
+
# endif
|
94
98
|
typedef uint8_t BYTE;
|
95
99
|
typedef uint16_t U16;
|
96
100
|
typedef int16_t S16;
|
@@ -52,7 +52,11 @@ extern "C" {
|
|
52
52
|
* Basic Types
|
53
53
|
*****************************************************************/
|
54
54
|
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
55
|
-
#
|
55
|
+
# if defined(_AIX)
|
56
|
+
# include <inttypes.h>
|
57
|
+
# else
|
58
|
+
# include <stdint.h> /* intptr_t */
|
59
|
+
# endif
|
56
60
|
typedef uint8_t BYTE;
|
57
61
|
typedef uint16_t U16;
|
58
62
|
typedef int16_t S16;
|
@@ -74,7 +78,7 @@ extern "C" {
|
|
74
78
|
/*-*************************************
|
75
79
|
* Debug
|
76
80
|
***************************************/
|
77
|
-
#include "debug.h"
|
81
|
+
#include "../common/debug.h"
|
78
82
|
#ifndef assert
|
79
83
|
# define assert(condition) ((void)0)
|
80
84
|
#endif
|