extzstd 0.3.1 → 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +28 -14
  3. data/contrib/zstd/CHANGELOG +114 -56
  4. data/contrib/zstd/CONTRIBUTING.md +14 -0
  5. data/contrib/zstd/Makefile +37 -31
  6. data/contrib/zstd/README.md +6 -0
  7. data/contrib/zstd/appveyor.yml +4 -1
  8. data/contrib/zstd/lib/Makefile +231 -134
  9. data/contrib/zstd/lib/README.md +28 -0
  10. data/contrib/zstd/lib/common/bitstream.h +24 -15
  11. data/contrib/zstd/lib/common/compiler.h +116 -3
  12. data/contrib/zstd/lib/common/cpu.h +0 -2
  13. data/contrib/zstd/lib/common/debug.h +11 -18
  14. data/contrib/zstd/lib/common/entropy_common.c +188 -42
  15. data/contrib/zstd/lib/common/error_private.c +1 -0
  16. data/contrib/zstd/lib/common/error_private.h +1 -1
  17. data/contrib/zstd/lib/common/fse.h +38 -11
  18. data/contrib/zstd/lib/common/fse_decompress.c +123 -16
  19. data/contrib/zstd/lib/common/huf.h +26 -5
  20. data/contrib/zstd/lib/common/mem.h +66 -93
  21. data/contrib/zstd/lib/common/pool.c +22 -16
  22. data/contrib/zstd/lib/common/pool.h +1 -1
  23. data/contrib/zstd/lib/common/threading.c +6 -5
  24. data/contrib/zstd/lib/common/xxhash.c +18 -56
  25. data/contrib/zstd/lib/common/xxhash.h +1 -1
  26. data/contrib/zstd/lib/common/zstd_common.c +9 -9
  27. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  28. data/contrib/zstd/lib/common/zstd_errors.h +1 -0
  29. data/contrib/zstd/lib/common/zstd_internal.h +89 -58
  30. data/contrib/zstd/lib/compress/fse_compress.c +30 -23
  31. data/contrib/zstd/lib/compress/hist.c +26 -28
  32. data/contrib/zstd/lib/compress/hist.h +1 -1
  33. data/contrib/zstd/lib/compress/huf_compress.c +210 -95
  34. data/contrib/zstd/lib/compress/zstd_compress.c +1339 -409
  35. data/contrib/zstd/lib/compress/zstd_compress_internal.h +119 -41
  36. data/contrib/zstd/lib/compress/zstd_compress_literals.c +4 -4
  37. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +17 -3
  38. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +23 -19
  39. data/contrib/zstd/lib/compress/zstd_cwksp.h +60 -24
  40. data/contrib/zstd/lib/compress/zstd_double_fast.c +22 -22
  41. data/contrib/zstd/lib/compress/zstd_fast.c +19 -19
  42. data/contrib/zstd/lib/compress/zstd_lazy.c +351 -77
  43. data/contrib/zstd/lib/compress/zstd_lazy.h +20 -0
  44. data/contrib/zstd/lib/compress/zstd_ldm.c +59 -18
  45. data/contrib/zstd/lib/compress/zstd_ldm.h +6 -0
  46. data/contrib/zstd/lib/compress/zstd_opt.c +190 -45
  47. data/contrib/zstd/lib/compress/zstdmt_compress.c +74 -406
  48. data/contrib/zstd/lib/compress/zstdmt_compress.h +26 -108
  49. data/contrib/zstd/lib/decompress/huf_decompress.c +302 -200
  50. data/contrib/zstd/lib/decompress/zstd_ddict.c +8 -8
  51. data/contrib/zstd/lib/decompress/zstd_ddict.h +1 -1
  52. data/contrib/zstd/lib/decompress/zstd_decompress.c +125 -80
  53. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +145 -37
  54. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +5 -2
  55. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +11 -10
  56. data/contrib/zstd/lib/dictBuilder/cover.c +29 -20
  57. data/contrib/zstd/lib/dictBuilder/cover.h +1 -1
  58. data/contrib/zstd/lib/dictBuilder/fastcover.c +20 -19
  59. data/contrib/zstd/lib/dictBuilder/zdict.c +15 -16
  60. data/contrib/zstd/lib/dictBuilder/zdict.h +1 -1
  61. data/contrib/zstd/lib/legacy/zstd_v01.c +5 -1
  62. data/contrib/zstd/lib/legacy/zstd_v02.c +5 -1
  63. data/contrib/zstd/lib/legacy/zstd_v03.c +5 -1
  64. data/contrib/zstd/lib/legacy/zstd_v04.c +6 -2
  65. data/contrib/zstd/lib/legacy/zstd_v05.c +5 -1
  66. data/contrib/zstd/lib/legacy/zstd_v06.c +5 -1
  67. data/contrib/zstd/lib/legacy/zstd_v07.c +5 -1
  68. data/contrib/zstd/lib/libzstd.pc.in +3 -3
  69. data/contrib/zstd/lib/zstd.h +348 -47
  70. data/ext/extzstd.c +6 -0
  71. data/ext/extzstd.h +6 -0
  72. data/gemstub.rb +3 -21
  73. data/lib/extzstd.rb +0 -2
  74. data/lib/extzstd/version.rb +6 -1
  75. data/test/test_basic.rb +0 -5
  76. metadata +5 -4
@@ -40,33 +40,42 @@
40
40
  * Constants
41
41
  ***************************************/
42
42
  #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
43
- #define DEFAULT_SPLITPOINT 1.0
43
+ #define COVER_DEFAULT_SPLITPOINT 1.0
44
44
 
45
45
  /*-*************************************
46
46
  * Console display
47
47
  ***************************************/
48
+ #ifndef LOCALDISPLAYLEVEL
48
49
  static int g_displayLevel = 2;
50
+ #endif
51
+ #undef DISPLAY
49
52
  #define DISPLAY(...) \
50
53
  { \
51
54
  fprintf(stderr, __VA_ARGS__); \
52
55
  fflush(stderr); \
53
56
  }
57
+ #undef LOCALDISPLAYLEVEL
54
58
  #define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
55
59
  if (displayLevel >= l) { \
56
60
  DISPLAY(__VA_ARGS__); \
57
61
  } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
62
+ #undef DISPLAYLEVEL
58
63
  #define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
59
64
 
65
+ #ifndef LOCALDISPLAYUPDATE
66
+ static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
67
+ static clock_t g_time = 0;
68
+ #endif
69
+ #undef LOCALDISPLAYUPDATE
60
70
  #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
61
71
  if (displayLevel >= l) { \
62
- if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
72
+ if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
63
73
  g_time = clock(); \
64
74
  DISPLAY(__VA_ARGS__); \
65
75
  } \
66
76
  }
77
+ #undef DISPLAYUPDATE
67
78
  #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
68
- static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
69
- static clock_t g_time = 0;
70
79
 
71
80
  /*-*************************************
72
81
  * Hash table
@@ -120,9 +129,9 @@ static int COVER_map_init(COVER_map_t *map, U32 size) {
120
129
  /**
121
130
  * Internal hash function
122
131
  */
123
- static const U32 prime4bytes = 2654435761U;
132
+ static const U32 COVER_prime4bytes = 2654435761U;
124
133
  static U32 COVER_map_hash(COVER_map_t *map, U32 key) {
125
- return (key * prime4bytes) >> (32 - map->sizeLog);
134
+ return (key * COVER_prime4bytes) >> (32 - map->sizeLog);
126
135
  }
127
136
 
128
137
  /**
@@ -215,7 +224,7 @@ typedef struct {
215
224
  } COVER_ctx_t;
216
225
 
217
226
  /* We need a global context for qsort... */
218
- static COVER_ctx_t *g_ctx = NULL;
227
+ static COVER_ctx_t *g_coverCtx = NULL;
219
228
 
220
229
  /*-*************************************
221
230
  * Helper functions
@@ -258,11 +267,11 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
258
267
 
259
268
  /**
260
269
  * Same as COVER_cmp() except ties are broken by pointer value
261
- * NOTE: g_ctx must be set to call this function. A global is required because
270
+ * NOTE: g_coverCtx must be set to call this function. A global is required because
262
271
  * qsort doesn't take an opaque pointer.
263
272
  */
264
- static int COVER_strict_cmp(const void *lp, const void *rp) {
265
- int result = COVER_cmp(g_ctx, lp, rp);
273
+ static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
274
+ int result = COVER_cmp(g_coverCtx, lp, rp);
266
275
  if (result == 0) {
267
276
  result = lp < rp ? -1 : 1;
268
277
  }
@@ -271,8 +280,8 @@ static int COVER_strict_cmp(const void *lp, const void *rp) {
271
280
  /**
272
281
  * Faster version for d <= 8.
273
282
  */
274
- static int COVER_strict_cmp8(const void *lp, const void *rp) {
275
- int result = COVER_cmp8(g_ctx, lp, rp);
283
+ static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) {
284
+ int result = COVER_cmp8(g_coverCtx, lp, rp);
276
285
  if (result == 0) {
277
286
  result = lp < rp ? -1 : 1;
278
287
  }
@@ -603,7 +612,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
603
612
  /* qsort doesn't take an opaque pointer, so pass as a global.
604
613
  * On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is.
605
614
  */
606
- g_ctx = ctx;
615
+ g_coverCtx = ctx;
607
616
  #if defined(__OpenBSD__)
608
617
  mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
609
618
  (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
@@ -946,7 +955,7 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection){
946
955
  free(selection.dictContent);
947
956
  }
948
957
 
949
- COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
958
+ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
950
959
  size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
951
960
  size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) {
952
961
 
@@ -954,8 +963,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
954
963
  size_t largestCompressed = 0;
955
964
  BYTE* customDictContentEnd = customDictContent + dictContentSize;
956
965
 
957
- BYTE * largestDictbuffer = (BYTE *)malloc(dictContentSize);
958
- BYTE * candidateDictBuffer = (BYTE *)malloc(dictContentSize);
966
+ BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity);
967
+ BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity);
959
968
  double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00;
960
969
 
961
970
  if (!largestDictbuffer || !candidateDictBuffer) {
@@ -967,7 +976,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
967
976
  /* Initial dictionary size and compressed size */
968
977
  memcpy(largestDictbuffer, customDictContent, dictContentSize);
969
978
  dictContentSize = ZDICT_finalizeDictionary(
970
- largestDictbuffer, dictContentSize, customDictContent, dictContentSize,
979
+ largestDictbuffer, dictBufferCapacity, customDictContent, dictContentSize,
971
980
  samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
972
981
 
973
982
  if (ZDICT_isError(dictContentSize)) {
@@ -1001,7 +1010,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
1001
1010
  while (dictContentSize < largestDict) {
1002
1011
  memcpy(candidateDictBuffer, largestDictbuffer, largestDict);
1003
1012
  dictContentSize = ZDICT_finalizeDictionary(
1004
- candidateDictBuffer, dictContentSize, customDictContentEnd - dictContentSize, dictContentSize,
1013
+ candidateDictBuffer, dictBufferCapacity, customDictContentEnd - dictContentSize, dictContentSize,
1005
1014
  samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
1006
1015
 
1007
1016
  if (ZDICT_isError(dictContentSize)) {
@@ -1079,7 +1088,7 @@ static void COVER_tryParameters(void *opaque) {
1079
1088
  {
1080
1089
  const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
1081
1090
  dictBufferCapacity, parameters);
1082
- selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
1091
+ selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
1083
1092
  ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
1084
1093
  totalCompressedSize);
1085
1094
 
@@ -1106,7 +1115,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
1106
1115
  /* constants */
1107
1116
  const unsigned nbThreads = parameters->nbThreads;
1108
1117
  const double splitPoint =
1109
- parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
1118
+ parameters->splitPoint <= 0.0 ? COVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
1110
1119
  const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
1111
1120
  const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
1112
1121
  const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
@@ -152,6 +152,6 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection);
152
152
  * smallest dictionary within a specified regression of the compressed size
153
153
  * from the largest dictionary.
154
154
  */
155
- COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
155
+ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
156
156
  size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
157
157
  size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
@@ -21,6 +21,7 @@
21
21
  #include "../common/threading.h"
22
22
  #include "cover.h"
23
23
  #include "../common/zstd_internal.h" /* includes zstd.h */
24
+ #include "../compress/zstd_compress_internal.h" /* ZSTD_hash*() */
24
25
  #ifndef ZDICT_STATIC_LINKING_ONLY
25
26
  #define ZDICT_STATIC_LINKING_ONLY
26
27
  #endif
@@ -33,7 +34,7 @@
33
34
  #define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
34
35
  #define FASTCOVER_MAX_F 31
35
36
  #define FASTCOVER_MAX_ACCEL 10
36
- #define DEFAULT_SPLITPOINT 0.75
37
+ #define FASTCOVER_DEFAULT_SPLITPOINT 0.75
37
38
  #define DEFAULT_F 20
38
39
  #define DEFAULT_ACCEL 1
39
40
 
@@ -41,50 +42,50 @@
41
42
  /*-*************************************
42
43
  * Console display
43
44
  ***************************************/
45
+ #ifndef LOCALDISPLAYLEVEL
44
46
  static int g_displayLevel = 2;
47
+ #endif
48
+ #undef DISPLAY
45
49
  #define DISPLAY(...) \
46
50
  { \
47
51
  fprintf(stderr, __VA_ARGS__); \
48
52
  fflush(stderr); \
49
53
  }
54
+ #undef LOCALDISPLAYLEVEL
50
55
  #define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
51
56
  if (displayLevel >= l) { \
52
57
  DISPLAY(__VA_ARGS__); \
53
58
  } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
59
+ #undef DISPLAYLEVEL
54
60
  #define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
55
61
 
62
+ #ifndef LOCALDISPLAYUPDATE
63
+ static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
64
+ static clock_t g_time = 0;
65
+ #endif
66
+ #undef LOCALDISPLAYUPDATE
56
67
  #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
57
68
  if (displayLevel >= l) { \
58
- if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
69
+ if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
59
70
  g_time = clock(); \
60
71
  DISPLAY(__VA_ARGS__); \
61
72
  } \
62
73
  }
74
+ #undef DISPLAYUPDATE
63
75
  #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
64
- static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
65
- static clock_t g_time = 0;
66
76
 
67
77
 
68
78
  /*-*************************************
69
79
  * Hash Functions
70
80
  ***************************************/
71
- static const U64 prime6bytes = 227718039650203ULL;
72
- static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
73
- static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
74
-
75
- static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
76
- static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
77
- static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
78
-
79
-
80
81
  /**
81
- * Hash the d-byte value pointed to by p and mod 2^f
82
+ * Hash the d-byte value pointed to by p and mod 2^f into the frequency vector
82
83
  */
83
- static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) {
84
+ static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 f, unsigned d) {
84
85
  if (d == 6) {
85
- return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1);
86
+ return ZSTD_hash6Ptr(p, f);
86
87
  }
87
- return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1);
88
+ return ZSTD_hash8Ptr(p, f);
88
89
  }
89
90
 
90
91
 
@@ -486,7 +487,7 @@ static void FASTCOVER_tryParameters(void *opaque)
486
487
  parameters, segmentFreqs);
487
488
 
488
489
  const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
489
- selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
490
+ selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
490
491
  ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
491
492
  totalCompressedSize);
492
493
 
@@ -617,7 +618,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
617
618
  /* constants */
618
619
  const unsigned nbThreads = parameters->nbThreads;
619
620
  const double splitPoint =
620
- parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
621
+ parameters->splitPoint <= 0.0 ? FASTCOVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
621
622
  const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
622
623
  const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
623
624
  const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
@@ -62,14 +62,15 @@
62
62
 
63
63
  #define NOISELENGTH 32
64
64
 
65
- static const int g_compressionLevel_default = 3;
66
65
  static const U32 g_selectivity_default = 9;
67
66
 
68
67
 
69
68
  /*-*************************************
70
69
  * Console display
71
70
  ***************************************/
71
+ #undef DISPLAY
72
72
  #define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
73
+ #undef DISPLAYLEVEL
73
74
  #define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
74
75
 
75
76
  static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
@@ -105,20 +106,17 @@ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
105
106
  size_t headerSize;
106
107
  if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);
107
108
 
108
- { unsigned offcodeMaxValue = MaxOff;
109
- ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
109
+ { ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
110
110
  U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE);
111
- short* offcodeNCount = (short*)malloc((MaxOff+1)*sizeof(short));
112
- if (!bs || !wksp || !offcodeNCount) {
111
+ if (!bs || !wksp) {
113
112
  headerSize = ERROR(memory_allocation);
114
113
  } else {
115
114
  ZSTD_reset_compressedBlockState(bs);
116
- headerSize = ZSTD_loadCEntropy(bs, wksp, offcodeNCount, &offcodeMaxValue, dictBuffer, dictSize);
115
+ headerSize = ZSTD_loadCEntropy(bs, wksp, dictBuffer, dictSize);
117
116
  }
118
117
 
119
118
  free(bs);
120
119
  free(wksp);
121
- free(offcodeNCount);
122
120
  }
123
121
 
124
122
  return headerSize;
@@ -532,6 +530,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
532
530
  clock_t displayClock = 0;
533
531
  clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
534
532
 
533
+ # undef DISPLAYUPDATE
535
534
  # define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \
536
535
  if (ZDICT_clockSpan(displayClock) > refreshRate) \
537
536
  { displayClock = clock(); DISPLAY(__VA_ARGS__); \
@@ -706,7 +705,7 @@ static void ZDICT_flatLit(unsigned* countLit)
706
705
 
707
706
  #define OFFCODE_MAX 30 /* only applicable to first block */
708
707
  static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
709
- unsigned compressionLevel,
708
+ int compressionLevel,
710
709
  const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles,
711
710
  const void* dictBuffer, size_t dictBufferSize,
712
711
  unsigned notificationLevel)
@@ -741,7 +740,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
741
740
  memset(repOffset, 0, sizeof(repOffset));
742
741
  repOffset[1] = repOffset[4] = repOffset[8] = 1;
743
742
  memset(bestRepOffset, 0, sizeof(bestRepOffset));
744
- if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
743
+ if (compressionLevel==0) compressionLevel = ZSTD_CLEVEL_DEFAULT;
745
744
  params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
746
745
 
747
746
  esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem);
@@ -786,7 +785,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
786
785
  /* note : the result of this phase should be used to better appreciate the impact on statistics */
787
786
 
788
787
  total=0; for (u=0; u<=offcodeMax; u++) total+=offcodeCount[u];
789
- errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax);
788
+ errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax, /* useLowProbCount */ 1);
790
789
  if (FSE_isError(errorCode)) {
791
790
  eSize = errorCode;
792
791
  DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n");
@@ -795,7 +794,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
795
794
  Offlog = (U32)errorCode;
796
795
 
797
796
  total=0; for (u=0; u<=MaxML; u++) total+=matchLengthCount[u];
798
- errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML);
797
+ errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML, /* useLowProbCount */ 1);
799
798
  if (FSE_isError(errorCode)) {
800
799
  eSize = errorCode;
801
800
  DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n");
@@ -804,7 +803,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
804
803
  mlLog = (U32)errorCode;
805
804
 
806
805
  total=0; for (u=0; u<=MaxLL; u++) total+=litLengthCount[u];
807
- errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL);
806
+ errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL, /* useLowProbCount */ 1);
808
807
  if (FSE_isError(errorCode)) {
809
808
  eSize = errorCode;
810
809
  DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n");
@@ -893,7 +892,7 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
893
892
  size_t hSize;
894
893
  #define HBUFFSIZE 256 /* should prove large enough for all entropy headers */
895
894
  BYTE header[HBUFFSIZE];
896
- int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
895
+ int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel;
897
896
  U32 const notificationLevel = params.notificationLevel;
898
897
 
899
898
  /* check conditions */
@@ -939,7 +938,7 @@ static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
939
938
  const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
940
939
  ZDICT_params_t params)
941
940
  {
942
- int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
941
+ int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel;
943
942
  U32 const notificationLevel = params.notificationLevel;
944
943
  size_t hSize = 8;
945
944
 
@@ -1114,8 +1113,8 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
1114
1113
  memset(&params, 0, sizeof(params));
1115
1114
  params.d = 8;
1116
1115
  params.steps = 4;
1117
- /* Default to level 6 since no compression level information is available */
1118
- params.zParams.compressionLevel = 3;
1116
+ /* Use default level since no compression level information is available */
1117
+ params.zParams.compressionLevel = ZSTD_CLEVEL_DEFAULT;
1119
1118
  #if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1)
1120
1119
  params.zParams.notificationLevel = DEBUGLEVEL;
1121
1120
  #endif
@@ -279,7 +279,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
279
279
  # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
280
280
  # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
281
281
  # define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
282
- # elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
282
+ # elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
283
283
  # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
284
284
  # elif (ZDICT_GCC_VERSION >= 301)
285
285
  # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
@@ -1280,7 +1280,11 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
1280
1280
  * Basic Types
1281
1281
  *********************************************************/
1282
1282
  #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
1283
- # include <stdint.h>
1283
+ # if defined(_AIX)
1284
+ # include <inttypes.h>
1285
+ # else
1286
+ # include <stdint.h> /* intptr_t */
1287
+ # endif
1284
1288
  typedef uint8_t BYTE;
1285
1289
  typedef uint16_t U16;
1286
1290
  typedef int16_t S16;
@@ -89,7 +89,11 @@ extern "C" {
89
89
  * Basic Types
90
90
  *****************************************************************/
91
91
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
92
- # include <stdint.h>
92
+ # if defined(_AIX)
93
+ # include <inttypes.h>
94
+ # else
95
+ # include <stdint.h> /* intptr_t */
96
+ # endif
93
97
  typedef uint8_t BYTE;
94
98
  typedef uint16_t U16;
95
99
  typedef int16_t S16;
@@ -90,7 +90,11 @@ extern "C" {
90
90
  * Basic Types
91
91
  *****************************************************************/
92
92
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
93
- # include <stdint.h>
93
+ # if defined(_AIX)
94
+ # include <inttypes.h>
95
+ # else
96
+ # include <stdint.h> /* intptr_t */
97
+ # endif
94
98
  typedef uint8_t BYTE;
95
99
  typedef uint16_t U16;
96
100
  typedef int16_t S16;
@@ -52,7 +52,11 @@ extern "C" {
52
52
  * Basic Types
53
53
  *****************************************************************/
54
54
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
55
- # include <stdint.h>
55
+ # if defined(_AIX)
56
+ # include <inttypes.h>
57
+ # else
58
+ # include <stdint.h> /* intptr_t */
59
+ # endif
56
60
  typedef uint8_t BYTE;
57
61
  typedef uint16_t U16;
58
62
  typedef int16_t S16;
@@ -74,7 +78,7 @@ extern "C" {
74
78
  /*-*************************************
75
79
  * Debug
76
80
  ***************************************/
77
- #include "debug.h"
81
+ #include "../common/debug.h"
78
82
  #ifndef assert
79
83
  # define assert(condition) ((void)0)
80
84
  #endif