extzstd 0.3.1 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +28 -14
  3. data/contrib/zstd/CHANGELOG +114 -56
  4. data/contrib/zstd/CONTRIBUTING.md +14 -0
  5. data/contrib/zstd/Makefile +37 -31
  6. data/contrib/zstd/README.md +6 -0
  7. data/contrib/zstd/appveyor.yml +4 -1
  8. data/contrib/zstd/lib/Makefile +231 -134
  9. data/contrib/zstd/lib/README.md +28 -0
  10. data/contrib/zstd/lib/common/bitstream.h +24 -15
  11. data/contrib/zstd/lib/common/compiler.h +116 -3
  12. data/contrib/zstd/lib/common/cpu.h +0 -2
  13. data/contrib/zstd/lib/common/debug.h +11 -18
  14. data/contrib/zstd/lib/common/entropy_common.c +188 -42
  15. data/contrib/zstd/lib/common/error_private.c +1 -0
  16. data/contrib/zstd/lib/common/error_private.h +1 -1
  17. data/contrib/zstd/lib/common/fse.h +38 -11
  18. data/contrib/zstd/lib/common/fse_decompress.c +123 -16
  19. data/contrib/zstd/lib/common/huf.h +26 -5
  20. data/contrib/zstd/lib/common/mem.h +66 -93
  21. data/contrib/zstd/lib/common/pool.c +22 -16
  22. data/contrib/zstd/lib/common/pool.h +1 -1
  23. data/contrib/zstd/lib/common/threading.c +6 -5
  24. data/contrib/zstd/lib/common/xxhash.c +18 -56
  25. data/contrib/zstd/lib/common/xxhash.h +1 -1
  26. data/contrib/zstd/lib/common/zstd_common.c +9 -9
  27. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  28. data/contrib/zstd/lib/common/zstd_errors.h +1 -0
  29. data/contrib/zstd/lib/common/zstd_internal.h +89 -58
  30. data/contrib/zstd/lib/compress/fse_compress.c +30 -23
  31. data/contrib/zstd/lib/compress/hist.c +26 -28
  32. data/contrib/zstd/lib/compress/hist.h +1 -1
  33. data/contrib/zstd/lib/compress/huf_compress.c +210 -95
  34. data/contrib/zstd/lib/compress/zstd_compress.c +1339 -409
  35. data/contrib/zstd/lib/compress/zstd_compress_internal.h +119 -41
  36. data/contrib/zstd/lib/compress/zstd_compress_literals.c +4 -4
  37. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +17 -3
  38. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +23 -19
  39. data/contrib/zstd/lib/compress/zstd_cwksp.h +60 -24
  40. data/contrib/zstd/lib/compress/zstd_double_fast.c +22 -22
  41. data/contrib/zstd/lib/compress/zstd_fast.c +19 -19
  42. data/contrib/zstd/lib/compress/zstd_lazy.c +351 -77
  43. data/contrib/zstd/lib/compress/zstd_lazy.h +20 -0
  44. data/contrib/zstd/lib/compress/zstd_ldm.c +59 -18
  45. data/contrib/zstd/lib/compress/zstd_ldm.h +6 -0
  46. data/contrib/zstd/lib/compress/zstd_opt.c +190 -45
  47. data/contrib/zstd/lib/compress/zstdmt_compress.c +74 -406
  48. data/contrib/zstd/lib/compress/zstdmt_compress.h +26 -108
  49. data/contrib/zstd/lib/decompress/huf_decompress.c +302 -200
  50. data/contrib/zstd/lib/decompress/zstd_ddict.c +8 -8
  51. data/contrib/zstd/lib/decompress/zstd_ddict.h +1 -1
  52. data/contrib/zstd/lib/decompress/zstd_decompress.c +125 -80
  53. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +145 -37
  54. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +5 -2
  55. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +11 -10
  56. data/contrib/zstd/lib/dictBuilder/cover.c +29 -20
  57. data/contrib/zstd/lib/dictBuilder/cover.h +1 -1
  58. data/contrib/zstd/lib/dictBuilder/fastcover.c +20 -19
  59. data/contrib/zstd/lib/dictBuilder/zdict.c +15 -16
  60. data/contrib/zstd/lib/dictBuilder/zdict.h +1 -1
  61. data/contrib/zstd/lib/legacy/zstd_v01.c +5 -1
  62. data/contrib/zstd/lib/legacy/zstd_v02.c +5 -1
  63. data/contrib/zstd/lib/legacy/zstd_v03.c +5 -1
  64. data/contrib/zstd/lib/legacy/zstd_v04.c +6 -2
  65. data/contrib/zstd/lib/legacy/zstd_v05.c +5 -1
  66. data/contrib/zstd/lib/legacy/zstd_v06.c +5 -1
  67. data/contrib/zstd/lib/legacy/zstd_v07.c +5 -1
  68. data/contrib/zstd/lib/libzstd.pc.in +3 -3
  69. data/contrib/zstd/lib/zstd.h +348 -47
  70. data/ext/extzstd.c +6 -0
  71. data/ext/extzstd.h +6 -0
  72. data/gemstub.rb +3 -21
  73. data/lib/extzstd.rb +0 -2
  74. data/lib/extzstd/version.rb +6 -1
  75. data/test/test_basic.rb +0 -5
  76. metadata +5 -4
@@ -40,33 +40,42 @@
40
40
  * Constants
41
41
  ***************************************/
42
42
  #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
43
- #define DEFAULT_SPLITPOINT 1.0
43
+ #define COVER_DEFAULT_SPLITPOINT 1.0
44
44
 
45
45
  /*-*************************************
46
46
  * Console display
47
47
  ***************************************/
48
+ #ifndef LOCALDISPLAYLEVEL
48
49
  static int g_displayLevel = 2;
50
+ #endif
51
+ #undef DISPLAY
49
52
  #define DISPLAY(...) \
50
53
  { \
51
54
  fprintf(stderr, __VA_ARGS__); \
52
55
  fflush(stderr); \
53
56
  }
57
+ #undef LOCALDISPLAYLEVEL
54
58
  #define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
55
59
  if (displayLevel >= l) { \
56
60
  DISPLAY(__VA_ARGS__); \
57
61
  } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
62
+ #undef DISPLAYLEVEL
58
63
  #define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
59
64
 
65
+ #ifndef LOCALDISPLAYUPDATE
66
+ static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
67
+ static clock_t g_time = 0;
68
+ #endif
69
+ #undef LOCALDISPLAYUPDATE
60
70
  #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
61
71
  if (displayLevel >= l) { \
62
- if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
72
+ if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
63
73
  g_time = clock(); \
64
74
  DISPLAY(__VA_ARGS__); \
65
75
  } \
66
76
  }
77
+ #undef DISPLAYUPDATE
67
78
  #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
68
- static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
69
- static clock_t g_time = 0;
70
79
 
71
80
  /*-*************************************
72
81
  * Hash table
@@ -120,9 +129,9 @@ static int COVER_map_init(COVER_map_t *map, U32 size) {
120
129
  /**
121
130
  * Internal hash function
122
131
  */
123
- static const U32 prime4bytes = 2654435761U;
132
+ static const U32 COVER_prime4bytes = 2654435761U;
124
133
  static U32 COVER_map_hash(COVER_map_t *map, U32 key) {
125
- return (key * prime4bytes) >> (32 - map->sizeLog);
134
+ return (key * COVER_prime4bytes) >> (32 - map->sizeLog);
126
135
  }
127
136
 
128
137
  /**
@@ -215,7 +224,7 @@ typedef struct {
215
224
  } COVER_ctx_t;
216
225
 
217
226
  /* We need a global context for qsort... */
218
- static COVER_ctx_t *g_ctx = NULL;
227
+ static COVER_ctx_t *g_coverCtx = NULL;
219
228
 
220
229
  /*-*************************************
221
230
  * Helper functions
@@ -258,11 +267,11 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
258
267
 
259
268
  /**
260
269
  * Same as COVER_cmp() except ties are broken by pointer value
261
- * NOTE: g_ctx must be set to call this function. A global is required because
270
+ * NOTE: g_coverCtx must be set to call this function. A global is required because
262
271
  * qsort doesn't take an opaque pointer.
263
272
  */
264
- static int COVER_strict_cmp(const void *lp, const void *rp) {
265
- int result = COVER_cmp(g_ctx, lp, rp);
273
+ static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
274
+ int result = COVER_cmp(g_coverCtx, lp, rp);
266
275
  if (result == 0) {
267
276
  result = lp < rp ? -1 : 1;
268
277
  }
@@ -271,8 +280,8 @@ static int COVER_strict_cmp(const void *lp, const void *rp) {
271
280
  /**
272
281
  * Faster version for d <= 8.
273
282
  */
274
- static int COVER_strict_cmp8(const void *lp, const void *rp) {
275
- int result = COVER_cmp8(g_ctx, lp, rp);
283
+ static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) {
284
+ int result = COVER_cmp8(g_coverCtx, lp, rp);
276
285
  if (result == 0) {
277
286
  result = lp < rp ? -1 : 1;
278
287
  }
@@ -603,7 +612,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
603
612
  /* qsort doesn't take an opaque pointer, so pass as a global.
604
613
  * On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is.
605
614
  */
606
- g_ctx = ctx;
615
+ g_coverCtx = ctx;
607
616
  #if defined(__OpenBSD__)
608
617
  mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
609
618
  (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
@@ -946,7 +955,7 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection){
946
955
  free(selection.dictContent);
947
956
  }
948
957
 
949
- COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
958
+ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
950
959
  size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
951
960
  size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) {
952
961
 
@@ -954,8 +963,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
954
963
  size_t largestCompressed = 0;
955
964
  BYTE* customDictContentEnd = customDictContent + dictContentSize;
956
965
 
957
- BYTE * largestDictbuffer = (BYTE *)malloc(dictContentSize);
958
- BYTE * candidateDictBuffer = (BYTE *)malloc(dictContentSize);
966
+ BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity);
967
+ BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity);
959
968
  double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00;
960
969
 
961
970
  if (!largestDictbuffer || !candidateDictBuffer) {
@@ -967,7 +976,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
967
976
  /* Initial dictionary size and compressed size */
968
977
  memcpy(largestDictbuffer, customDictContent, dictContentSize);
969
978
  dictContentSize = ZDICT_finalizeDictionary(
970
- largestDictbuffer, dictContentSize, customDictContent, dictContentSize,
979
+ largestDictbuffer, dictBufferCapacity, customDictContent, dictContentSize,
971
980
  samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
972
981
 
973
982
  if (ZDICT_isError(dictContentSize)) {
@@ -1001,7 +1010,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
1001
1010
  while (dictContentSize < largestDict) {
1002
1011
  memcpy(candidateDictBuffer, largestDictbuffer, largestDict);
1003
1012
  dictContentSize = ZDICT_finalizeDictionary(
1004
- candidateDictBuffer, dictContentSize, customDictContentEnd - dictContentSize, dictContentSize,
1013
+ candidateDictBuffer, dictBufferCapacity, customDictContentEnd - dictContentSize, dictContentSize,
1005
1014
  samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
1006
1015
 
1007
1016
  if (ZDICT_isError(dictContentSize)) {
@@ -1079,7 +1088,7 @@ static void COVER_tryParameters(void *opaque) {
1079
1088
  {
1080
1089
  const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
1081
1090
  dictBufferCapacity, parameters);
1082
- selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
1091
+ selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
1083
1092
  ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
1084
1093
  totalCompressedSize);
1085
1094
 
@@ -1106,7 +1115,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
1106
1115
  /* constants */
1107
1116
  const unsigned nbThreads = parameters->nbThreads;
1108
1117
  const double splitPoint =
1109
- parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
1118
+ parameters->splitPoint <= 0.0 ? COVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
1110
1119
  const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
1111
1120
  const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
1112
1121
  const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
@@ -152,6 +152,6 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection);
152
152
  * smallest dictionary within a specified regression of the compressed size
153
153
  * from the largest dictionary.
154
154
  */
155
- COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
155
+ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
156
156
  size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
157
157
  size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
@@ -21,6 +21,7 @@
21
21
  #include "../common/threading.h"
22
22
  #include "cover.h"
23
23
  #include "../common/zstd_internal.h" /* includes zstd.h */
24
+ #include "../compress/zstd_compress_internal.h" /* ZSTD_hash*() */
24
25
  #ifndef ZDICT_STATIC_LINKING_ONLY
25
26
  #define ZDICT_STATIC_LINKING_ONLY
26
27
  #endif
@@ -33,7 +34,7 @@
33
34
  #define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
34
35
  #define FASTCOVER_MAX_F 31
35
36
  #define FASTCOVER_MAX_ACCEL 10
36
- #define DEFAULT_SPLITPOINT 0.75
37
+ #define FASTCOVER_DEFAULT_SPLITPOINT 0.75
37
38
  #define DEFAULT_F 20
38
39
  #define DEFAULT_ACCEL 1
39
40
 
@@ -41,50 +42,50 @@
41
42
  /*-*************************************
42
43
  * Console display
43
44
  ***************************************/
45
+ #ifndef LOCALDISPLAYLEVEL
44
46
  static int g_displayLevel = 2;
47
+ #endif
48
+ #undef DISPLAY
45
49
  #define DISPLAY(...) \
46
50
  { \
47
51
  fprintf(stderr, __VA_ARGS__); \
48
52
  fflush(stderr); \
49
53
  }
54
+ #undef LOCALDISPLAYLEVEL
50
55
  #define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
51
56
  if (displayLevel >= l) { \
52
57
  DISPLAY(__VA_ARGS__); \
53
58
  } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
59
+ #undef DISPLAYLEVEL
54
60
  #define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
55
61
 
62
+ #ifndef LOCALDISPLAYUPDATE
63
+ static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
64
+ static clock_t g_time = 0;
65
+ #endif
66
+ #undef LOCALDISPLAYUPDATE
56
67
  #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
57
68
  if (displayLevel >= l) { \
58
- if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
69
+ if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
59
70
  g_time = clock(); \
60
71
  DISPLAY(__VA_ARGS__); \
61
72
  } \
62
73
  }
74
+ #undef DISPLAYUPDATE
63
75
  #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
64
- static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
65
- static clock_t g_time = 0;
66
76
 
67
77
 
68
78
  /*-*************************************
69
79
  * Hash Functions
70
80
  ***************************************/
71
- static const U64 prime6bytes = 227718039650203ULL;
72
- static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
73
- static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
74
-
75
- static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
76
- static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
77
- static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
78
-
79
-
80
81
  /**
81
- * Hash the d-byte value pointed to by p and mod 2^f
82
+ * Hash the d-byte value pointed to by p and mod 2^f into the frequency vector
82
83
  */
83
- static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) {
84
+ static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 f, unsigned d) {
84
85
  if (d == 6) {
85
- return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1);
86
+ return ZSTD_hash6Ptr(p, f);
86
87
  }
87
- return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1);
88
+ return ZSTD_hash8Ptr(p, f);
88
89
  }
89
90
 
90
91
 
@@ -486,7 +487,7 @@ static void FASTCOVER_tryParameters(void *opaque)
486
487
  parameters, segmentFreqs);
487
488
 
488
489
  const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
489
- selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
490
+ selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
490
491
  ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
491
492
  totalCompressedSize);
492
493
 
@@ -617,7 +618,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
617
618
  /* constants */
618
619
  const unsigned nbThreads = parameters->nbThreads;
619
620
  const double splitPoint =
620
- parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
621
+ parameters->splitPoint <= 0.0 ? FASTCOVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
621
622
  const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
622
623
  const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
623
624
  const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
@@ -62,14 +62,15 @@
62
62
 
63
63
  #define NOISELENGTH 32
64
64
 
65
- static const int g_compressionLevel_default = 3;
66
65
  static const U32 g_selectivity_default = 9;
67
66
 
68
67
 
69
68
  /*-*************************************
70
69
  * Console display
71
70
  ***************************************/
71
+ #undef DISPLAY
72
72
  #define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
73
+ #undef DISPLAYLEVEL
73
74
  #define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
74
75
 
75
76
  static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
@@ -105,20 +106,17 @@ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
105
106
  size_t headerSize;
106
107
  if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);
107
108
 
108
- { unsigned offcodeMaxValue = MaxOff;
109
- ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
109
+ { ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
110
110
  U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE);
111
- short* offcodeNCount = (short*)malloc((MaxOff+1)*sizeof(short));
112
- if (!bs || !wksp || !offcodeNCount) {
111
+ if (!bs || !wksp) {
113
112
  headerSize = ERROR(memory_allocation);
114
113
  } else {
115
114
  ZSTD_reset_compressedBlockState(bs);
116
- headerSize = ZSTD_loadCEntropy(bs, wksp, offcodeNCount, &offcodeMaxValue, dictBuffer, dictSize);
115
+ headerSize = ZSTD_loadCEntropy(bs, wksp, dictBuffer, dictSize);
117
116
  }
118
117
 
119
118
  free(bs);
120
119
  free(wksp);
121
- free(offcodeNCount);
122
120
  }
123
121
 
124
122
  return headerSize;
@@ -532,6 +530,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
532
530
  clock_t displayClock = 0;
533
531
  clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
534
532
 
533
+ # undef DISPLAYUPDATE
535
534
  # define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \
536
535
  if (ZDICT_clockSpan(displayClock) > refreshRate) \
537
536
  { displayClock = clock(); DISPLAY(__VA_ARGS__); \
@@ -706,7 +705,7 @@ static void ZDICT_flatLit(unsigned* countLit)
706
705
 
707
706
  #define OFFCODE_MAX 30 /* only applicable to first block */
708
707
  static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
709
- unsigned compressionLevel,
708
+ int compressionLevel,
710
709
  const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles,
711
710
  const void* dictBuffer, size_t dictBufferSize,
712
711
  unsigned notificationLevel)
@@ -741,7 +740,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
741
740
  memset(repOffset, 0, sizeof(repOffset));
742
741
  repOffset[1] = repOffset[4] = repOffset[8] = 1;
743
742
  memset(bestRepOffset, 0, sizeof(bestRepOffset));
744
- if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
743
+ if (compressionLevel==0) compressionLevel = ZSTD_CLEVEL_DEFAULT;
745
744
  params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
746
745
 
747
746
  esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem);
@@ -786,7 +785,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
786
785
  /* note : the result of this phase should be used to better appreciate the impact on statistics */
787
786
 
788
787
  total=0; for (u=0; u<=offcodeMax; u++) total+=offcodeCount[u];
789
- errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax);
788
+ errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax, /* useLowProbCount */ 1);
790
789
  if (FSE_isError(errorCode)) {
791
790
  eSize = errorCode;
792
791
  DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n");
@@ -795,7 +794,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
795
794
  Offlog = (U32)errorCode;
796
795
 
797
796
  total=0; for (u=0; u<=MaxML; u++) total+=matchLengthCount[u];
798
- errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML);
797
+ errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML, /* useLowProbCount */ 1);
799
798
  if (FSE_isError(errorCode)) {
800
799
  eSize = errorCode;
801
800
  DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n");
@@ -804,7 +803,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
804
803
  mlLog = (U32)errorCode;
805
804
 
806
805
  total=0; for (u=0; u<=MaxLL; u++) total+=litLengthCount[u];
807
- errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL);
806
+ errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL, /* useLowProbCount */ 1);
808
807
  if (FSE_isError(errorCode)) {
809
808
  eSize = errorCode;
810
809
  DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n");
@@ -893,7 +892,7 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
893
892
  size_t hSize;
894
893
  #define HBUFFSIZE 256 /* should prove large enough for all entropy headers */
895
894
  BYTE header[HBUFFSIZE];
896
- int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
895
+ int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel;
897
896
  U32 const notificationLevel = params.notificationLevel;
898
897
 
899
898
  /* check conditions */
@@ -939,7 +938,7 @@ static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
939
938
  const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
940
939
  ZDICT_params_t params)
941
940
  {
942
- int const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
941
+ int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel;
943
942
  U32 const notificationLevel = params.notificationLevel;
944
943
  size_t hSize = 8;
945
944
 
@@ -1114,8 +1113,8 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
1114
1113
  memset(&params, 0, sizeof(params));
1115
1114
  params.d = 8;
1116
1115
  params.steps = 4;
1117
- /* Default to level 6 since no compression level information is available */
1118
- params.zParams.compressionLevel = 3;
1116
+ /* Use default level since no compression level information is available */
1117
+ params.zParams.compressionLevel = ZSTD_CLEVEL_DEFAULT;
1119
1118
  #if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1)
1120
1119
  params.zParams.notificationLevel = DEBUGLEVEL;
1121
1120
  #endif
@@ -279,7 +279,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
279
279
  # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
280
280
  # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
281
281
  # define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
282
- # elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
282
+ # elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
283
283
  # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
284
284
  # elif (ZDICT_GCC_VERSION >= 301)
285
285
  # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
@@ -1280,7 +1280,11 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
1280
1280
  * Basic Types
1281
1281
  *********************************************************/
1282
1282
  #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
1283
- # include <stdint.h>
1283
+ # if defined(_AIX)
1284
+ # include <inttypes.h>
1285
+ # else
1286
+ # include <stdint.h> /* intptr_t */
1287
+ # endif
1284
1288
  typedef uint8_t BYTE;
1285
1289
  typedef uint16_t U16;
1286
1290
  typedef int16_t S16;
@@ -89,7 +89,11 @@ extern "C" {
89
89
  * Basic Types
90
90
  *****************************************************************/
91
91
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
92
- # include <stdint.h>
92
+ # if defined(_AIX)
93
+ # include <inttypes.h>
94
+ # else
95
+ # include <stdint.h> /* intptr_t */
96
+ # endif
93
97
  typedef uint8_t BYTE;
94
98
  typedef uint16_t U16;
95
99
  typedef int16_t S16;
@@ -90,7 +90,11 @@ extern "C" {
90
90
  * Basic Types
91
91
  *****************************************************************/
92
92
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
93
- # include <stdint.h>
93
+ # if defined(_AIX)
94
+ # include <inttypes.h>
95
+ # else
96
+ # include <stdint.h> /* intptr_t */
97
+ # endif
94
98
  typedef uint8_t BYTE;
95
99
  typedef uint16_t U16;
96
100
  typedef int16_t S16;
@@ -52,7 +52,11 @@ extern "C" {
52
52
  * Basic Types
53
53
  *****************************************************************/
54
54
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
55
- # include <stdint.h>
55
+ # if defined(_AIX)
56
+ # include <inttypes.h>
57
+ # else
58
+ # include <stdint.h> /* intptr_t */
59
+ # endif
56
60
  typedef uint8_t BYTE;
57
61
  typedef uint16_t U16;
58
62
  typedef int16_t S16;
@@ -74,7 +78,7 @@ extern "C" {
74
78
  /*-*************************************
75
79
  * Debug
76
80
  ***************************************/
77
- #include "debug.h"
81
+ #include "../common/debug.h"
78
82
  #ifndef assert
79
83
  # define assert(condition) ((void)0)
80
84
  #endif