zstd-ruby 1.4.4.0 → 1.4.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/ext/zstdruby/libzstd/Makefile +123 -58
  4. data/ext/zstdruby/libzstd/README.md +34 -14
  5. data/ext/zstdruby/libzstd/common/bitstream.h +31 -37
  6. data/ext/zstdruby/libzstd/common/compiler.h +19 -3
  7. data/ext/zstdruby/libzstd/common/cpu.h +1 -1
  8. data/ext/zstdruby/libzstd/common/debug.c +11 -31
  9. data/ext/zstdruby/libzstd/common/debug.h +11 -31
  10. data/ext/zstdruby/libzstd/common/entropy_common.c +13 -33
  11. data/ext/zstdruby/libzstd/common/error_private.c +2 -1
  12. data/ext/zstdruby/libzstd/common/error_private.h +6 -2
  13. data/ext/zstdruby/libzstd/common/fse.h +11 -31
  14. data/ext/zstdruby/libzstd/common/fse_decompress.c +12 -37
  15. data/ext/zstdruby/libzstd/common/huf.h +15 -33
  16. data/ext/zstdruby/libzstd/common/mem.h +1 -1
  17. data/ext/zstdruby/libzstd/common/pool.c +1 -1
  18. data/ext/zstdruby/libzstd/common/pool.h +2 -2
  19. data/ext/zstdruby/libzstd/common/threading.c +4 -3
  20. data/ext/zstdruby/libzstd/common/threading.h +4 -3
  21. data/ext/zstdruby/libzstd/common/xxhash.c +15 -33
  22. data/ext/zstdruby/libzstd/common/xxhash.h +11 -31
  23. data/ext/zstdruby/libzstd/common/zstd_common.c +1 -1
  24. data/ext/zstdruby/libzstd/common/zstd_errors.h +2 -1
  25. data/ext/zstdruby/libzstd/common/zstd_internal.h +112 -15
  26. data/ext/zstdruby/libzstd/compress/fse_compress.c +17 -40
  27. data/ext/zstdruby/libzstd/compress/hist.c +15 -35
  28. data/ext/zstdruby/libzstd/compress/hist.h +12 -32
  29. data/ext/zstdruby/libzstd/compress/huf_compress.c +92 -92
  30. data/ext/zstdruby/libzstd/compress/zstd_compress.c +450 -275
  31. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +136 -14
  32. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +10 -6
  33. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +1 -1
  34. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +24 -20
  35. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +10 -3
  36. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +845 -0
  37. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
  38. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +3 -13
  39. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +11 -8
  40. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
  41. data/ext/zstdruby/libzstd/compress/zstd_fast.c +36 -24
  42. data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
  43. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +34 -11
  44. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +1 -1
  45. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +27 -5
  46. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +7 -2
  47. data/ext/zstdruby/libzstd/compress/zstd_opt.c +38 -84
  48. data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
  49. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +48 -21
  50. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +2 -2
  51. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +76 -62
  52. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +12 -8
  53. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
  54. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +264 -148
  55. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +312 -203
  56. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +3 -3
  57. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +18 -4
  58. data/ext/zstdruby/libzstd/deprecated/zbuff.h +3 -3
  59. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
  60. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
  61. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
  62. data/ext/zstdruby/libzstd/dictBuilder/cover.c +5 -5
  63. data/ext/zstdruby/libzstd/dictBuilder/cover.h +14 -4
  64. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +14 -4
  65. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +33 -9
  66. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +51 -28
  67. data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
  68. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +4 -4
  69. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +18 -12
  70. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +1 -1
  71. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +10 -6
  72. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +1 -1
  73. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +10 -6
  74. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +1 -1
  75. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +13 -7
  76. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +1 -1
  77. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +17 -13
  78. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +2 -2
  79. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +17 -13
  80. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +1 -1
  81. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +22 -14
  82. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +1 -1
  83. data/ext/zstdruby/libzstd/libzstd.pc.in +2 -2
  84. data/ext/zstdruby/libzstd/zstd.h +62 -21
  85. data/lib/zstd-ruby/version.rb +1 -1
  86. metadata +7 -5
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,8 +16,8 @@
16
16
  * Dependencies
17
17
  *********************************************************/
18
18
  #include <stddef.h> /* size_t */
19
- #include "zstd.h" /* DCtx, and some public functions */
20
- #include "zstd_internal.h" /* blockProperties_t, and some public functions */
19
+ #include "../zstd.h" /* DCtx, and some public functions */
20
+ #include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */
21
21
  #include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
22
22
 
23
23
 
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -19,8 +19,8 @@
19
19
  /*-*******************************************************
20
20
  * Dependencies
21
21
  *********************************************************/
22
- #include "mem.h" /* BYTE, U16, U32 */
23
- #include "zstd_internal.h" /* ZSTD_seqSymbol */
22
+ #include "../common/mem.h" /* BYTE, U16, U32 */
23
+ #include "../common/zstd_internal.h" /* ZSTD_seqSymbol */
24
24
 
25
25
 
26
26
 
@@ -95,6 +95,11 @@ typedef enum {
95
95
  ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
96
96
  } ZSTD_dictUses_e;
97
97
 
98
+ typedef enum {
99
+ ZSTD_obm_buffered = 0, /* Buffer the output */
100
+ ZSTD_obm_stable = 1 /* ZSTD_outBuffer is stable */
101
+ } ZSTD_outBufferMode_e;
102
+
98
103
  struct ZSTD_DCtx_s
99
104
  {
100
105
  const ZSTD_seqSymbol* LLTptr;
@@ -147,10 +152,19 @@ struct ZSTD_DCtx_s
147
152
  U32 legacyVersion;
148
153
  U32 hostageByte;
149
154
  int noForwardProgress;
155
+ ZSTD_outBufferMode_e outBufferMode;
156
+ ZSTD_outBuffer expectedOutBuffer;
150
157
 
151
158
  /* workspace */
152
159
  BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
153
160
  BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
161
+
162
+ size_t oversizedDuration;
163
+
164
+ #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
165
+ void const* dictContentBeginForFuzzing;
166
+ void const* dictContentEndForFuzzing;
167
+ #endif
154
168
  }; /* typedef'd to ZSTD_DCtx within "zstd.h" */
155
169
 
156
170
 
@@ -160,7 +174,7 @@ struct ZSTD_DCtx_s
160
174
 
161
175
  /*! ZSTD_loadDEntropy() :
162
176
  * dict : must point at beginning of a valid zstd dictionary.
163
- * @return : size of entropy tables read */
177
+ * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
164
178
  size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
165
179
  const void* const dict, size_t const dictSize);
166
180
 
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -28,7 +28,7 @@ extern "C" {
28
28
  * Dependencies
29
29
  ***************************************/
30
30
  #include <stddef.h> /* size_t */
31
- #include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
31
+ #include "../zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
32
32
 
33
33
 
34
34
  /* ***************************************************************
@@ -186,7 +186,7 @@ ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(voi
186
186
 
187
187
  /*--- Dependency ---*/
188
188
  #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */
189
- #include "zstd.h"
189
+ #include "../zstd.h"
190
190
 
191
191
 
192
192
  /*--- Custom memory allocator ---*/
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -11,7 +11,7 @@
11
11
  /*-*************************************
12
12
  * Dependencies
13
13
  ***************************************/
14
- #include "error_private.h"
14
+ #include "../common/error_private.h"
15
15
  #include "zbuff.h"
16
16
 
17
17
  /*-****************************************
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -26,11 +26,11 @@
26
26
  #include <string.h> /* memset */
27
27
  #include <time.h> /* clock */
28
28
 
29
- #include "mem.h" /* read */
30
- #include "pool.h"
31
- #include "threading.h"
29
+ #include "../common/mem.h" /* read */
30
+ #include "../common/pool.h"
31
+ #include "../common/threading.h"
32
32
  #include "cover.h"
33
- #include "zstd_internal.h" /* includes zstd.h */
33
+ #include "../common/zstd_internal.h" /* includes zstd.h */
34
34
  #ifndef ZDICT_STATIC_LINKING_ONLY
35
35
  #define ZDICT_STATIC_LINKING_ONLY
36
36
  #endif
@@ -1,11 +1,21 @@
1
+ /*
2
+ * Copyright (c) 2017-2020, Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
1
11
  #include <stdio.h> /* fprintf */
2
12
  #include <stdlib.h> /* malloc, free, qsort */
3
13
  #include <string.h> /* memset */
4
14
  #include <time.h> /* clock */
5
- #include "mem.h" /* read */
6
- #include "pool.h"
7
- #include "threading.h"
8
- #include "zstd_internal.h" /* includes zstd.h */
15
+ #include "../common/mem.h" /* read */
16
+ #include "../common/pool.h"
17
+ #include "../common/threading.h"
18
+ #include "../common/zstd_internal.h" /* includes zstd.h */
9
19
  #ifndef ZDICT_STATIC_LINKING_ONLY
10
20
  #define ZDICT_STATIC_LINKING_ONLY
11
21
  #endif
@@ -1,3 +1,13 @@
1
+ /*
2
+ * Copyright (c) 2018-2020, Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
1
11
  /*-*************************************
2
12
  * Dependencies
3
13
  ***************************************/
@@ -6,11 +16,11 @@
6
16
  #include <string.h> /* memset */
7
17
  #include <time.h> /* clock */
8
18
 
9
- #include "mem.h" /* read */
10
- #include "pool.h"
11
- #include "threading.h"
19
+ #include "../common/mem.h" /* read */
20
+ #include "../common/pool.h"
21
+ #include "../common/threading.h"
12
22
  #include "cover.h"
13
- #include "zstd_internal.h" /* includes zstd.h */
23
+ #include "../common/zstd_internal.h" /* includes zstd.h */
14
24
  #ifndef ZDICT_STATIC_LINKING_ONLY
15
25
  #define ZDICT_STATIC_LINKING_ONLY
16
26
  #endif
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -37,17 +37,18 @@
37
37
  #include <stdio.h> /* fprintf, fopen, ftello64 */
38
38
  #include <time.h> /* clock */
39
39
 
40
- #include "mem.h" /* read */
41
- #include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */
40
+ #include "../common/mem.h" /* read */
41
+ #include "../common/fse.h" /* FSE_normalizeCount, FSE_writeNCount */
42
42
  #define HUF_STATIC_LINKING_ONLY
43
- #include "huf.h" /* HUF_buildCTable, HUF_writeCTable */
44
- #include "zstd_internal.h" /* includes zstd.h */
45
- #include "xxhash.h" /* XXH64 */
43
+ #include "../common/huf.h" /* HUF_buildCTable, HUF_writeCTable */
44
+ #include "../common/zstd_internal.h" /* includes zstd.h */
45
+ #include "../common/xxhash.h" /* XXH64 */
46
46
  #include "divsufsort.h"
47
47
  #ifndef ZDICT_STATIC_LINKING_ONLY
48
48
  # define ZDICT_STATIC_LINKING_ONLY
49
49
  #endif
50
50
  #include "zdict.h"
51
+ #include "../compress/zstd_compress_internal.h" /* ZSTD_loadCEntropy() */
51
52
 
52
53
 
53
54
  /*-*************************************
@@ -99,6 +100,29 @@ unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
99
100
  return MEM_readLE32((const char*)dictBuffer + 4);
100
101
  }
101
102
 
103
+ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
104
+ {
105
+ size_t headerSize;
106
+ if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);
107
+
108
+ { unsigned offcodeMaxValue = MaxOff;
109
+ ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
110
+ U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE);
111
+ short* offcodeNCount = (short*)malloc((MaxOff+1)*sizeof(short));
112
+ if (!bs || !wksp || !offcodeNCount) {
113
+ headerSize = ERROR(memory_allocation);
114
+ } else {
115
+ ZSTD_reset_compressedBlockState(bs);
116
+ headerSize = ZSTD_loadCEntropy(bs, wksp, offcodeNCount, &offcodeMaxValue, dictBuffer, dictSize);
117
+ }
118
+
119
+ free(bs);
120
+ free(wksp);
121
+ free(offcodeNCount);
122
+ }
123
+
124
+ return headerSize;
125
+ }
102
126
 
103
127
  /*-********************************************************
104
128
  * Dictionary training functions
@@ -588,12 +612,12 @@ typedef struct
588
612
 
589
613
  #define MAXREPOFFSET 1024
590
614
 
591
- static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
615
+ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
592
616
  unsigned* countLit, unsigned* offsetcodeCount, unsigned* matchlengthCount, unsigned* litlengthCount, U32* repOffsets,
593
617
  const void* src, size_t srcSize,
594
618
  U32 notificationLevel)
595
619
  {
596
- size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params.cParams.windowLog);
620
+ size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params->cParams.windowLog);
597
621
  size_t cSize;
598
622
 
599
623
  if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
@@ -731,7 +755,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
731
755
 
732
756
  /* collect stats on all samples */
733
757
  for (u=0; u<nbFiles; u++) {
734
- ZDICT_countEStats(esr, params,
758
+ ZDICT_countEStats(esr, &params,
735
759
  countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
736
760
  (const char*)srcBuffer + pos, fileSizes[u],
737
761
  notificationLevel);
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -61,9 +61,57 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
61
61
  const void* samplesBuffer,
62
62
  const size_t* samplesSizes, unsigned nbSamples);
63
63
 
64
+ typedef struct {
65
+ int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
66
+ unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
67
+ unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */
68
+ } ZDICT_params_t;
69
+
70
+ /*! ZDICT_finalizeDictionary():
71
+ * Given a custom content as a basis for dictionary, and a set of samples,
72
+ * finalize dictionary by adding headers and statistics according to the zstd
73
+ * dictionary format.
74
+ *
75
+ * Samples must be stored concatenated in a flat buffer `samplesBuffer`,
76
+ * supplied with an array of sizes `samplesSizes`, providing the size of each
77
+ * sample in order. The samples are used to construct the statistics, so they
78
+ * should be representative of what you will compress with this dictionary.
79
+ *
80
+ * The compression level can be set in `parameters`. You should pass the
81
+ * compression level you expect to use in production. The statistics for each
82
+ * compression level differ, so tuning the dictionary for the compression level
83
+ * can help quite a bit.
84
+ *
85
+ * You can set an explicit dictionary ID in `parameters`, or allow us to pick
86
+ * a random dictionary ID for you, but we can't guarantee no collisions.
87
+ *
88
+ * The dstDictBuffer and the dictContent may overlap, and the content will be
89
+ * appended to the end of the header. If the header + the content doesn't fit in
90
+ * maxDictSize the beginning of the content is truncated to make room, since it
91
+ * is presumed that the most profitable content is at the end of the dictionary,
92
+ * since that is the cheapest to reference.
93
+ *
94
+ * `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
95
+ * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
96
+ *
97
+ * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
98
+ * or an error code, which can be tested by ZDICT_isError().
99
+ * Note: ZDICT_finalizeDictionary() will push notifications into stderr if
100
+ * instructed to, using notificationLevel>0.
101
+ * NOTE: This function currently may fail in several edge cases including:
102
+ * * Not enough samples
103
+ * * Samples are uncompressible
104
+ * * Samples are all exactly the same
105
+ */
106
+ ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
107
+ const void* dictContent, size_t dictContentSize,
108
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
109
+ ZDICT_params_t parameters);
110
+
64
111
 
65
112
  /*====== Helper functions ======*/
66
113
  ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
114
+ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
67
115
  ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
68
116
  ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
69
117
 
@@ -78,11 +126,8 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
78
126
  * Use them only in association with static linking.
79
127
  * ==================================================================================== */
80
128
 
81
- typedef struct {
82
- int compressionLevel; /* optimize for a specific zstd compression level; 0 means default */
83
- unsigned notificationLevel; /* Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
84
- unsigned dictID; /* force dictID value; 0 means auto mode (32-bits random value) */
85
- } ZDICT_params_t;
129
+ #define ZDICT_CONTENTSIZE_MIN 128
130
+ #define ZDICT_DICTSIZE_MIN 256
86
131
 
87
132
  /*! ZDICT_cover_params_t:
88
133
  * k and d are the only required parameters.
@@ -198,28 +243,6 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
198
243
  const size_t* samplesSizes, unsigned nbSamples,
199
244
  ZDICT_fastCover_params_t* parameters);
200
245
 
201
- /*! ZDICT_finalizeDictionary():
202
- * Given a custom content as a basis for dictionary, and a set of samples,
203
- * finalize dictionary by adding headers and statistics.
204
- *
205
- * Samples must be stored concatenated in a flat buffer `samplesBuffer`,
206
- * supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
207
- *
208
- * dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes.
209
- * maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes.
210
- *
211
- * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
212
- * or an error code, which can be tested by ZDICT_isError().
213
- * Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
214
- * Note 2: dictBuffer and dictContent can overlap
215
- */
216
- #define ZDICT_CONTENTSIZE_MIN 128
217
- #define ZDICT_DICTSIZE_MIN 256
218
- ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
219
- const void* dictContent, size_t dictContentSize,
220
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
221
- ZDICT_params_t parameters);
222
-
223
246
  typedef struct {
224
247
  unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
225
248
  ZDICT_params_t zParams;
@@ -1,10 +1,11 @@
1
1
  # ################################################################
2
- # Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ # Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  # All rights reserved.
4
4
  #
5
5
  # This source code is licensed under both the BSD-style license (found in the
6
6
  # LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
7
  # in the COPYING file in the root directory of this source tree).
8
+ # You may select, at your option, one of the above-listed licenses.
8
9
  # ################################################################
9
10
 
10
11
  VOID := /dev/null
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -18,9 +18,9 @@ extern "C" {
18
18
  /* *************************************
19
19
  * Includes
20
20
  ***************************************/
21
- #include "mem.h" /* MEM_STATIC */
22
- #include "error_private.h" /* ERROR */
23
- #include "zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
21
+ #include "../common/mem.h" /* MEM_STATIC */
22
+ #include "../common/error_private.h" /* ERROR */
23
+ #include "../common/zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
24
24
 
25
25
  #if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
26
26
  # undef ZSTD_LEGACY_SUPPORT
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +14,7 @@
14
14
  ******************************************/
15
15
  #include <stddef.h> /* size_t, ptrdiff_t */
16
16
  #include "zstd_v01.h"
17
- #include "error_private.h"
17
+ #include "../common/error_private.h"
18
18
 
19
19
 
20
20
  /******************************************
@@ -257,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
257
257
  U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
258
258
  }
259
259
 
260
- #endif // FSE_FORCE_MEMORY_ACCESS
260
+ #endif /* FSE_FORCE_MEMORY_ACCESS */
261
261
 
262
262
  static U16 FSE_readLE16(const void* memPtr)
263
263
  {
@@ -1078,7 +1078,7 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
1078
1078
  BYTE* const ostart = (BYTE*) dst;
1079
1079
  BYTE* op = ostart;
1080
1080
  BYTE* const omax = op + maxDstSize;
1081
- BYTE* const olimit = omax-15;
1081
+ BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
1082
1082
 
1083
1083
  const void* ptr = DTable;
1084
1084
  const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
@@ -1092,7 +1092,7 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
1092
1092
  const size_t length1 = FSE_readLE16(jumpTable);
1093
1093
  const size_t length2 = FSE_readLE16(jumpTable+1);
1094
1094
  const size_t length3 = FSE_readLE16(jumpTable+2);
1095
- const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
1095
+ const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
1096
1096
  const char* const start1 = (const char*)(cSrc) + 6;
1097
1097
  const char* const start2 = start1 + length1;
1098
1098
  const char* const start3 = start2 + length2;
@@ -1150,11 +1150,11 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
1150
1150
 
1151
1151
  /* tail */
1152
1152
  {
1153
- // bitTail = bitD1; // *much* slower : -20% !??!
1153
+ /* bitTail = bitD1; */ /* *much* slower : -20% !??! */
1154
1154
  FSE_DStream_t bitTail;
1155
1155
  bitTail.ptr = bitD1.ptr;
1156
1156
  bitTail.bitsConsumed = bitD1.bitsConsumed;
1157
- bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
1157
+ bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
1158
1158
  bitTail.start = start1;
1159
1159
  for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
1160
1160
  {
@@ -1483,7 +1483,9 @@ static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProper
1483
1483
  static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
1484
1484
  {
1485
1485
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
1486
- memcpy(dst, src, srcSize);
1486
+ if (srcSize > 0) {
1487
+ memcpy(dst, src, srcSize);
1488
+ }
1487
1489
  return srcSize;
1488
1490
  }
1489
1491
 
@@ -1502,7 +1504,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
1502
1504
  if (srcSize <= 3) return ERROR(corruption_detected);
1503
1505
 
1504
1506
  litSize = ip[1] + (ip[0]<<8);
1505
- litSize += ((ip[-3] >> 3) & 7) << 16; // mmmmh....
1507
+ litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
1506
1508
  op = oend - litSize;
1507
1509
 
1508
1510
  (void)ctx;
@@ -1541,7 +1543,9 @@ static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
1541
1543
  size_t rleSize = litbp.origSize;
1542
1544
  if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
1543
1545
  if (!srcSize) return ERROR(srcSize_wrong);
1544
- memset(oend - rleSize, *ip, rleSize);
1546
+ if (rleSize > 0) {
1547
+ memset(oend - rleSize, *ip, rleSize);
1548
+ }
1545
1549
  *litStart = oend - rleSize;
1546
1550
  *litSize = rleSize;
1547
1551
  ip++;
@@ -1901,8 +1905,10 @@ static size_t ZSTD_decompressSequences(
1901
1905
  {
1902
1906
  size_t lastLLSize = litEnd - litPtr;
1903
1907
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
1904
- if (op != litPtr) memmove(op, litPtr, lastLLSize);
1905
- op += lastLLSize;
1908
+ if (lastLLSize > 0) {
1909
+ if (op != litPtr) memmove(op, litPtr, lastLLSize);
1910
+ op += lastLLSize;
1911
+ }
1906
1912
  }
1907
1913
  }
1908
1914