zstd-ruby 1.4.4.0 → 1.4.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/ext/zstdruby/libzstd/Makefile +123 -58
  4. data/ext/zstdruby/libzstd/README.md +34 -14
  5. data/ext/zstdruby/libzstd/common/bitstream.h +31 -37
  6. data/ext/zstdruby/libzstd/common/compiler.h +19 -3
  7. data/ext/zstdruby/libzstd/common/cpu.h +1 -1
  8. data/ext/zstdruby/libzstd/common/debug.c +11 -31
  9. data/ext/zstdruby/libzstd/common/debug.h +11 -31
  10. data/ext/zstdruby/libzstd/common/entropy_common.c +13 -33
  11. data/ext/zstdruby/libzstd/common/error_private.c +2 -1
  12. data/ext/zstdruby/libzstd/common/error_private.h +6 -2
  13. data/ext/zstdruby/libzstd/common/fse.h +11 -31
  14. data/ext/zstdruby/libzstd/common/fse_decompress.c +12 -37
  15. data/ext/zstdruby/libzstd/common/huf.h +15 -33
  16. data/ext/zstdruby/libzstd/common/mem.h +1 -1
  17. data/ext/zstdruby/libzstd/common/pool.c +1 -1
  18. data/ext/zstdruby/libzstd/common/pool.h +2 -2
  19. data/ext/zstdruby/libzstd/common/threading.c +4 -3
  20. data/ext/zstdruby/libzstd/common/threading.h +4 -3
  21. data/ext/zstdruby/libzstd/common/xxhash.c +15 -33
  22. data/ext/zstdruby/libzstd/common/xxhash.h +11 -31
  23. data/ext/zstdruby/libzstd/common/zstd_common.c +1 -1
  24. data/ext/zstdruby/libzstd/common/zstd_errors.h +2 -1
  25. data/ext/zstdruby/libzstd/common/zstd_internal.h +112 -15
  26. data/ext/zstdruby/libzstd/compress/fse_compress.c +17 -40
  27. data/ext/zstdruby/libzstd/compress/hist.c +15 -35
  28. data/ext/zstdruby/libzstd/compress/hist.h +12 -32
  29. data/ext/zstdruby/libzstd/compress/huf_compress.c +92 -92
  30. data/ext/zstdruby/libzstd/compress/zstd_compress.c +450 -275
  31. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +136 -14
  32. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +10 -6
  33. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +1 -1
  34. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +24 -20
  35. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +10 -3
  36. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +845 -0
  37. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
  38. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +3 -13
  39. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +11 -8
  40. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
  41. data/ext/zstdruby/libzstd/compress/zstd_fast.c +36 -24
  42. data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
  43. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +34 -11
  44. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +1 -1
  45. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +27 -5
  46. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +7 -2
  47. data/ext/zstdruby/libzstd/compress/zstd_opt.c +38 -84
  48. data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
  49. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +48 -21
  50. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +2 -2
  51. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +76 -62
  52. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +12 -8
  53. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
  54. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +264 -148
  55. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +312 -203
  56. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +3 -3
  57. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +18 -4
  58. data/ext/zstdruby/libzstd/deprecated/zbuff.h +3 -3
  59. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
  60. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
  61. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
  62. data/ext/zstdruby/libzstd/dictBuilder/cover.c +5 -5
  63. data/ext/zstdruby/libzstd/dictBuilder/cover.h +14 -4
  64. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +14 -4
  65. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +33 -9
  66. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +51 -28
  67. data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
  68. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +4 -4
  69. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +18 -12
  70. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +1 -1
  71. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +10 -6
  72. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +1 -1
  73. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +10 -6
  74. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +1 -1
  75. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +13 -7
  76. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +1 -1
  77. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +17 -13
  78. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +2 -2
  79. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +17 -13
  80. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +1 -1
  81. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +22 -14
  82. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +1 -1
  83. data/ext/zstdruby/libzstd/libzstd.pc.in +2 -2
  84. data/ext/zstdruby/libzstd/zstd.h +62 -21
  85. data/lib/zstd-ruby/version.rb +1 -1
  86. metadata +7 -5
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,8 +16,8 @@
16
16
  * Dependencies
17
17
  *********************************************************/
18
18
  #include <stddef.h> /* size_t */
19
- #include "zstd.h" /* DCtx, and some public functions */
20
- #include "zstd_internal.h" /* blockProperties_t, and some public functions */
19
+ #include "../zstd.h" /* DCtx, and some public functions */
20
+ #include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */
21
21
  #include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
22
22
 
23
23
 
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -19,8 +19,8 @@
19
19
  /*-*******************************************************
20
20
  * Dependencies
21
21
  *********************************************************/
22
- #include "mem.h" /* BYTE, U16, U32 */
23
- #include "zstd_internal.h" /* ZSTD_seqSymbol */
22
+ #include "../common/mem.h" /* BYTE, U16, U32 */
23
+ #include "../common/zstd_internal.h" /* ZSTD_seqSymbol */
24
24
 
25
25
 
26
26
 
@@ -95,6 +95,11 @@ typedef enum {
95
95
  ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
96
96
  } ZSTD_dictUses_e;
97
97
 
98
+ typedef enum {
99
+ ZSTD_obm_buffered = 0, /* Buffer the output */
100
+ ZSTD_obm_stable = 1 /* ZSTD_outBuffer is stable */
101
+ } ZSTD_outBufferMode_e;
102
+
98
103
  struct ZSTD_DCtx_s
99
104
  {
100
105
  const ZSTD_seqSymbol* LLTptr;
@@ -147,10 +152,19 @@ struct ZSTD_DCtx_s
147
152
  U32 legacyVersion;
148
153
  U32 hostageByte;
149
154
  int noForwardProgress;
155
+ ZSTD_outBufferMode_e outBufferMode;
156
+ ZSTD_outBuffer expectedOutBuffer;
150
157
 
151
158
  /* workspace */
152
159
  BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
153
160
  BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
161
+
162
+ size_t oversizedDuration;
163
+
164
+ #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
165
+ void const* dictContentBeginForFuzzing;
166
+ void const* dictContentEndForFuzzing;
167
+ #endif
154
168
  }; /* typedef'd to ZSTD_DCtx within "zstd.h" */
155
169
 
156
170
 
@@ -160,7 +174,7 @@ struct ZSTD_DCtx_s
160
174
 
161
175
  /*! ZSTD_loadDEntropy() :
162
176
  * dict : must point at beginning of a valid zstd dictionary.
163
- * @return : size of entropy tables read */
177
+ * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
164
178
  size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
165
179
  const void* const dict, size_t const dictSize);
166
180
 
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -28,7 +28,7 @@ extern "C" {
28
28
  * Dependencies
29
29
  ***************************************/
30
30
  #include <stddef.h> /* size_t */
31
- #include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
31
+ #include "../zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
32
32
 
33
33
 
34
34
  /* ***************************************************************
@@ -186,7 +186,7 @@ ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(voi
186
186
 
187
187
  /*--- Dependency ---*/
188
188
  #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */
189
- #include "zstd.h"
189
+ #include "../zstd.h"
190
190
 
191
191
 
192
192
  /*--- Custom memory allocator ---*/
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -11,7 +11,7 @@
11
11
  /*-*************************************
12
12
  * Dependencies
13
13
  ***************************************/
14
- #include "error_private.h"
14
+ #include "../common/error_private.h"
15
15
  #include "zbuff.h"
16
16
 
17
17
  /*-****************************************
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -26,11 +26,11 @@
26
26
  #include <string.h> /* memset */
27
27
  #include <time.h> /* clock */
28
28
 
29
- #include "mem.h" /* read */
30
- #include "pool.h"
31
- #include "threading.h"
29
+ #include "../common/mem.h" /* read */
30
+ #include "../common/pool.h"
31
+ #include "../common/threading.h"
32
32
  #include "cover.h"
33
- #include "zstd_internal.h" /* includes zstd.h */
33
+ #include "../common/zstd_internal.h" /* includes zstd.h */
34
34
  #ifndef ZDICT_STATIC_LINKING_ONLY
35
35
  #define ZDICT_STATIC_LINKING_ONLY
36
36
  #endif
@@ -1,11 +1,21 @@
1
+ /*
2
+ * Copyright (c) 2017-2020, Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
1
11
  #include <stdio.h> /* fprintf */
2
12
  #include <stdlib.h> /* malloc, free, qsort */
3
13
  #include <string.h> /* memset */
4
14
  #include <time.h> /* clock */
5
- #include "mem.h" /* read */
6
- #include "pool.h"
7
- #include "threading.h"
8
- #include "zstd_internal.h" /* includes zstd.h */
15
+ #include "../common/mem.h" /* read */
16
+ #include "../common/pool.h"
17
+ #include "../common/threading.h"
18
+ #include "../common/zstd_internal.h" /* includes zstd.h */
9
19
  #ifndef ZDICT_STATIC_LINKING_ONLY
10
20
  #define ZDICT_STATIC_LINKING_ONLY
11
21
  #endif
@@ -1,3 +1,13 @@
1
+ /*
2
+ * Copyright (c) 2018-2020, Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
1
11
  /*-*************************************
2
12
  * Dependencies
3
13
  ***************************************/
@@ -6,11 +16,11 @@
6
16
  #include <string.h> /* memset */
7
17
  #include <time.h> /* clock */
8
18
 
9
- #include "mem.h" /* read */
10
- #include "pool.h"
11
- #include "threading.h"
19
+ #include "../common/mem.h" /* read */
20
+ #include "../common/pool.h"
21
+ #include "../common/threading.h"
12
22
  #include "cover.h"
13
- #include "zstd_internal.h" /* includes zstd.h */
23
+ #include "../common/zstd_internal.h" /* includes zstd.h */
14
24
  #ifndef ZDICT_STATIC_LINKING_ONLY
15
25
  #define ZDICT_STATIC_LINKING_ONLY
16
26
  #endif
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -37,17 +37,18 @@
37
37
  #include <stdio.h> /* fprintf, fopen, ftello64 */
38
38
  #include <time.h> /* clock */
39
39
 
40
- #include "mem.h" /* read */
41
- #include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */
40
+ #include "../common/mem.h" /* read */
41
+ #include "../common/fse.h" /* FSE_normalizeCount, FSE_writeNCount */
42
42
  #define HUF_STATIC_LINKING_ONLY
43
- #include "huf.h" /* HUF_buildCTable, HUF_writeCTable */
44
- #include "zstd_internal.h" /* includes zstd.h */
45
- #include "xxhash.h" /* XXH64 */
43
+ #include "../common/huf.h" /* HUF_buildCTable, HUF_writeCTable */
44
+ #include "../common/zstd_internal.h" /* includes zstd.h */
45
+ #include "../common/xxhash.h" /* XXH64 */
46
46
  #include "divsufsort.h"
47
47
  #ifndef ZDICT_STATIC_LINKING_ONLY
48
48
  # define ZDICT_STATIC_LINKING_ONLY
49
49
  #endif
50
50
  #include "zdict.h"
51
+ #include "../compress/zstd_compress_internal.h" /* ZSTD_loadCEntropy() */
51
52
 
52
53
 
53
54
  /*-*************************************
@@ -99,6 +100,29 @@ unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
99
100
  return MEM_readLE32((const char*)dictBuffer + 4);
100
101
  }
101
102
 
103
+ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
104
+ {
105
+ size_t headerSize;
106
+ if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);
107
+
108
+ { unsigned offcodeMaxValue = MaxOff;
109
+ ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
110
+ U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE);
111
+ short* offcodeNCount = (short*)malloc((MaxOff+1)*sizeof(short));
112
+ if (!bs || !wksp || !offcodeNCount) {
113
+ headerSize = ERROR(memory_allocation);
114
+ } else {
115
+ ZSTD_reset_compressedBlockState(bs);
116
+ headerSize = ZSTD_loadCEntropy(bs, wksp, offcodeNCount, &offcodeMaxValue, dictBuffer, dictSize);
117
+ }
118
+
119
+ free(bs);
120
+ free(wksp);
121
+ free(offcodeNCount);
122
+ }
123
+
124
+ return headerSize;
125
+ }
102
126
 
103
127
  /*-********************************************************
104
128
  * Dictionary training functions
@@ -588,12 +612,12 @@ typedef struct
588
612
 
589
613
  #define MAXREPOFFSET 1024
590
614
 
591
- static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
615
+ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
592
616
  unsigned* countLit, unsigned* offsetcodeCount, unsigned* matchlengthCount, unsigned* litlengthCount, U32* repOffsets,
593
617
  const void* src, size_t srcSize,
594
618
  U32 notificationLevel)
595
619
  {
596
- size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params.cParams.windowLog);
620
+ size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params->cParams.windowLog);
597
621
  size_t cSize;
598
622
 
599
623
  if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
@@ -731,7 +755,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
731
755
 
732
756
  /* collect stats on all samples */
733
757
  for (u=0; u<nbFiles; u++) {
734
- ZDICT_countEStats(esr, params,
758
+ ZDICT_countEStats(esr, &params,
735
759
  countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
736
760
  (const char*)srcBuffer + pos, fileSizes[u],
737
761
  notificationLevel);
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -61,9 +61,57 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
61
61
  const void* samplesBuffer,
62
62
  const size_t* samplesSizes, unsigned nbSamples);
63
63
 
64
+ typedef struct {
65
+ int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
66
+ unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
67
+ unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */
68
+ } ZDICT_params_t;
69
+
70
+ /*! ZDICT_finalizeDictionary():
71
+ * Given a custom content as a basis for dictionary, and a set of samples,
72
+ * finalize dictionary by adding headers and statistics according to the zstd
73
+ * dictionary format.
74
+ *
75
+ * Samples must be stored concatenated in a flat buffer `samplesBuffer`,
76
+ * supplied with an array of sizes `samplesSizes`, providing the size of each
77
+ * sample in order. The samples are used to construct the statistics, so they
78
+ * should be representative of what you will compress with this dictionary.
79
+ *
80
+ * The compression level can be set in `parameters`. You should pass the
81
+ * compression level you expect to use in production. The statistics for each
82
+ * compression level differ, so tuning the dictionary for the compression level
83
+ * can help quite a bit.
84
+ *
85
+ * You can set an explicit dictionary ID in `parameters`, or allow us to pick
86
+ * a random dictionary ID for you, but we can't guarantee no collisions.
87
+ *
88
+ * The dstDictBuffer and the dictContent may overlap, and the content will be
89
+ * appended to the end of the header. If the header + the content doesn't fit in
90
+ * maxDictSize the beginning of the content is truncated to make room, since it
91
+ * is presumed that the most profitable content is at the end of the dictionary,
92
+ * since that is the cheapest to reference.
93
+ *
94
+ * `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
95
+ * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
96
+ *
97
+ * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
98
+ * or an error code, which can be tested by ZDICT_isError().
99
+ * Note: ZDICT_finalizeDictionary() will push notifications into stderr if
100
+ * instructed to, using notificationLevel>0.
101
+ * NOTE: This function currently may fail in several edge cases including:
102
+ * * Not enough samples
103
+ * * Samples are uncompressible
104
+ * * Samples are all exactly the same
105
+ */
106
+ ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
107
+ const void* dictContent, size_t dictContentSize,
108
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
109
+ ZDICT_params_t parameters);
110
+
64
111
 
65
112
  /*====== Helper functions ======*/
66
113
  ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
114
+ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
67
115
  ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
68
116
  ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
69
117
 
@@ -78,11 +126,8 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
78
126
  * Use them only in association with static linking.
79
127
  * ==================================================================================== */
80
128
 
81
- typedef struct {
82
- int compressionLevel; /* optimize for a specific zstd compression level; 0 means default */
83
- unsigned notificationLevel; /* Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
84
- unsigned dictID; /* force dictID value; 0 means auto mode (32-bits random value) */
85
- } ZDICT_params_t;
129
+ #define ZDICT_CONTENTSIZE_MIN 128
130
+ #define ZDICT_DICTSIZE_MIN 256
86
131
 
87
132
  /*! ZDICT_cover_params_t:
88
133
  * k and d are the only required parameters.
@@ -198,28 +243,6 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
198
243
  const size_t* samplesSizes, unsigned nbSamples,
199
244
  ZDICT_fastCover_params_t* parameters);
200
245
 
201
- /*! ZDICT_finalizeDictionary():
202
- * Given a custom content as a basis for dictionary, and a set of samples,
203
- * finalize dictionary by adding headers and statistics.
204
- *
205
- * Samples must be stored concatenated in a flat buffer `samplesBuffer`,
206
- * supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
207
- *
208
- * dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes.
209
- * maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes.
210
- *
211
- * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
212
- * or an error code, which can be tested by ZDICT_isError().
213
- * Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
214
- * Note 2: dictBuffer and dictContent can overlap
215
- */
216
- #define ZDICT_CONTENTSIZE_MIN 128
217
- #define ZDICT_DICTSIZE_MIN 256
218
- ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
219
- const void* dictContent, size_t dictContentSize,
220
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
221
- ZDICT_params_t parameters);
222
-
223
246
  typedef struct {
224
247
  unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
225
248
  ZDICT_params_t zParams;
@@ -1,10 +1,11 @@
1
1
  # ################################################################
2
- # Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ # Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  # All rights reserved.
4
4
  #
5
5
  # This source code is licensed under both the BSD-style license (found in the
6
6
  # LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
7
  # in the COPYING file in the root directory of this source tree).
8
+ # You may select, at your option, one of the above-listed licenses.
8
9
  # ################################################################
9
10
 
10
11
  VOID := /dev/null
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -18,9 +18,9 @@ extern "C" {
18
18
  /* *************************************
19
19
  * Includes
20
20
  ***************************************/
21
- #include "mem.h" /* MEM_STATIC */
22
- #include "error_private.h" /* ERROR */
23
- #include "zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
21
+ #include "../common/mem.h" /* MEM_STATIC */
22
+ #include "../common/error_private.h" /* ERROR */
23
+ #include "../common/zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
24
24
 
25
25
  #if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
26
26
  # undef ZSTD_LEGACY_SUPPORT
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +14,7 @@
14
14
  ******************************************/
15
15
  #include <stddef.h> /* size_t, ptrdiff_t */
16
16
  #include "zstd_v01.h"
17
- #include "error_private.h"
17
+ #include "../common/error_private.h"
18
18
 
19
19
 
20
20
  /******************************************
@@ -257,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
257
257
  U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
258
258
  }
259
259
 
260
- #endif // FSE_FORCE_MEMORY_ACCESS
260
+ #endif /* FSE_FORCE_MEMORY_ACCESS */
261
261
 
262
262
  static U16 FSE_readLE16(const void* memPtr)
263
263
  {
@@ -1078,7 +1078,7 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
1078
1078
  BYTE* const ostart = (BYTE*) dst;
1079
1079
  BYTE* op = ostart;
1080
1080
  BYTE* const omax = op + maxDstSize;
1081
- BYTE* const olimit = omax-15;
1081
+ BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
1082
1082
 
1083
1083
  const void* ptr = DTable;
1084
1084
  const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
@@ -1092,7 +1092,7 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
1092
1092
  const size_t length1 = FSE_readLE16(jumpTable);
1093
1093
  const size_t length2 = FSE_readLE16(jumpTable+1);
1094
1094
  const size_t length3 = FSE_readLE16(jumpTable+2);
1095
- const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
1095
+ const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
1096
1096
  const char* const start1 = (const char*)(cSrc) + 6;
1097
1097
  const char* const start2 = start1 + length1;
1098
1098
  const char* const start3 = start2 + length2;
@@ -1150,11 +1150,11 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
1150
1150
 
1151
1151
  /* tail */
1152
1152
  {
1153
- // bitTail = bitD1; // *much* slower : -20% !??!
1153
+ /* bitTail = bitD1; */ /* *much* slower : -20% !??! */
1154
1154
  FSE_DStream_t bitTail;
1155
1155
  bitTail.ptr = bitD1.ptr;
1156
1156
  bitTail.bitsConsumed = bitD1.bitsConsumed;
1157
- bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
1157
+ bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
1158
1158
  bitTail.start = start1;
1159
1159
  for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
1160
1160
  {
@@ -1483,7 +1483,9 @@ static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProper
1483
1483
  static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
1484
1484
  {
1485
1485
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
1486
- memcpy(dst, src, srcSize);
1486
+ if (srcSize > 0) {
1487
+ memcpy(dst, src, srcSize);
1488
+ }
1487
1489
  return srcSize;
1488
1490
  }
1489
1491
 
@@ -1502,7 +1504,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
1502
1504
  if (srcSize <= 3) return ERROR(corruption_detected);
1503
1505
 
1504
1506
  litSize = ip[1] + (ip[0]<<8);
1505
- litSize += ((ip[-3] >> 3) & 7) << 16; // mmmmh....
1507
+ litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
1506
1508
  op = oend - litSize;
1507
1509
 
1508
1510
  (void)ctx;
@@ -1541,7 +1543,9 @@ static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
1541
1543
  size_t rleSize = litbp.origSize;
1542
1544
  if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
1543
1545
  if (!srcSize) return ERROR(srcSize_wrong);
1544
- memset(oend - rleSize, *ip, rleSize);
1546
+ if (rleSize > 0) {
1547
+ memset(oend - rleSize, *ip, rleSize);
1548
+ }
1545
1549
  *litStart = oend - rleSize;
1546
1550
  *litSize = rleSize;
1547
1551
  ip++;
@@ -1901,8 +1905,10 @@ static size_t ZSTD_decompressSequences(
1901
1905
  {
1902
1906
  size_t lastLLSize = litEnd - litPtr;
1903
1907
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
1904
- if (op != litPtr) memmove(op, litPtr, lastLLSize);
1905
- op += lastLLSize;
1908
+ if (lastLLSize > 0) {
1909
+ if (op != litPtr) memmove(op, litPtr, lastLLSize);
1910
+ op += lastLLSize;
1911
+ }
1906
1912
  }
1907
1913
  }
1908
1914