zstdlib 0.7.0-x64-mingw32 → 0.8.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +5 -0
  3. data/ext/zstdlib/extconf.rb +1 -1
  4. data/ext/zstdlib/ruby/zlib-3.0/zstdlib.c +4994 -0
  5. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/bitstream.h +25 -16
  6. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/compiler.h +118 -4
  7. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/cpu.h +1 -3
  8. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/debug.c +1 -1
  9. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/debug.h +12 -19
  10. data/ext/zstdlib/zstd-1.5.0/lib/common/entropy_common.c +362 -0
  11. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/error_private.c +2 -1
  12. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/error_private.h +3 -3
  13. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/fse.h +40 -12
  14. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/fse_decompress.c +139 -22
  15. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/huf.h +29 -7
  16. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/mem.h +69 -98
  17. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/pool.c +23 -17
  18. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/pool.h +2 -2
  19. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/threading.c +6 -5
  20. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/threading.h +0 -0
  21. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/xxhash.c +20 -60
  22. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/xxhash.h +2 -2
  23. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/zstd_common.c +10 -10
  24. data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_deps.h +111 -0
  25. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/zstd_internal.h +105 -62
  26. data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_trace.h +154 -0
  27. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/fse_compress.c +31 -24
  28. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/hist.c +27 -29
  29. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/hist.h +2 -2
  30. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/huf_compress.c +265 -126
  31. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress.c +2843 -728
  32. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_internal.h +305 -63
  33. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_literals.c +8 -8
  34. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_literals.h +1 -1
  35. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.c +29 -7
  36. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.h +1 -1
  37. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_superblock.c +22 -295
  38. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_superblock.h +1 -1
  39. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_cwksp.h +204 -67
  40. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_double_fast.c +25 -25
  41. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_double_fast.h +1 -1
  42. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_fast.c +23 -23
  43. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_fast.h +1 -1
  44. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.c +2184 -0
  45. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.h +125 -0
  46. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_ldm.c +314 -211
  47. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_ldm.h +9 -2
  48. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_ldm_geartab.h +103 -0
  49. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_opt.c +191 -46
  50. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_opt.h +1 -1
  51. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstdmt_compress.c +93 -415
  52. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstdmt_compress.h +110 -0
  53. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/huf_decompress.c +342 -239
  54. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_ddict.c +9 -9
  55. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_ddict.h +2 -2
  56. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress.c +369 -87
  57. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.c +191 -75
  58. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.h +6 -3
  59. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress_internal.h +27 -11
  60. data/ext/zstdlib/zstd-1.5.0/lib/zdict.h +452 -0
  61. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/zstd.h +568 -126
  62. data/ext/zstdlib/{zstd-1.4.5/lib/common → zstd-1.5.0/lib}/zstd_errors.h +2 -1
  63. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzclose.c +0 -0
  64. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzcompatibility.h +1 -1
  65. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzguts.h +0 -0
  66. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzlib.c +0 -0
  67. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzread.c +0 -0
  68. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzwrite.c +0 -0
  69. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.c +126 -44
  70. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  71. data/lib/2.2/zstdlib.so +0 -0
  72. data/lib/2.3/zstdlib.so +0 -0
  73. data/lib/2.4/zstdlib.so +0 -0
  74. data/lib/2.5/zstdlib.so +0 -0
  75. data/lib/2.6/zstdlib.so +0 -0
  76. data/lib/2.7/zstdlib.so +0 -0
  77. metadata +69 -64
  78. data/ext/zstdlib/zstd-1.4.5/lib/common/entropy_common.c +0 -216
  79. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.c +0 -1138
  80. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.h +0 -67
  81. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstdmt_compress.h +0 -192
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -15,7 +15,7 @@
15
15
 
16
16
  size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
17
17
  {
18
- BYTE* const ostart = (BYTE* const)dst;
18
+ BYTE* const ostart = (BYTE*)dst;
19
19
  U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
20
20
 
21
21
  RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
@@ -35,14 +35,14 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
35
35
  assert(0);
36
36
  }
37
37
 
38
- memcpy(ostart + flSize, src, srcSize);
38
+ ZSTD_memcpy(ostart + flSize, src, srcSize);
39
39
  DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
40
40
  return srcSize + flSize;
41
41
  }
42
42
 
43
43
  size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
44
44
  {
45
- BYTE* const ostart = (BYTE* const)dst;
45
+ BYTE* const ostart = (BYTE*)dst;
46
46
  U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
47
47
 
48
48
  (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
@@ -86,7 +86,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
86
86
  disableLiteralCompression, (U32)srcSize);
87
87
 
88
88
  /* Prepare nextEntropy assuming reusing the existing table */
89
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
89
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
90
90
 
91
91
  if (disableLiteralCompression)
92
92
  return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
@@ -117,12 +117,12 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
117
117
  }
118
118
  }
119
119
 
120
- if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) {
121
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
120
+ if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
121
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
122
122
  return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
123
123
  }
124
124
  if (cLitSize==1) {
125
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
125
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
126
126
  return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
127
127
  }
128
128
 
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -50,6 +50,19 @@ static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) {
50
50
  return maxSymbolValue;
51
51
  }
52
52
 
53
+ /**
54
+ * Returns true if we should use ncount=-1 else we should
55
+ * use ncount=1 for low probability symbols instead.
56
+ */
57
+ static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
58
+ {
59
+ /* Heuristic: This should cover most blocks <= 16K and
60
+ * start to fade out after 16K to about 32K depending on
61
+ * comprssibility.
62
+ */
63
+ return nbSeq >= 2048;
64
+ }
65
+
53
66
  /**
54
67
  * Returns the cost in bytes of encoding the normalized count header.
55
68
  * Returns an error if any of the helper functions return an error.
@@ -60,7 +73,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
60
73
  BYTE wksp[FSE_NCOUNTBOUND];
61
74
  S16 norm[MaxSeq + 1];
62
75
  const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
63
- FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max), "");
76
+ FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), "");
64
77
  return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
65
78
  }
66
79
 
@@ -72,6 +85,8 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
72
85
  {
73
86
  unsigned cost = 0;
74
87
  unsigned s;
88
+
89
+ assert(total > 0);
75
90
  for (s = 0; s <= max; ++s) {
76
91
  unsigned norm = (unsigned)((256 * count[s]) / total);
77
92
  if (count[s] != 0 && norm == 0)
@@ -219,6 +234,11 @@ ZSTD_selectEncodingType(
219
234
  return set_compressed;
220
235
  }
221
236
 
237
+ typedef struct {
238
+ S16 norm[MaxSeq + 1];
239
+ U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)];
240
+ } ZSTD_BuildCTableWksp;
241
+
222
242
  size_t
223
243
  ZSTD_buildCTable(void* dst, size_t dstCapacity,
224
244
  FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
@@ -239,13 +259,13 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
239
259
  *op = codeTable[0];
240
260
  return 1;
241
261
  case set_repeat:
242
- memcpy(nextCTable, prevCTable, prevCTableSize);
262
+ ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize);
243
263
  return 0;
244
264
  case set_basic:
245
265
  FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */
246
266
  return 0;
247
267
  case set_compressed: {
248
- S16 norm[MaxSeq + 1];
268
+ ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace;
249
269
  size_t nbSeq_1 = nbSeq;
250
270
  const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
251
271
  if (count[codeTable[nbSeq-1]] > 1) {
@@ -253,10 +273,12 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
253
273
  nbSeq_1--;
254
274
  }
255
275
  assert(nbSeq_1 > 1);
256
- FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max), "");
257
- { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
276
+ assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp));
277
+ (void)entropyWorkspaceSize;
278
+ FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "");
279
+ { size_t const NCountSize = FSE_writeNCount(op, oend - op, wksp->norm, max, tableLog); /* overflow protected */
258
280
  FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed");
259
- FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize), "");
281
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "");
260
282
  return NCountSize;
261
283
  }
262
284
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -15,288 +15,10 @@
15
15
 
16
16
  #include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */
17
17
  #include "hist.h" /* HIST_countFast_wksp */
18
- #include "zstd_compress_internal.h"
18
+ #include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */
19
19
  #include "zstd_compress_sequences.h"
20
20
  #include "zstd_compress_literals.h"
21
21
 
22
- /*-*************************************
23
- * Superblock entropy buffer structs
24
- ***************************************/
25
- /** ZSTD_hufCTablesMetadata_t :
26
- * Stores Literals Block Type for a super-block in hType, and
27
- * huffman tree description in hufDesBuffer.
28
- * hufDesSize refers to the size of huffman tree description in bytes.
29
- * This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */
30
- typedef struct {
31
- symbolEncodingType_e hType;
32
- BYTE hufDesBuffer[500]; /* TODO give name to this value */
33
- size_t hufDesSize;
34
- } ZSTD_hufCTablesMetadata_t;
35
-
36
- /** ZSTD_fseCTablesMetadata_t :
37
- * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
38
- * fse tables in fseTablesBuffer.
39
- * fseTablesSize refers to the size of fse tables in bytes.
40
- * This metadata is populated in ZSTD_buildSuperBlockEntropy_sequences() */
41
- typedef struct {
42
- symbolEncodingType_e llType;
43
- symbolEncodingType_e ofType;
44
- symbolEncodingType_e mlType;
45
- BYTE fseTablesBuffer[500]; /* TODO give name to this value */
46
- size_t fseTablesSize;
47
- size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */
48
- } ZSTD_fseCTablesMetadata_t;
49
-
50
- typedef struct {
51
- ZSTD_hufCTablesMetadata_t hufMetadata;
52
- ZSTD_fseCTablesMetadata_t fseMetadata;
53
- } ZSTD_entropyCTablesMetadata_t;
54
-
55
-
56
- /** ZSTD_buildSuperBlockEntropy_literal() :
57
- * Builds entropy for the super-block literals.
58
- * Stores literals block type (raw, rle, compressed, repeat) and
59
- * huffman description table to hufMetadata.
60
- * @return : size of huffman description table or error code */
61
- static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSize,
62
- const ZSTD_hufCTables_t* prevHuf,
63
- ZSTD_hufCTables_t* nextHuf,
64
- ZSTD_hufCTablesMetadata_t* hufMetadata,
65
- const int disableLiteralsCompression,
66
- void* workspace, size_t wkspSize)
67
- {
68
- BYTE* const wkspStart = (BYTE*)workspace;
69
- BYTE* const wkspEnd = wkspStart + wkspSize;
70
- BYTE* const countWkspStart = wkspStart;
71
- unsigned* const countWksp = (unsigned*)workspace;
72
- const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
73
- BYTE* const nodeWksp = countWkspStart + countWkspSize;
74
- const size_t nodeWkspSize = wkspEnd-nodeWksp;
75
- unsigned maxSymbolValue = 255;
76
- unsigned huffLog = HUF_TABLELOG_DEFAULT;
77
- HUF_repeat repeat = prevHuf->repeatMode;
78
-
79
- DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize);
80
-
81
- /* Prepare nextEntropy assuming reusing the existing table */
82
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
83
-
84
- if (disableLiteralsCompression) {
85
- DEBUGLOG(5, "set_basic - disabled");
86
- hufMetadata->hType = set_basic;
87
- return 0;
88
- }
89
-
90
- /* small ? don't even attempt compression (speed opt) */
91
- # define COMPRESS_LITERALS_SIZE_MIN 63
92
- { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
93
- if (srcSize <= minLitSize) {
94
- DEBUGLOG(5, "set_basic - too small");
95
- hufMetadata->hType = set_basic;
96
- return 0;
97
- }
98
- }
99
-
100
- /* Scan input and build symbol stats */
101
- { size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize);
102
- FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
103
- if (largest == srcSize) {
104
- DEBUGLOG(5, "set_rle");
105
- hufMetadata->hType = set_rle;
106
- return 0;
107
- }
108
- if (largest <= (srcSize >> 7)+4) {
109
- DEBUGLOG(5, "set_basic - no gain");
110
- hufMetadata->hType = set_basic;
111
- return 0;
112
- }
113
- }
114
-
115
- /* Validate the previous Huffman table */
116
- if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
117
- repeat = HUF_repeat_none;
118
- }
119
-
120
- /* Build Huffman Tree */
121
- memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
122
- huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
123
- { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
124
- maxSymbolValue, huffLog,
125
- nodeWksp, nodeWkspSize);
126
- FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
127
- huffLog = (U32)maxBits;
128
- { /* Build and write the CTable */
129
- size_t const newCSize = HUF_estimateCompressedSize(
130
- (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
131
- size_t const hSize = HUF_writeCTable(
132
- hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
133
- (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog);
134
- /* Check against repeating the previous CTable */
135
- if (repeat != HUF_repeat_none) {
136
- size_t const oldCSize = HUF_estimateCompressedSize(
137
- (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
138
- if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
139
- DEBUGLOG(5, "set_repeat - smaller");
140
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
141
- hufMetadata->hType = set_repeat;
142
- return 0;
143
- }
144
- }
145
- if (newCSize + hSize >= srcSize) {
146
- DEBUGLOG(5, "set_basic - no gains");
147
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
148
- hufMetadata->hType = set_basic;
149
- return 0;
150
- }
151
- DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
152
- hufMetadata->hType = set_compressed;
153
- nextHuf->repeatMode = HUF_repeat_check;
154
- return hSize;
155
- }
156
- }
157
- }
158
-
159
- /** ZSTD_buildSuperBlockEntropy_sequences() :
160
- * Builds entropy for the super-block sequences.
161
- * Stores symbol compression modes and fse table to fseMetadata.
162
- * @return : size of fse tables or error code */
163
- static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr,
164
- const ZSTD_fseCTables_t* prevEntropy,
165
- ZSTD_fseCTables_t* nextEntropy,
166
- const ZSTD_CCtx_params* cctxParams,
167
- ZSTD_fseCTablesMetadata_t* fseMetadata,
168
- void* workspace, size_t wkspSize)
169
- {
170
- BYTE* const wkspStart = (BYTE*)workspace;
171
- BYTE* const wkspEnd = wkspStart + wkspSize;
172
- BYTE* const countWkspStart = wkspStart;
173
- unsigned* const countWksp = (unsigned*)workspace;
174
- const size_t countWkspSize = (MaxSeq + 1) * sizeof(unsigned);
175
- BYTE* const cTableWksp = countWkspStart + countWkspSize;
176
- const size_t cTableWkspSize = wkspEnd-cTableWksp;
177
- ZSTD_strategy const strategy = cctxParams->cParams.strategy;
178
- FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
179
- FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
180
- FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
181
- const BYTE* const ofCodeTable = seqStorePtr->ofCode;
182
- const BYTE* const llCodeTable = seqStorePtr->llCode;
183
- const BYTE* const mlCodeTable = seqStorePtr->mlCode;
184
- size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
185
- BYTE* const ostart = fseMetadata->fseTablesBuffer;
186
- BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
187
- BYTE* op = ostart;
188
-
189
- assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE));
190
- DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq);
191
- memset(workspace, 0, wkspSize);
192
-
193
- fseMetadata->lastCountSize = 0;
194
- /* convert length/distances into codes */
195
- ZSTD_seqToCodes(seqStorePtr);
196
- /* build CTable for Literal Lengths */
197
- { U32 LLtype;
198
- unsigned max = MaxLL;
199
- size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
200
- DEBUGLOG(5, "Building LL table");
201
- nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
202
- LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode,
203
- countWksp, max, mostFrequent, nbSeq,
204
- LLFSELog, prevEntropy->litlengthCTable,
205
- LL_defaultNorm, LL_defaultNormLog,
206
- ZSTD_defaultAllowed, strategy);
207
- assert(set_basic < set_compressed && set_rle < set_compressed);
208
- assert(!(LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
209
- { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
210
- countWksp, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
211
- prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable),
212
- cTableWksp, cTableWkspSize);
213
- FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed");
214
- if (LLtype == set_compressed)
215
- fseMetadata->lastCountSize = countSize;
216
- op += countSize;
217
- fseMetadata->llType = (symbolEncodingType_e) LLtype;
218
- } }
219
- /* build CTable for Offsets */
220
- { U32 Offtype;
221
- unsigned max = MaxOff;
222
- size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
223
- /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
224
- ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
225
- DEBUGLOG(5, "Building OF table");
226
- nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
227
- Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
228
- countWksp, max, mostFrequent, nbSeq,
229
- OffFSELog, prevEntropy->offcodeCTable,
230
- OF_defaultNorm, OF_defaultNormLog,
231
- defaultPolicy, strategy);
232
- assert(!(Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
233
- { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
234
- countWksp, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
235
- prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable),
236
- cTableWksp, cTableWkspSize);
237
- FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed");
238
- if (Offtype == set_compressed)
239
- fseMetadata->lastCountSize = countSize;
240
- op += countSize;
241
- fseMetadata->ofType = (symbolEncodingType_e) Offtype;
242
- } }
243
- /* build CTable for MatchLengths */
244
- { U32 MLtype;
245
- unsigned max = MaxML;
246
- size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
247
- DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
248
- nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
249
- MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode,
250
- countWksp, max, mostFrequent, nbSeq,
251
- MLFSELog, prevEntropy->matchlengthCTable,
252
- ML_defaultNorm, ML_defaultNormLog,
253
- ZSTD_defaultAllowed, strategy);
254
- assert(!(MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
255
- { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
256
- countWksp, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
257
- prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable),
258
- cTableWksp, cTableWkspSize);
259
- FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed");
260
- if (MLtype == set_compressed)
261
- fseMetadata->lastCountSize = countSize;
262
- op += countSize;
263
- fseMetadata->mlType = (symbolEncodingType_e) MLtype;
264
- } }
265
- assert((size_t) (op-ostart) <= sizeof(fseMetadata->fseTablesBuffer));
266
- return op-ostart;
267
- }
268
-
269
-
270
- /** ZSTD_buildSuperBlockEntropy() :
271
- * Builds entropy for the super-block.
272
- * @return : 0 on success or error code */
273
- static size_t
274
- ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
275
- const ZSTD_entropyCTables_t* prevEntropy,
276
- ZSTD_entropyCTables_t* nextEntropy,
277
- const ZSTD_CCtx_params* cctxParams,
278
- ZSTD_entropyCTablesMetadata_t* entropyMetadata,
279
- void* workspace, size_t wkspSize)
280
- {
281
- size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
282
- DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy");
283
- entropyMetadata->hufMetadata.hufDesSize =
284
- ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize,
285
- &prevEntropy->huf, &nextEntropy->huf,
286
- &entropyMetadata->hufMetadata,
287
- ZSTD_disableLiteralsCompression(cctxParams),
288
- workspace, wkspSize);
289
- FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal failed");
290
- entropyMetadata->fseMetadata.fseTablesSize =
291
- ZSTD_buildSuperBlockEntropy_sequences(seqStorePtr,
292
- &prevEntropy->fse, &nextEntropy->fse,
293
- cctxParams,
294
- &entropyMetadata->fseMetadata,
295
- workspace, wkspSize);
296
- FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequences failed");
297
- return 0;
298
- }
299
-
300
22
  /** ZSTD_compressSubBlock_literal() :
301
23
  * Compresses literals section for a sub-block.
302
24
  * When we have to write the Huffman table we will sometimes choose a header
@@ -304,7 +26,7 @@ ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
304
26
  * before we know the table size + compressed size, so we have a bound on the
305
27
  * table size. If we guessed incorrectly, we fall back to uncompressed literals.
306
28
  *
307
- * We write the header when writeEntropy=1 and set entropyWrriten=1 when we succeeded
29
+ * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded
308
30
  * in writing the header, otherwise it is set to 0.
309
31
  *
310
32
  * hufMetadata->hType has literals block type info.
@@ -348,7 +70,7 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
348
70
  assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat);
349
71
 
350
72
  if (writeEntropy && hufMetadata->hType == set_compressed) {
351
- memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize);
73
+ ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize);
352
74
  op += hufMetadata->hufDesSize;
353
75
  cLitSize += hufMetadata->hufDesSize;
354
76
  DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
@@ -474,7 +196,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
474
196
  const U32 MLtype = fseMetadata->mlType;
475
197
  DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize);
476
198
  *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
477
- memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize);
199
+ ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize);
478
200
  op += fseMetadata->fseTablesSize;
479
201
  } else {
480
202
  const U32 repeat = set_repeat;
@@ -603,7 +325,7 @@ static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
603
325
  const BYTE* codeTable, unsigned maxCode,
604
326
  size_t nbSeq, const FSE_CTable* fseCTable,
605
327
  const U32* additionalBits,
606
- short const* defaultNorm, U32 defaultNormLog,
328
+ short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
607
329
  void* workspace, size_t wkspSize)
608
330
  {
609
331
  unsigned* const countWksp = (unsigned*)workspace;
@@ -615,7 +337,11 @@ static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
615
337
 
616
338
  HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */
617
339
  if (type == set_basic) {
618
- cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max);
340
+ /* We selected this encoding type, so it must be valid. */
341
+ assert(max <= defaultMax);
342
+ cSymbolTypeSizeEstimateInBits = max <= defaultMax
343
+ ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max)
344
+ : ERROR(GENERIC);
619
345
  } else if (type == set_rle) {
620
346
  cSymbolTypeSizeEstimateInBits = 0;
621
347
  } else if (type == set_compressed || type == set_repeat) {
@@ -639,19 +365,20 @@ static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable,
639
365
  void* workspace, size_t wkspSize,
640
366
  int writeEntropy)
641
367
  {
642
- size_t sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
368
+ size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
643
369
  size_t cSeqSizeEstimate = 0;
370
+ if (nbSeq == 0) return sequencesSectionHeaderSize;
644
371
  cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff,
645
372
  nbSeq, fseTables->offcodeCTable, NULL,
646
- OF_defaultNorm, OF_defaultNormLog,
373
+ OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
647
374
  workspace, wkspSize);
648
375
  cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL,
649
376
  nbSeq, fseTables->litlengthCTable, LL_bits,
650
- LL_defaultNorm, LL_defaultNormLog,
377
+ LL_defaultNorm, LL_defaultNormLog, MaxLL,
651
378
  workspace, wkspSize);
652
379
  cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML,
653
380
  nbSeq, fseTables->matchlengthCTable, ML_bits,
654
- ML_defaultNorm, ML_defaultNormLog,
381
+ ML_defaultNorm, ML_defaultNormLog, MaxML,
655
382
  workspace, wkspSize);
656
383
  if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
657
384
  return cSeqSizeEstimate + sequencesSectionHeaderSize;
@@ -790,7 +517,7 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
790
517
  } while (!lastSequence);
791
518
  if (writeLitEntropy) {
792
519
  DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten");
793
- memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
520
+ ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
794
521
  }
795
522
  if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) {
796
523
  /* If we haven't written our entropy tables, then we've violated our contract and
@@ -809,11 +536,11 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
809
536
  if (sp < send) {
810
537
  seqDef const* seq;
811
538
  repcodes_t rep;
812
- memcpy(&rep, prevCBlock->rep, sizeof(rep));
539
+ ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
813
540
  for (seq = sstart; seq < sp; ++seq) {
814
541
  rep = ZSTD_updateRep(rep.rep, seq->offset - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
815
542
  }
816
- memcpy(nextCBlock->rep, &rep, sizeof(rep));
543
+ ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep));
817
544
  }
818
545
  }
819
546
  DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed");
@@ -826,12 +553,12 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
826
553
  unsigned lastBlock) {
827
554
  ZSTD_entropyCTablesMetadata_t entropyMetadata;
828
555
 
829
- FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(&zc->seqStore,
556
+ FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore,
830
557
  &zc->blockState.prevCBlock->entropy,
831
558
  &zc->blockState.nextCBlock->entropy,
832
559
  &zc->appliedParams,
833
560
  &entropyMetadata,
834
- zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
561
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
835
562
 
836
563
  return ZSTD_compressSubBlock_multi(&zc->seqStore,
837
564
  zc->blockState.prevCBlock,
@@ -841,5 +568,5 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
841
568
  dst, dstCapacity,
842
569
  src, srcSize,
843
570
  zc->bmi2, lastBlock,
844
- zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
571
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
845
572
  }