zstdlib 0.6.0-x86-mingw32 → 0.9.0-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +20 -0
  3. data/README.md +7 -1
  4. data/Rakefile +38 -8
  5. data/ext/{zstdlib → zstdlib_c}/extconf.rb +10 -5
  6. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.2/zstdlib.c +2 -2
  7. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.3/zstdlib.c +2 -2
  8. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.4/zstdlib.c +2 -2
  9. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.5/zstdlib.c +2 -2
  10. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.6/zstdlib.c +2 -2
  11. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.7/zstdlib.c +2 -2
  12. data/ext/zstdlib_c/ruby/zlib-3.0/zstdlib.c +4994 -0
  13. data/ext/zstdlib_c/ruby/zlib-3.1/zstdlib.c +5076 -0
  14. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/adler32.c +0 -0
  15. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/compress.c +0 -0
  16. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/crc32.c +0 -0
  17. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/crc32.h +0 -0
  18. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/deflate.c +0 -0
  19. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/deflate.h +0 -0
  20. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzclose.c +0 -0
  21. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzguts.h +0 -0
  22. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzlib.c +0 -0
  23. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzread.c +0 -0
  24. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzwrite.c +0 -0
  25. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/infback.c +0 -0
  26. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffast.c +0 -0
  27. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffast.h +0 -0
  28. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffixed.h +0 -0
  29. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inflate.c +0 -0
  30. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inflate.h +0 -0
  31. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inftrees.c +0 -0
  32. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inftrees.h +0 -0
  33. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/trees.c +0 -0
  34. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/trees.h +0 -0
  35. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/uncompr.c +0 -0
  36. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zconf.h +0 -0
  37. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zlib.h +0 -0
  38. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zutil.c +0 -0
  39. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zutil.h +0 -0
  40. data/ext/{zstdlib → zstdlib_c}/zlib.mk +0 -0
  41. data/ext/{zstdlib → zstdlib_c}/zlibwrapper/zlibwrapper.c +1 -5
  42. data/ext/{zstdlib → zstdlib_c}/zlibwrapper.mk +0 -0
  43. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/bitstream.h +75 -57
  44. data/ext/zstdlib_c/zstd-1.5.2/lib/common/compiler.h +335 -0
  45. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/cpu.h +1 -3
  46. data/ext/zstdlib_c/zstd-1.5.2/lib/common/debug.c +24 -0
  47. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/debug.h +22 -49
  48. data/ext/zstdlib_c/zstd-1.5.2/lib/common/entropy_common.c +368 -0
  49. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/error_private.c +3 -1
  50. data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.h +159 -0
  51. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/fse.h +51 -42
  52. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/fse_decompress.c +149 -57
  53. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/huf.h +60 -54
  54. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/mem.h +87 -98
  55. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/pool.c +34 -23
  56. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/pool.h +5 -5
  57. data/ext/zstdlib_c/zstd-1.5.2/lib/common/portability_macros.h +137 -0
  58. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/threading.c +10 -8
  59. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/threading.h +4 -3
  60. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.c +24 -0
  61. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.h +5686 -0
  62. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_common.c +10 -10
  63. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_deps.h +111 -0
  64. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_internal.h +493 -0
  65. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_trace.h +163 -0
  66. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/clevels.h +134 -0
  67. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/fse_compress.c +105 -85
  68. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.c +41 -63
  69. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.h +13 -33
  70. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/huf_compress.c +1370 -0
  71. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress.c +6327 -0
  72. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_internal.h +537 -82
  73. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.c +21 -16
  74. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.h +4 -2
  75. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.c +61 -34
  76. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.h +10 -3
  77. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_superblock.c +573 -0
  78. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_superblock.h +32 -0
  79. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_cwksp.h +236 -95
  80. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.c +321 -143
  81. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.h +2 -2
  82. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.c +328 -137
  83. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.h +2 -2
  84. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.c +2104 -0
  85. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.h +125 -0
  86. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.c +336 -209
  87. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.h +15 -3
  88. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm_geartab.h +106 -0
  89. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.c +439 -239
  90. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.h +1 -1
  91. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstdmt_compress.c +205 -462
  92. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.h +113 -0
  93. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress.c +1889 -0
  94. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress_amd64.S +585 -0
  95. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.c +20 -16
  96. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.h +3 -3
  97. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress.c +691 -230
  98. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/zstd_decompress_block.c +2072 -0
  99. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.h +16 -7
  100. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_internal.h +71 -10
  101. data/ext/zstdlib_c/zstd-1.5.2/lib/zdict.h +452 -0
  102. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/zstd.h +760 -234
  103. data/ext/{zstdlib/zstd-1.4.4/lib/common → zstdlib_c/zstd-1.5.2/lib}/zstd_errors.h +3 -1
  104. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzclose.c +0 -0
  105. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzcompatibility.h +1 -1
  106. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzguts.h +0 -0
  107. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzlib.c +0 -0
  108. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzread.c +0 -0
  109. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzwrite.c +0 -0
  110. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.c +133 -44
  111. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  112. data/ext/zstdlib_c/zstd.mk +15 -0
  113. data/lib/2.4/zstdlib_c.so +0 -0
  114. data/lib/2.5/zstdlib_c.so +0 -0
  115. data/lib/2.6/zstdlib_c.so +0 -0
  116. data/lib/2.7/zstdlib_c.so +0 -0
  117. data/lib/3.0/zstdlib_c.so +0 -0
  118. data/lib/3.1/zstdlib_c.so +0 -0
  119. data/lib/zstdlib.rb +2 -2
  120. metadata +125 -114
  121. data/ext/zstdlib/zstd-1.4.4/lib/common/compiler.h +0 -159
  122. data/ext/zstdlib/zstd-1.4.4/lib/common/debug.c +0 -44
  123. data/ext/zstdlib/zstd-1.4.4/lib/common/entropy_common.c +0 -236
  124. data/ext/zstdlib/zstd-1.4.4/lib/common/error_private.h +0 -76
  125. data/ext/zstdlib/zstd-1.4.4/lib/common/xxhash.c +0 -882
  126. data/ext/zstdlib/zstd-1.4.4/lib/common/xxhash.h +0 -305
  127. data/ext/zstdlib/zstd-1.4.4/lib/common/zstd_internal.h +0 -350
  128. data/ext/zstdlib/zstd-1.4.4/lib/compress/huf_compress.c +0 -798
  129. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_compress.c +0 -4103
  130. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_lazy.c +0 -1115
  131. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_lazy.h +0 -67
  132. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstdmt_compress.h +0 -192
  133. data/ext/zstdlib/zstd-1.4.4/lib/decompress/huf_decompress.c +0 -1234
  134. data/ext/zstdlib/zstd-1.4.4/lib/decompress/zstd_decompress_block.c +0 -1323
  135. data/ext/zstdlib/zstd.mk +0 -14
  136. data/lib/2.2/zstdlib.so +0 -0
  137. data/lib/2.3/zstdlib.so +0 -0
  138. data/lib/2.4/zstdlib.so +0 -0
  139. data/lib/2.5/zstdlib.so +0 -0
  140. data/lib/2.6/zstdlib.so +0 -0
  141. data/lib/2.7/zstdlib.so +0 -0
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -15,10 +15,10 @@
15
15
 
16
16
  size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
17
17
  {
18
- BYTE* const ostart = (BYTE* const)dst;
18
+ BYTE* const ostart = (BYTE*)dst;
19
19
  U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
20
20
 
21
- RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall);
21
+ RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
22
22
 
23
23
  switch(flSize)
24
24
  {
@@ -35,13 +35,14 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
35
35
  assert(0);
36
36
  }
37
37
 
38
- memcpy(ostart + flSize, src, srcSize);
38
+ ZSTD_memcpy(ostart + flSize, src, srcSize);
39
+ DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
39
40
  return srcSize + flSize;
40
41
  }
41
42
 
42
43
  size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
43
44
  {
44
- BYTE* const ostart = (BYTE* const)dst;
45
+ BYTE* const ostart = (BYTE*)dst;
45
46
  U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
46
47
 
47
48
  (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
@@ -62,6 +63,7 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
62
63
  }
63
64
 
64
65
  ostart[flSize] = *(const BYTE*)src;
66
+ DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1);
65
67
  return flSize+1;
66
68
  }
67
69
 
@@ -71,7 +73,8 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
71
73
  void* dst, size_t dstCapacity,
72
74
  const void* src, size_t srcSize,
73
75
  void* entropyWorkspace, size_t entropyWorkspaceSize,
74
- const int bmi2)
76
+ const int bmi2,
77
+ unsigned suspectUncompressible)
75
78
  {
76
79
  size_t const minGain = ZSTD_minGain(srcSize, strategy);
77
80
  size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
@@ -80,11 +83,11 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
80
83
  symbolEncodingType_e hType = set_compressed;
81
84
  size_t cLitSize;
82
85
 
83
- DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i)",
84
- disableLiteralCompression);
86
+ DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)",
87
+ disableLiteralCompression, (U32)srcSize);
85
88
 
86
89
  /* Prepare nextEntropy assuming reusing the existing table */
87
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
90
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
88
91
 
89
92
  if (disableLiteralCompression)
90
93
  return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
@@ -102,24 +105,25 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
102
105
  cLitSize = singleStream ?
103
106
  HUF_compress1X_repeat(
104
107
  ostart+lhSize, dstCapacity-lhSize, src, srcSize,
105
- 255, 11, entropyWorkspace, entropyWorkspaceSize,
106
- (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) :
108
+ HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
109
+ (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible) :
107
110
  HUF_compress4X_repeat(
108
111
  ostart+lhSize, dstCapacity-lhSize, src, srcSize,
109
- 255, 11, entropyWorkspace, entropyWorkspaceSize,
110
- (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2);
112
+ HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
113
+ (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible);
111
114
  if (repeat != HUF_repeat_none) {
112
115
  /* reused the existing table */
116
+ DEBUGLOG(5, "Reusing previous huffman table");
113
117
  hType = set_repeat;
114
118
  }
115
119
  }
116
120
 
117
- if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) {
118
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
121
+ if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
122
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
119
123
  return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
120
124
  }
121
125
  if (cLitSize==1) {
122
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
126
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
123
127
  return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
124
128
  }
125
129
 
@@ -150,5 +154,6 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
150
154
  default: /* not possible : lhSize is {3,4,5} */
151
155
  assert(0);
152
156
  }
157
+ DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize));
153
158
  return lhSize+cLitSize;
154
159
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -18,12 +18,14 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
18
18
 
19
19
  size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
20
20
 
21
+ /* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
21
22
  size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
22
23
  ZSTD_hufCTables_t* nextHuf,
23
24
  ZSTD_strategy strategy, int disableLiteralCompression,
24
25
  void* dst, size_t dstCapacity,
25
26
  const void* src, size_t srcSize,
26
27
  void* entropyWorkspace, size_t entropyWorkspaceSize,
27
- const int bmi2);
28
+ const int bmi2,
29
+ unsigned suspectUncompressible);
28
30
 
29
31
  #endif /* ZSTD_COMPRESS_LITERALS_H */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -50,6 +50,19 @@ static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) {
50
50
  return maxSymbolValue;
51
51
  }
52
52
 
53
+ /**
54
+ * Returns true if we should use ncount=-1 else we should
55
+ * use ncount=1 for low probability symbols instead.
56
+ */
57
+ static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
58
+ {
59
+ /* Heuristic: This should cover most blocks <= 16K and
60
+ * start to fade out after 16K to about 32K depending on
61
+ * comprssibility.
62
+ */
63
+ return nbSeq >= 2048;
64
+ }
65
+
53
66
  /**
54
67
  * Returns the cost in bytes of encoding the normalized count header.
55
68
  * Returns an error if any of the helper functions return an error.
@@ -60,7 +73,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
60
73
  BYTE wksp[FSE_NCOUNTBOUND];
61
74
  S16 norm[MaxSeq + 1];
62
75
  const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
63
- FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max));
76
+ FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), "");
64
77
  return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
65
78
  }
66
79
 
@@ -72,6 +85,8 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
72
85
  {
73
86
  unsigned cost = 0;
74
87
  unsigned s;
88
+
89
+ assert(total > 0);
75
90
  for (s = 0; s <= max; ++s) {
76
91
  unsigned norm = (unsigned)((256 * count[s]) / total);
77
92
  if (count[s] != 0 && norm == 0)
@@ -86,7 +101,7 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
86
101
  * Returns the cost in bits of encoding the distribution in count using ctable.
87
102
  * Returns an error if ctable cannot represent all the symbols in count.
88
103
  */
89
- static size_t ZSTD_fseBitCost(
104
+ size_t ZSTD_fseBitCost(
90
105
  FSE_CTable const* ctable,
91
106
  unsigned const* count,
92
107
  unsigned const max)
@@ -96,18 +111,22 @@ static size_t ZSTD_fseBitCost(
96
111
  unsigned s;
97
112
  FSE_CState_t cstate;
98
113
  FSE_initCState(&cstate, ctable);
99
- RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC,
100
- "Repeat FSE_CTable has maxSymbolValue %u < %u",
114
+ if (ZSTD_getFSEMaxSymbolValue(ctable) < max) {
115
+ DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u",
101
116
  ZSTD_getFSEMaxSymbolValue(ctable), max);
117
+ return ERROR(GENERIC);
118
+ }
102
119
  for (s = 0; s <= max; ++s) {
103
120
  unsigned const tableLog = cstate.stateLog;
104
121
  unsigned const badCost = (tableLog + 1) << kAccuracyLog;
105
122
  unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog);
106
123
  if (count[s] == 0)
107
124
  continue;
108
- RETURN_ERROR_IF(bitCost >= badCost, GENERIC,
109
- "Repeat FSE_CTable has Prob[%u] == 0", s);
110
- cost += count[s] * bitCost;
125
+ if (bitCost >= badCost) {
126
+ DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s);
127
+ return ERROR(GENERIC);
128
+ }
129
+ cost += (size_t)count[s] * bitCost;
111
130
  }
112
131
  return cost >> kAccuracyLog;
113
132
  }
@@ -117,15 +136,15 @@ static size_t ZSTD_fseBitCost(
117
136
  * table described by norm. The max symbol support by norm is assumed >= max.
118
137
  * norm must be valid for every symbol with non-zero probability in count.
119
138
  */
120
- static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
121
- unsigned const* count, unsigned const max)
139
+ size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
140
+ unsigned const* count, unsigned const max)
122
141
  {
123
142
  unsigned const shift = 8 - accuracyLog;
124
143
  size_t cost = 0;
125
144
  unsigned s;
126
145
  assert(accuracyLog <= 8);
127
146
  for (s = 0; s <= max; ++s) {
128
- unsigned const normAcc = norm[s] != -1 ? norm[s] : 1;
147
+ unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1;
129
148
  unsigned const norm256 = normAcc << shift;
130
149
  assert(norm256 > 0);
131
150
  assert(norm256 < 256);
@@ -215,6 +234,11 @@ ZSTD_selectEncodingType(
215
234
  return set_compressed;
216
235
  }
217
236
 
237
+ typedef struct {
238
+ S16 norm[MaxSeq + 1];
239
+ U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)];
240
+ } ZSTD_BuildCTableWksp;
241
+
218
242
  size_t
219
243
  ZSTD_buildCTable(void* dst, size_t dstCapacity,
220
244
  FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
@@ -230,18 +254,18 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
230
254
 
231
255
  switch (type) {
232
256
  case set_rle:
233
- FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max));
234
- RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall);
257
+ FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), "");
258
+ RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space");
235
259
  *op = codeTable[0];
236
260
  return 1;
237
261
  case set_repeat:
238
- memcpy(nextCTable, prevCTable, prevCTableSize);
262
+ ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize);
239
263
  return 0;
240
264
  case set_basic:
241
- FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */
265
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */
242
266
  return 0;
243
267
  case set_compressed: {
244
- S16 norm[MaxSeq + 1];
268
+ ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace;
245
269
  size_t nbSeq_1 = nbSeq;
246
270
  const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
247
271
  if (count[codeTable[nbSeq-1]] > 1) {
@@ -249,14 +273,17 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
249
273
  nbSeq_1--;
250
274
  }
251
275
  assert(nbSeq_1 > 1);
252
- FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
253
- { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
254
- FORWARD_IF_ERROR(NCountSize);
255
- FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize));
276
+ assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp));
277
+ (void)entropyWorkspaceSize;
278
+ FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed");
279
+ assert(oend >= op);
280
+ { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */
281
+ FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed");
282
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed");
256
283
  return NCountSize;
257
284
  }
258
285
  }
259
- default: assert(0); RETURN_ERROR(GENERIC);
286
+ default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach");
260
287
  }
261
288
  }
262
289
 
@@ -286,19 +313,19 @@ ZSTD_encodeSequences_body(
286
313
  FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]);
287
314
  BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]);
288
315
  if (MEM_32bits()) BIT_flushBits(&blockStream);
289
- BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]);
316
+ BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]);
290
317
  if (MEM_32bits()) BIT_flushBits(&blockStream);
291
318
  if (longOffsets) {
292
319
  U32 const ofBits = ofCodeTable[nbSeq-1];
293
- int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
320
+ unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
294
321
  if (extraBits) {
295
- BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits);
322
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits);
296
323
  BIT_flushBits(&blockStream);
297
324
  }
298
- BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits,
325
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits,
299
326
  ofBits - extraBits);
300
327
  } else {
301
- BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
328
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]);
302
329
  }
303
330
  BIT_flushBits(&blockStream);
304
331
 
@@ -312,8 +339,8 @@ ZSTD_encodeSequences_body(
312
339
  U32 const mlBits = ML_bits[mlCode];
313
340
  DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u",
314
341
  (unsigned)sequences[n].litLength,
315
- (unsigned)sequences[n].matchLength + MINMATCH,
316
- (unsigned)sequences[n].offset);
342
+ (unsigned)sequences[n].mlBase + MINMATCH,
343
+ (unsigned)sequences[n].offBase);
317
344
  /* 32b*/ /* 64b*/
318
345
  /* (7)*/ /* (7)*/
319
346
  FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
@@ -324,18 +351,18 @@ ZSTD_encodeSequences_body(
324
351
  BIT_flushBits(&blockStream); /* (7)*/
325
352
  BIT_addBits(&blockStream, sequences[n].litLength, llBits);
326
353
  if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
327
- BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
354
+ BIT_addBits(&blockStream, sequences[n].mlBase, mlBits);
328
355
  if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream);
329
356
  if (longOffsets) {
330
- int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
357
+ unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
331
358
  if (extraBits) {
332
- BIT_addBits(&blockStream, sequences[n].offset, extraBits);
359
+ BIT_addBits(&blockStream, sequences[n].offBase, extraBits);
333
360
  BIT_flushBits(&blockStream); /* (7)*/
334
361
  }
335
- BIT_addBits(&blockStream, sequences[n].offset >> extraBits,
362
+ BIT_addBits(&blockStream, sequences[n].offBase >> extraBits,
336
363
  ofBits - extraBits); /* 31 */
337
364
  } else {
338
- BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */
365
+ BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */
339
366
  }
340
367
  BIT_flushBits(&blockStream); /* (7)*/
341
368
  DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr));
@@ -372,7 +399,7 @@ ZSTD_encodeSequences_default(
372
399
 
373
400
  #if DYNAMIC_BMI2
374
401
 
375
- static TARGET_ATTRIBUTE("bmi2") size_t
402
+ static BMI2_TARGET_ATTRIBUTE size_t
376
403
  ZSTD_encodeSequences_bmi2(
377
404
  void* dst, size_t dstCapacity,
378
405
  FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -11,8 +11,8 @@
11
11
  #ifndef ZSTD_COMPRESS_SEQUENCES_H
12
12
  #define ZSTD_COMPRESS_SEQUENCES_H
13
13
 
14
- #include "fse.h" /* FSE_repeat, FSE_CTable */
15
- #include "zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */
14
+ #include "../common/fse.h" /* FSE_repeat, FSE_CTable */
15
+ #include "../common/zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */
16
16
 
17
17
  typedef enum {
18
18
  ZSTD_defaultDisallowed = 0,
@@ -44,4 +44,11 @@ size_t ZSTD_encodeSequences(
44
44
  FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
45
45
  seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
46
46
 
47
+ size_t ZSTD_fseBitCost(
48
+ FSE_CTable const* ctable,
49
+ unsigned const* count,
50
+ unsigned const max);
51
+
52
+ size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
53
+ unsigned const* count, unsigned const max);
47
54
  #endif /* ZSTD_COMPRESS_SEQUENCES_H */