zstdlib 0.6.0-x86-mingw32 → 0.9.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (141) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +20 -0
  3. data/README.md +7 -1
  4. data/Rakefile +38 -8
  5. data/ext/{zstdlib → zstdlib_c}/extconf.rb +10 -5
  6. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.2/zstdlib.c +2 -2
  7. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.3/zstdlib.c +2 -2
  8. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.4/zstdlib.c +2 -2
  9. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.5/zstdlib.c +2 -2
  10. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.6/zstdlib.c +2 -2
  11. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.7/zstdlib.c +2 -2
  12. data/ext/zstdlib_c/ruby/zlib-3.0/zstdlib.c +4994 -0
  13. data/ext/zstdlib_c/ruby/zlib-3.1/zstdlib.c +5076 -0
  14. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/adler32.c +0 -0
  15. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/compress.c +0 -0
  16. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/crc32.c +0 -0
  17. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/crc32.h +0 -0
  18. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/deflate.c +0 -0
  19. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/deflate.h +0 -0
  20. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzclose.c +0 -0
  21. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzguts.h +0 -0
  22. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzlib.c +0 -0
  23. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzread.c +0 -0
  24. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzwrite.c +0 -0
  25. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/infback.c +0 -0
  26. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffast.c +0 -0
  27. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffast.h +0 -0
  28. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffixed.h +0 -0
  29. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inflate.c +0 -0
  30. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inflate.h +0 -0
  31. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inftrees.c +0 -0
  32. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inftrees.h +0 -0
  33. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/trees.c +0 -0
  34. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/trees.h +0 -0
  35. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/uncompr.c +0 -0
  36. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zconf.h +0 -0
  37. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zlib.h +0 -0
  38. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zutil.c +0 -0
  39. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zutil.h +0 -0
  40. data/ext/{zstdlib → zstdlib_c}/zlib.mk +0 -0
  41. data/ext/{zstdlib → zstdlib_c}/zlibwrapper/zlibwrapper.c +1 -5
  42. data/ext/{zstdlib → zstdlib_c}/zlibwrapper.mk +0 -0
  43. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/bitstream.h +75 -57
  44. data/ext/zstdlib_c/zstd-1.5.2/lib/common/compiler.h +335 -0
  45. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/cpu.h +1 -3
  46. data/ext/zstdlib_c/zstd-1.5.2/lib/common/debug.c +24 -0
  47. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/debug.h +22 -49
  48. data/ext/zstdlib_c/zstd-1.5.2/lib/common/entropy_common.c +368 -0
  49. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/error_private.c +3 -1
  50. data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.h +159 -0
  51. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/fse.h +51 -42
  52. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/fse_decompress.c +149 -57
  53. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/huf.h +60 -54
  54. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/mem.h +87 -98
  55. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/pool.c +34 -23
  56. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/pool.h +5 -5
  57. data/ext/zstdlib_c/zstd-1.5.2/lib/common/portability_macros.h +137 -0
  58. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/threading.c +10 -8
  59. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/threading.h +4 -3
  60. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.c +24 -0
  61. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.h +5686 -0
  62. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_common.c +10 -10
  63. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_deps.h +111 -0
  64. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_internal.h +493 -0
  65. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_trace.h +163 -0
  66. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/clevels.h +134 -0
  67. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/fse_compress.c +105 -85
  68. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.c +41 -63
  69. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.h +13 -33
  70. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/huf_compress.c +1370 -0
  71. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress.c +6327 -0
  72. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_internal.h +537 -82
  73. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.c +21 -16
  74. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.h +4 -2
  75. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.c +61 -34
  76. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.h +10 -3
  77. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_superblock.c +573 -0
  78. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_superblock.h +32 -0
  79. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_cwksp.h +236 -95
  80. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.c +321 -143
  81. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.h +2 -2
  82. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.c +328 -137
  83. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.h +2 -2
  84. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.c +2104 -0
  85. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.h +125 -0
  86. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.c +336 -209
  87. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.h +15 -3
  88. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm_geartab.h +106 -0
  89. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.c +439 -239
  90. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.h +1 -1
  91. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstdmt_compress.c +205 -462
  92. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.h +113 -0
  93. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress.c +1889 -0
  94. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress_amd64.S +585 -0
  95. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.c +20 -16
  96. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.h +3 -3
  97. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress.c +691 -230
  98. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/zstd_decompress_block.c +2072 -0
  99. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.h +16 -7
  100. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_internal.h +71 -10
  101. data/ext/zstdlib_c/zstd-1.5.2/lib/zdict.h +452 -0
  102. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/zstd.h +760 -234
  103. data/ext/{zstdlib/zstd-1.4.4/lib/common → zstdlib_c/zstd-1.5.2/lib}/zstd_errors.h +3 -1
  104. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzclose.c +0 -0
  105. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzcompatibility.h +1 -1
  106. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzguts.h +0 -0
  107. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzlib.c +0 -0
  108. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzread.c +0 -0
  109. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzwrite.c +0 -0
  110. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.c +133 -44
  111. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  112. data/ext/zstdlib_c/zstd.mk +15 -0
  113. data/lib/2.4/zstdlib_c.so +0 -0
  114. data/lib/2.5/zstdlib_c.so +0 -0
  115. data/lib/2.6/zstdlib_c.so +0 -0
  116. data/lib/2.7/zstdlib_c.so +0 -0
  117. data/lib/3.0/zstdlib_c.so +0 -0
  118. data/lib/3.1/zstdlib_c.so +0 -0
  119. data/lib/zstdlib.rb +2 -2
  120. metadata +125 -114
  121. data/ext/zstdlib/zstd-1.4.4/lib/common/compiler.h +0 -159
  122. data/ext/zstdlib/zstd-1.4.4/lib/common/debug.c +0 -44
  123. data/ext/zstdlib/zstd-1.4.4/lib/common/entropy_common.c +0 -236
  124. data/ext/zstdlib/zstd-1.4.4/lib/common/error_private.h +0 -76
  125. data/ext/zstdlib/zstd-1.4.4/lib/common/xxhash.c +0 -882
  126. data/ext/zstdlib/zstd-1.4.4/lib/common/xxhash.h +0 -305
  127. data/ext/zstdlib/zstd-1.4.4/lib/common/zstd_internal.h +0 -350
  128. data/ext/zstdlib/zstd-1.4.4/lib/compress/huf_compress.c +0 -798
  129. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_compress.c +0 -4103
  130. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_lazy.c +0 -1115
  131. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_lazy.h +0 -67
  132. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstdmt_compress.h +0 -192
  133. data/ext/zstdlib/zstd-1.4.4/lib/decompress/huf_decompress.c +0 -1234
  134. data/ext/zstdlib/zstd-1.4.4/lib/decompress/zstd_decompress_block.c +0 -1323
  135. data/ext/zstdlib/zstd.mk +0 -14
  136. data/lib/2.2/zstdlib.so +0 -0
  137. data/lib/2.3/zstdlib.so +0 -0
  138. data/lib/2.4/zstdlib.so +0 -0
  139. data/lib/2.5/zstdlib.so +0 -0
  140. data/lib/2.6/zstdlib.so +0 -0
  141. data/lib/2.7/zstdlib.so +0 -0
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -15,10 +15,10 @@
15
15
 
16
16
  size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
17
17
  {
18
- BYTE* const ostart = (BYTE* const)dst;
18
+ BYTE* const ostart = (BYTE*)dst;
19
19
  U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
20
20
 
21
- RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall);
21
+ RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
22
22
 
23
23
  switch(flSize)
24
24
  {
@@ -35,13 +35,14 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
35
35
  assert(0);
36
36
  }
37
37
 
38
- memcpy(ostart + flSize, src, srcSize);
38
+ ZSTD_memcpy(ostart + flSize, src, srcSize);
39
+ DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
39
40
  return srcSize + flSize;
40
41
  }
41
42
 
42
43
  size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
43
44
  {
44
- BYTE* const ostart = (BYTE* const)dst;
45
+ BYTE* const ostart = (BYTE*)dst;
45
46
  U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
46
47
 
47
48
  (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
@@ -62,6 +63,7 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
62
63
  }
63
64
 
64
65
  ostart[flSize] = *(const BYTE*)src;
66
+ DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1);
65
67
  return flSize+1;
66
68
  }
67
69
 
@@ -71,7 +73,8 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
71
73
  void* dst, size_t dstCapacity,
72
74
  const void* src, size_t srcSize,
73
75
  void* entropyWorkspace, size_t entropyWorkspaceSize,
74
- const int bmi2)
76
+ const int bmi2,
77
+ unsigned suspectUncompressible)
75
78
  {
76
79
  size_t const minGain = ZSTD_minGain(srcSize, strategy);
77
80
  size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
@@ -80,11 +83,11 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
80
83
  symbolEncodingType_e hType = set_compressed;
81
84
  size_t cLitSize;
82
85
 
83
- DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i)",
84
- disableLiteralCompression);
86
+ DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)",
87
+ disableLiteralCompression, (U32)srcSize);
85
88
 
86
89
  /* Prepare nextEntropy assuming reusing the existing table */
87
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
90
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
88
91
 
89
92
  if (disableLiteralCompression)
90
93
  return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
@@ -102,24 +105,25 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
102
105
  cLitSize = singleStream ?
103
106
  HUF_compress1X_repeat(
104
107
  ostart+lhSize, dstCapacity-lhSize, src, srcSize,
105
- 255, 11, entropyWorkspace, entropyWorkspaceSize,
106
- (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) :
108
+ HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
109
+ (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible) :
107
110
  HUF_compress4X_repeat(
108
111
  ostart+lhSize, dstCapacity-lhSize, src, srcSize,
109
- 255, 11, entropyWorkspace, entropyWorkspaceSize,
110
- (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2);
112
+ HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
113
+ (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible);
111
114
  if (repeat != HUF_repeat_none) {
112
115
  /* reused the existing table */
116
+ DEBUGLOG(5, "Reusing previous huffman table");
113
117
  hType = set_repeat;
114
118
  }
115
119
  }
116
120
 
117
- if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) {
118
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
121
+ if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
122
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
119
123
  return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
120
124
  }
121
125
  if (cLitSize==1) {
122
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
126
+ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
123
127
  return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
124
128
  }
125
129
 
@@ -150,5 +154,6 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
150
154
  default: /* not possible : lhSize is {3,4,5} */
151
155
  assert(0);
152
156
  }
157
+ DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize));
153
158
  return lhSize+cLitSize;
154
159
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -18,12 +18,14 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
18
18
 
19
19
  size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
20
20
 
21
+ /* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
21
22
  size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
22
23
  ZSTD_hufCTables_t* nextHuf,
23
24
  ZSTD_strategy strategy, int disableLiteralCompression,
24
25
  void* dst, size_t dstCapacity,
25
26
  const void* src, size_t srcSize,
26
27
  void* entropyWorkspace, size_t entropyWorkspaceSize,
27
- const int bmi2);
28
+ const int bmi2,
29
+ unsigned suspectUncompressible);
28
30
 
29
31
  #endif /* ZSTD_COMPRESS_LITERALS_H */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -50,6 +50,19 @@ static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) {
50
50
  return maxSymbolValue;
51
51
  }
52
52
 
53
+ /**
54
+ * Returns true if we should use ncount=-1 else we should
55
+ * use ncount=1 for low probability symbols instead.
56
+ */
57
+ static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
58
+ {
59
+ /* Heuristic: This should cover most blocks <= 16K and
60
+ * start to fade out after 16K to about 32K depending on
61
+ * comprssibility.
62
+ */
63
+ return nbSeq >= 2048;
64
+ }
65
+
53
66
  /**
54
67
  * Returns the cost in bytes of encoding the normalized count header.
55
68
  * Returns an error if any of the helper functions return an error.
@@ -60,7 +73,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
60
73
  BYTE wksp[FSE_NCOUNTBOUND];
61
74
  S16 norm[MaxSeq + 1];
62
75
  const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
63
- FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max));
76
+ FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), "");
64
77
  return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
65
78
  }
66
79
 
@@ -72,6 +85,8 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
72
85
  {
73
86
  unsigned cost = 0;
74
87
  unsigned s;
88
+
89
+ assert(total > 0);
75
90
  for (s = 0; s <= max; ++s) {
76
91
  unsigned norm = (unsigned)((256 * count[s]) / total);
77
92
  if (count[s] != 0 && norm == 0)
@@ -86,7 +101,7 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
86
101
  * Returns the cost in bits of encoding the distribution in count using ctable.
87
102
  * Returns an error if ctable cannot represent all the symbols in count.
88
103
  */
89
- static size_t ZSTD_fseBitCost(
104
+ size_t ZSTD_fseBitCost(
90
105
  FSE_CTable const* ctable,
91
106
  unsigned const* count,
92
107
  unsigned const max)
@@ -96,18 +111,22 @@ static size_t ZSTD_fseBitCost(
96
111
  unsigned s;
97
112
  FSE_CState_t cstate;
98
113
  FSE_initCState(&cstate, ctable);
99
- RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC,
100
- "Repeat FSE_CTable has maxSymbolValue %u < %u",
114
+ if (ZSTD_getFSEMaxSymbolValue(ctable) < max) {
115
+ DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u",
101
116
  ZSTD_getFSEMaxSymbolValue(ctable), max);
117
+ return ERROR(GENERIC);
118
+ }
102
119
  for (s = 0; s <= max; ++s) {
103
120
  unsigned const tableLog = cstate.stateLog;
104
121
  unsigned const badCost = (tableLog + 1) << kAccuracyLog;
105
122
  unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog);
106
123
  if (count[s] == 0)
107
124
  continue;
108
- RETURN_ERROR_IF(bitCost >= badCost, GENERIC,
109
- "Repeat FSE_CTable has Prob[%u] == 0", s);
110
- cost += count[s] * bitCost;
125
+ if (bitCost >= badCost) {
126
+ DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s);
127
+ return ERROR(GENERIC);
128
+ }
129
+ cost += (size_t)count[s] * bitCost;
111
130
  }
112
131
  return cost >> kAccuracyLog;
113
132
  }
@@ -117,15 +136,15 @@ static size_t ZSTD_fseBitCost(
117
136
  * table described by norm. The max symbol support by norm is assumed >= max.
118
137
  * norm must be valid for every symbol with non-zero probability in count.
119
138
  */
120
- static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
121
- unsigned const* count, unsigned const max)
139
+ size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
140
+ unsigned const* count, unsigned const max)
122
141
  {
123
142
  unsigned const shift = 8 - accuracyLog;
124
143
  size_t cost = 0;
125
144
  unsigned s;
126
145
  assert(accuracyLog <= 8);
127
146
  for (s = 0; s <= max; ++s) {
128
- unsigned const normAcc = norm[s] != -1 ? norm[s] : 1;
147
+ unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1;
129
148
  unsigned const norm256 = normAcc << shift;
130
149
  assert(norm256 > 0);
131
150
  assert(norm256 < 256);
@@ -215,6 +234,11 @@ ZSTD_selectEncodingType(
215
234
  return set_compressed;
216
235
  }
217
236
 
237
+ typedef struct {
238
+ S16 norm[MaxSeq + 1];
239
+ U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)];
240
+ } ZSTD_BuildCTableWksp;
241
+
218
242
  size_t
219
243
  ZSTD_buildCTable(void* dst, size_t dstCapacity,
220
244
  FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
@@ -230,18 +254,18 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
230
254
 
231
255
  switch (type) {
232
256
  case set_rle:
233
- FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max));
234
- RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall);
257
+ FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), "");
258
+ RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space");
235
259
  *op = codeTable[0];
236
260
  return 1;
237
261
  case set_repeat:
238
- memcpy(nextCTable, prevCTable, prevCTableSize);
262
+ ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize);
239
263
  return 0;
240
264
  case set_basic:
241
- FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */
265
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */
242
266
  return 0;
243
267
  case set_compressed: {
244
- S16 norm[MaxSeq + 1];
268
+ ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace;
245
269
  size_t nbSeq_1 = nbSeq;
246
270
  const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
247
271
  if (count[codeTable[nbSeq-1]] > 1) {
@@ -249,14 +273,17 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
249
273
  nbSeq_1--;
250
274
  }
251
275
  assert(nbSeq_1 > 1);
252
- FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
253
- { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
254
- FORWARD_IF_ERROR(NCountSize);
255
- FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize));
276
+ assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp));
277
+ (void)entropyWorkspaceSize;
278
+ FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed");
279
+ assert(oend >= op);
280
+ { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */
281
+ FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed");
282
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed");
256
283
  return NCountSize;
257
284
  }
258
285
  }
259
- default: assert(0); RETURN_ERROR(GENERIC);
286
+ default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach");
260
287
  }
261
288
  }
262
289
 
@@ -286,19 +313,19 @@ ZSTD_encodeSequences_body(
286
313
  FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]);
287
314
  BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]);
288
315
  if (MEM_32bits()) BIT_flushBits(&blockStream);
289
- BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]);
316
+ BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]);
290
317
  if (MEM_32bits()) BIT_flushBits(&blockStream);
291
318
  if (longOffsets) {
292
319
  U32 const ofBits = ofCodeTable[nbSeq-1];
293
- int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
320
+ unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
294
321
  if (extraBits) {
295
- BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits);
322
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits);
296
323
  BIT_flushBits(&blockStream);
297
324
  }
298
- BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits,
325
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits,
299
326
  ofBits - extraBits);
300
327
  } else {
301
- BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
328
+ BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]);
302
329
  }
303
330
  BIT_flushBits(&blockStream);
304
331
 
@@ -312,8 +339,8 @@ ZSTD_encodeSequences_body(
312
339
  U32 const mlBits = ML_bits[mlCode];
313
340
  DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u",
314
341
  (unsigned)sequences[n].litLength,
315
- (unsigned)sequences[n].matchLength + MINMATCH,
316
- (unsigned)sequences[n].offset);
342
+ (unsigned)sequences[n].mlBase + MINMATCH,
343
+ (unsigned)sequences[n].offBase);
317
344
  /* 32b*/ /* 64b*/
318
345
  /* (7)*/ /* (7)*/
319
346
  FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
@@ -324,18 +351,18 @@ ZSTD_encodeSequences_body(
324
351
  BIT_flushBits(&blockStream); /* (7)*/
325
352
  BIT_addBits(&blockStream, sequences[n].litLength, llBits);
326
353
  if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
327
- BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
354
+ BIT_addBits(&blockStream, sequences[n].mlBase, mlBits);
328
355
  if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream);
329
356
  if (longOffsets) {
330
- int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
357
+ unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
331
358
  if (extraBits) {
332
- BIT_addBits(&blockStream, sequences[n].offset, extraBits);
359
+ BIT_addBits(&blockStream, sequences[n].offBase, extraBits);
333
360
  BIT_flushBits(&blockStream); /* (7)*/
334
361
  }
335
- BIT_addBits(&blockStream, sequences[n].offset >> extraBits,
362
+ BIT_addBits(&blockStream, sequences[n].offBase >> extraBits,
336
363
  ofBits - extraBits); /* 31 */
337
364
  } else {
338
- BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */
365
+ BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */
339
366
  }
340
367
  BIT_flushBits(&blockStream); /* (7)*/
341
368
  DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr));
@@ -372,7 +399,7 @@ ZSTD_encodeSequences_default(
372
399
 
373
400
  #if DYNAMIC_BMI2
374
401
 
375
- static TARGET_ATTRIBUTE("bmi2") size_t
402
+ static BMI2_TARGET_ATTRIBUTE size_t
376
403
  ZSTD_encodeSequences_bmi2(
377
404
  void* dst, size_t dstCapacity,
378
405
  FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -11,8 +11,8 @@
11
11
  #ifndef ZSTD_COMPRESS_SEQUENCES_H
12
12
  #define ZSTD_COMPRESS_SEQUENCES_H
13
13
 
14
- #include "fse.h" /* FSE_repeat, FSE_CTable */
15
- #include "zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */
14
+ #include "../common/fse.h" /* FSE_repeat, FSE_CTable */
15
+ #include "../common/zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */
16
16
 
17
17
  typedef enum {
18
18
  ZSTD_defaultDisallowed = 0,
@@ -44,4 +44,11 @@ size_t ZSTD_encodeSequences(
44
44
  FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
45
45
  seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
46
46
 
47
+ size_t ZSTD_fseBitCost(
48
+ FSE_CTable const* ctable,
49
+ unsigned const* count,
50
+ unsigned const max);
51
+
52
+ size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
53
+ unsigned const* count, unsigned const max);
47
54
  #endif /* ZSTD_COMPRESS_SEQUENCES_H */