extzstd 0.1 → 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (134) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +39 -0
  3. data/README.md +38 -56
  4. data/contrib/zstd/CHANGELOG +613 -0
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/CONTRIBUTING.md +406 -0
  7. data/contrib/zstd/COPYING +339 -0
  8. data/contrib/zstd/Makefile +420 -0
  9. data/contrib/zstd/README.md +179 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +292 -0
  12. data/contrib/zstd/lib/BUCK +234 -0
  13. data/contrib/zstd/lib/Makefile +451 -0
  14. data/contrib/zstd/lib/README.md +207 -0
  15. data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
  16. data/contrib/zstd/lib/common/compiler.h +288 -0
  17. data/contrib/zstd/lib/common/cpu.h +213 -0
  18. data/contrib/zstd/lib/common/debug.c +24 -0
  19. data/contrib/zstd/lib/common/debug.h +107 -0
  20. data/contrib/zstd/lib/common/entropy_common.c +362 -0
  21. data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
  22. data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
  23. data/contrib/zstd/{common → lib/common}/fse.h +173 -92
  24. data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
  25. data/contrib/zstd/lib/common/huf.h +361 -0
  26. data/contrib/zstd/{common → lib/common}/mem.h +115 -59
  27. data/contrib/zstd/lib/common/pool.c +350 -0
  28. data/contrib/zstd/lib/common/pool.h +84 -0
  29. data/contrib/zstd/lib/common/threading.c +122 -0
  30. data/contrib/zstd/lib/common/threading.h +155 -0
  31. data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
  32. data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
  33. data/contrib/zstd/lib/common/zstd_common.c +83 -0
  34. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  35. data/contrib/zstd/lib/common/zstd_errors.h +95 -0
  36. data/contrib/zstd/lib/common/zstd_internal.h +478 -0
  37. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
  38. data/contrib/zstd/lib/compress/hist.c +181 -0
  39. data/contrib/zstd/lib/compress/hist.h +75 -0
  40. data/contrib/zstd/lib/compress/huf_compress.c +913 -0
  41. data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
  42. data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
  43. data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
  44. data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
  45. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
  46. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
  47. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
  48. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  49. data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
  50. data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
  51. data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
  52. data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
  53. data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
  54. data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
  55. data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
  56. data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
  57. data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
  58. data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
  59. data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
  60. data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
  61. data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
  62. data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
  63. data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
  64. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  65. data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
  66. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
  67. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
  68. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
  69. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
  70. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  71. data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
  72. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
  73. data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
  74. data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
  75. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
  76. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  77. data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
  78. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
  79. data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
  80. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
  81. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
  82. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
  83. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
  84. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
  85. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
  86. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
  87. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
  88. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
  89. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
  90. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
  91. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
  92. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
  93. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
  94. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
  95. data/contrib/zstd/lib/libzstd.pc.in +15 -0
  96. data/contrib/zstd/lib/zstd.h +2391 -0
  97. data/ext/depend +2 -0
  98. data/ext/extconf.rb +15 -6
  99. data/ext/extzstd.c +76 -145
  100. data/ext/extzstd.h +80 -31
  101. data/ext/extzstd_stream.c +417 -142
  102. data/ext/libzstd_conf.h +8 -0
  103. data/ext/zstd_common.c +10 -7
  104. data/ext/zstd_compress.c +14 -5
  105. data/ext/zstd_decompress.c +5 -4
  106. data/ext/zstd_dictbuilder.c +9 -4
  107. data/ext/zstd_dictbuilder_fastcover.c +3 -0
  108. data/ext/zstd_legacy_v01.c +3 -1
  109. data/ext/zstd_legacy_v02.c +3 -1
  110. data/ext/zstd_legacy_v03.c +3 -1
  111. data/ext/zstd_legacy_v04.c +3 -1
  112. data/ext/zstd_legacy_v05.c +3 -1
  113. data/ext/zstd_legacy_v06.c +3 -1
  114. data/ext/zstd_legacy_v07.c +3 -1
  115. data/gemstub.rb +10 -24
  116. data/lib/extzstd.rb +64 -179
  117. data/lib/extzstd/version.rb +6 -1
  118. data/test/test_basic.rb +9 -6
  119. metadata +113 -57
  120. data/HISTORY.ja +0 -5
  121. data/contrib/zstd/common/entropy_common.c +0 -225
  122. data/contrib/zstd/common/huf.h +0 -228
  123. data/contrib/zstd/common/zstd_common.c +0 -83
  124. data/contrib/zstd/common/zstd_errors.h +0 -60
  125. data/contrib/zstd/common/zstd_internal.h +0 -267
  126. data/contrib/zstd/compress/huf_compress.c +0 -533
  127. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  128. data/contrib/zstd/compress/zstd_compress.c +0 -3264
  129. data/contrib/zstd/compress/zstd_opt.h +0 -900
  130. data/contrib/zstd/decompress/huf_decompress.c +0 -883
  131. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  132. data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
  133. data/contrib/zstd/dictBuilder/zdict.h +0 -111
  134. data/contrib/zstd/zstd.h +0 -640
@@ -1,533 +0,0 @@
1
- /* ******************************************************************
2
- Huffman encoder, part of New Generation Entropy library
3
- Copyright (C) 2013-2016, Yann Collet.
4
-
5
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
-
7
- Redistribution and use in source and binary forms, with or without
8
- modification, are permitted provided that the following conditions are
9
- met:
10
-
11
- * Redistributions of source code must retain the above copyright
12
- notice, this list of conditions and the following disclaimer.
13
- * Redistributions in binary form must reproduce the above
14
- copyright notice, this list of conditions and the following disclaimer
15
- in the documentation and/or other materials provided with the
16
- distribution.
17
-
18
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
-
30
- You can contact the author at :
31
- - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
32
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
33
- ****************************************************************** */
34
-
35
- /* **************************************************************
36
- * Compiler specifics
37
- ****************************************************************/
38
- #ifdef _MSC_VER /* Visual Studio */
39
- # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
40
- #endif
41
-
42
-
43
- /* **************************************************************
44
- * Includes
45
- ****************************************************************/
46
- #include <string.h> /* memcpy, memset */
47
- #include <stdio.h> /* printf (debug) */
48
- #include "bitstream.h"
49
- #define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
50
- #include "fse.h" /* header compression */
51
- #define HUF_STATIC_LINKING_ONLY
52
- #include "huf.h"
53
-
54
-
55
- /* **************************************************************
56
- * Error Management
57
- ****************************************************************/
58
- #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
59
-
60
-
61
- /* **************************************************************
62
- * Utils
63
- ****************************************************************/
64
- unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
65
- {
66
- return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
67
- }
68
-
69
-
70
- /* *******************************************************
71
- * HUF : Huffman block compression
72
- *********************************************************/
73
- struct HUF_CElt_s {
74
- U16 val;
75
- BYTE nbBits;
76
- }; /* typedef'd to HUF_CElt within "huf.h" */
77
-
78
- typedef struct nodeElt_s {
79
- U32 count;
80
- U16 parent;
81
- BYTE byte;
82
- BYTE nbBits;
83
- } nodeElt;
84
-
85
- /*! HUF_writeCTable() :
86
- `CTable` : huffman tree to save, using huf representation.
87
- @return : size of saved CTable */
88
- size_t HUF_writeCTable (void* dst, size_t maxDstSize,
89
- const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
90
- {
91
- BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
92
- BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
93
- BYTE* op = (BYTE*)dst;
94
- U32 n;
95
-
96
- /* check conditions */
97
- if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
98
-
99
- /* convert to weight */
100
- bitsToWeight[0] = 0;
101
- for (n=1; n<huffLog+1; n++)
102
- bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
103
- for (n=0; n<maxSymbolValue; n++)
104
- huffWeight[n] = bitsToWeight[CTable[n].nbBits];
105
-
106
- { size_t const size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue);
107
- if (FSE_isError(size)) return size;
108
- if ((size>1) & (size < maxSymbolValue/2)) { /* FSE compressed */
109
- op[0] = (BYTE)size;
110
- return size+1;
111
- }
112
- }
113
-
114
- /* raw values */
115
- if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */
116
- if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
117
- op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
118
- huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
119
- for (n=0; n<maxSymbolValue; n+=2)
120
- op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
121
- return ((maxSymbolValue+1)/2) + 1;
122
-
123
- }
124
-
125
-
126
- size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
127
- {
128
- BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
129
- U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
130
- U32 tableLog = 0;
131
- size_t readSize;
132
- U32 nbSymbols = 0;
133
- /*memset(huffWeight, 0, sizeof(huffWeight));*/ /* is not necessary, even though some analyzer complain ... */
134
-
135
- /* get symbol weights */
136
- readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
137
- if (HUF_isError(readSize)) return readSize;
138
-
139
- /* check result */
140
- if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
141
- if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
142
-
143
- /* Prepare base value per rank */
144
- { U32 n, nextRankStart = 0;
145
- for (n=1; n<=tableLog; n++) {
146
- U32 current = nextRankStart;
147
- nextRankStart += (rankVal[n] << (n-1));
148
- rankVal[n] = current;
149
- } }
150
-
151
- /* fill nbBits */
152
- { U32 n; for (n=0; n<nbSymbols; n++) {
153
- const U32 w = huffWeight[n];
154
- CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
155
- } }
156
-
157
- /* fill val */
158
- { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
159
- U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
160
- { U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
161
- /* determine stating value per rank */
162
- valPerRank[tableLog+1] = 0; /* for w==0 */
163
- { U16 min = 0;
164
- U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
165
- valPerRank[n] = min; /* get starting value within each rank */
166
- min += nbPerRank[n];
167
- min >>= 1;
168
- } }
169
- /* assign value within rank, symbol order */
170
- { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; }
171
- }
172
-
173
- return readSize;
174
- }
175
-
176
-
177
- static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
178
- {
179
- const U32 largestBits = huffNode[lastNonNull].nbBits;
180
- if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */
181
-
182
- /* there are several too large elements (at least >= 2) */
183
- { int totalCost = 0;
184
- const U32 baseCost = 1 << (largestBits - maxNbBits);
185
- U32 n = lastNonNull;
186
-
187
- while (huffNode[n].nbBits > maxNbBits) {
188
- totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
189
- huffNode[n].nbBits = (BYTE)maxNbBits;
190
- n --;
191
- } /* n stops at huffNode[n].nbBits <= maxNbBits */
192
- while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */
193
-
194
- /* renorm totalCost */
195
- totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
196
-
197
- /* repay normalized cost */
198
- { U32 const noSymbol = 0xF0F0F0F0;
199
- U32 rankLast[HUF_TABLELOG_MAX+2];
200
- int pos;
201
-
202
- /* Get pos of last (smallest) symbol per rank */
203
- memset(rankLast, 0xF0, sizeof(rankLast));
204
- { U32 currentNbBits = maxNbBits;
205
- for (pos=n ; pos >= 0; pos--) {
206
- if (huffNode[pos].nbBits >= currentNbBits) continue;
207
- currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */
208
- rankLast[maxNbBits-currentNbBits] = pos;
209
- } }
210
-
211
- while (totalCost > 0) {
212
- U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
213
- for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
214
- U32 highPos = rankLast[nBitsToDecrease];
215
- U32 lowPos = rankLast[nBitsToDecrease-1];
216
- if (highPos == noSymbol) continue;
217
- if (lowPos == noSymbol) break;
218
- { U32 const highTotal = huffNode[highPos].count;
219
- U32 const lowTotal = 2 * huffNode[lowPos].count;
220
- if (highTotal <= lowTotal) break;
221
- } }
222
- /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
223
- while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
224
- nBitsToDecrease ++;
225
- totalCost -= 1 << (nBitsToDecrease-1);
226
- if (rankLast[nBitsToDecrease-1] == noSymbol)
227
- rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
228
- huffNode[rankLast[nBitsToDecrease]].nbBits ++;
229
- if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
230
- rankLast[nBitsToDecrease] = noSymbol;
231
- else {
232
- rankLast[nBitsToDecrease]--;
233
- if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
234
- rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
235
- } } /* while (totalCost > 0) */
236
-
237
- while (totalCost < 0) { /* Sometimes, cost correction overshoot */
238
- if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
239
- while (huffNode[n].nbBits == maxNbBits) n--;
240
- huffNode[n+1].nbBits--;
241
- rankLast[1] = n+1;
242
- totalCost++;
243
- continue;
244
- }
245
- huffNode[ rankLast[1] + 1 ].nbBits--;
246
- rankLast[1]++;
247
- totalCost ++;
248
- } } } /* there are several too large elements (at least >= 2) */
249
-
250
- return maxNbBits;
251
- }
252
-
253
-
254
- typedef struct {
255
- U32 base;
256
- U32 current;
257
- } rankPos;
258
-
259
- static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
260
- {
261
- rankPos rank[32];
262
- U32 n;
263
-
264
- memset(rank, 0, sizeof(rank));
265
- for (n=0; n<=maxSymbolValue; n++) {
266
- U32 r = BIT_highbit32(count[n] + 1);
267
- rank[r].base ++;
268
- }
269
- for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
270
- for (n=0; n<32; n++) rank[n].current = rank[n].base;
271
- for (n=0; n<=maxSymbolValue; n++) {
272
- U32 const c = count[n];
273
- U32 const r = BIT_highbit32(c+1) + 1;
274
- U32 pos = rank[r].current++;
275
- while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--;
276
- huffNode[pos].count = c;
277
- huffNode[pos].byte = (BYTE)n;
278
- }
279
- }
280
-
281
-
282
- #define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
283
- size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
284
- {
285
- nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1];
286
- nodeElt* huffNode = huffNode0 + 1;
287
- U32 n, nonNullRank;
288
- int lowS, lowN;
289
- U16 nodeNb = STARTNODE;
290
- U32 nodeRoot;
291
-
292
- /* safety checks */
293
- if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
294
- if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
295
- memset(huffNode0, 0, sizeof(huffNode0));
296
-
297
- /* sort, decreasing order */
298
- HUF_sort(huffNode, count, maxSymbolValue);
299
-
300
- /* init for parents */
301
- nonNullRank = maxSymbolValue;
302
- while(huffNode[nonNullRank].count == 0) nonNullRank--;
303
- lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
304
- huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
305
- huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
306
- nodeNb++; lowS-=2;
307
- for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
308
- huffNode0[0].count = (U32)(1U<<31);
309
-
310
- /* create parents */
311
- while (nodeNb <= nodeRoot) {
312
- U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
313
- U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
314
- huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
315
- huffNode[n1].parent = huffNode[n2].parent = nodeNb;
316
- nodeNb++;
317
- }
318
-
319
- /* distribute weights (unlimited tree height) */
320
- huffNode[nodeRoot].nbBits = 0;
321
- for (n=nodeRoot-1; n>=STARTNODE; n--)
322
- huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
323
- for (n=0; n<=nonNullRank; n++)
324
- huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
325
-
326
- /* enforce maxTableLog */
327
- maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
328
-
329
- /* fill result into tree (val, nbBits) */
330
- { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
331
- U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
332
- if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
333
- for (n=0; n<=nonNullRank; n++)
334
- nbPerRank[huffNode[n].nbBits]++;
335
- /* determine stating value per rank */
336
- { U16 min = 0;
337
- for (n=maxNbBits; n>0; n--) {
338
- valPerRank[n] = min; /* get starting value within each rank */
339
- min += nbPerRank[n];
340
- min >>= 1;
341
- } }
342
- for (n=0; n<=maxSymbolValue; n++)
343
- tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
344
- for (n=0; n<=maxSymbolValue; n++)
345
- tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
346
- }
347
-
348
- return maxNbBits;
349
- }
350
-
351
- static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
352
- {
353
- BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
354
- }
355
-
356
- size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
357
-
358
- #define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
359
-
360
- #define HUF_FLUSHBITS_1(stream) \
361
- if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream)
362
-
363
- #define HUF_FLUSHBITS_2(stream) \
364
- if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream)
365
-
366
- size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
367
- {
368
- const BYTE* ip = (const BYTE*) src;
369
- BYTE* const ostart = (BYTE*)dst;
370
- BYTE* const oend = ostart + dstSize;
371
- BYTE* op = ostart;
372
- size_t n;
373
- const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
374
- BIT_CStream_t bitC;
375
-
376
- /* init */
377
- if (dstSize < 8) return 0; /* not enough space to compress */
378
- { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op);
379
- if (HUF_isError(errorCode)) return 0; }
380
-
381
- n = srcSize & ~3; /* join to mod 4 */
382
- switch (srcSize & 3)
383
- {
384
- case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
385
- HUF_FLUSHBITS_2(&bitC);
386
- case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
387
- HUF_FLUSHBITS_1(&bitC);
388
- case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
389
- HUF_FLUSHBITS(&bitC);
390
- case 0 :
391
- default: ;
392
- }
393
-
394
- for (; n>0; n-=4) { /* note : n&3==0 at this stage */
395
- HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
396
- HUF_FLUSHBITS_1(&bitC);
397
- HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
398
- HUF_FLUSHBITS_2(&bitC);
399
- HUF_encodeSymbol(&bitC, ip[n- 3], CTable);
400
- HUF_FLUSHBITS_1(&bitC);
401
- HUF_encodeSymbol(&bitC, ip[n- 4], CTable);
402
- HUF_FLUSHBITS(&bitC);
403
- }
404
-
405
- return BIT_closeCStream(&bitC);
406
- }
407
-
408
-
409
- size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
410
- {
411
- size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */
412
- const BYTE* ip = (const BYTE*) src;
413
- const BYTE* const iend = ip + srcSize;
414
- BYTE* const ostart = (BYTE*) dst;
415
- BYTE* const oend = ostart + dstSize;
416
- BYTE* op = ostart;
417
-
418
- if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
419
- if (srcSize < 12) return 0; /* no saving possible : too small input */
420
- op += 6; /* jumpTable */
421
-
422
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
423
- if (HUF_isError(cSize)) return cSize;
424
- if (cSize==0) return 0;
425
- MEM_writeLE16(ostart, (U16)cSize);
426
- op += cSize;
427
- }
428
-
429
- ip += segmentSize;
430
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
431
- if (HUF_isError(cSize)) return cSize;
432
- if (cSize==0) return 0;
433
- MEM_writeLE16(ostart+2, (U16)cSize);
434
- op += cSize;
435
- }
436
-
437
- ip += segmentSize;
438
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
439
- if (HUF_isError(cSize)) return cSize;
440
- if (cSize==0) return 0;
441
- MEM_writeLE16(ostart+4, (U16)cSize);
442
- op += cSize;
443
- }
444
-
445
- ip += segmentSize;
446
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
447
- if (HUF_isError(cSize)) return cSize;
448
- if (cSize==0) return 0;
449
- op += cSize;
450
- }
451
-
452
- return op-ostart;
453
- }
454
-
455
-
456
- static size_t HUF_compress_internal (
457
- void* dst, size_t dstSize,
458
- const void* src, size_t srcSize,
459
- unsigned maxSymbolValue, unsigned huffLog,
460
- unsigned singleStream)
461
- {
462
- BYTE* const ostart = (BYTE*)dst;
463
- BYTE* const oend = ostart + dstSize;
464
- BYTE* op = ostart;
465
-
466
- U32 count[HUF_SYMBOLVALUE_MAX+1];
467
- HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
468
-
469
- /* checks & inits */
470
- if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
471
- if (!dstSize) return 0; /* cannot fit within dst budget */
472
- if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
473
- if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
474
- if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
475
- if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
476
-
477
- /* Scan input and build symbol stats */
478
- { size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
479
- if (HUF_isError(largest)) return largest;
480
- if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
481
- if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
482
- }
483
-
484
- /* Build Huffman Tree */
485
- huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
486
- { size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
487
- if (HUF_isError(maxBits)) return maxBits;
488
- huffLog = (U32)maxBits;
489
- }
490
-
491
- /* Write table description header */
492
- { size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
493
- if (HUF_isError(hSize)) return hSize;
494
- if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */
495
- op += hSize;
496
- }
497
-
498
- /* Compress */
499
- { size_t const cSize = (singleStream) ?
500
- HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */
501
- HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
502
- if (HUF_isError(cSize)) return cSize;
503
- if (cSize==0) return 0; /* uncompressible */
504
- op += cSize;
505
- }
506
-
507
- /* check compressibility */
508
- if ((size_t)(op-ostart) >= srcSize-1)
509
- return 0;
510
-
511
- return op-ostart;
512
- }
513
-
514
-
515
- size_t HUF_compress1X (void* dst, size_t dstSize,
516
- const void* src, size_t srcSize,
517
- unsigned maxSymbolValue, unsigned huffLog)
518
- {
519
- return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1);
520
- }
521
-
522
- size_t HUF_compress2 (void* dst, size_t dstSize,
523
- const void* src, size_t srcSize,
524
- unsigned maxSymbolValue, unsigned huffLog)
525
- {
526
- return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
527
- }
528
-
529
-
530
- size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
531
- {
532
- return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);
533
- }