extzstd 0.1 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.ja +5 -0
- data/README.md +5 -5
- data/contrib/zstd/CONTRIBUTING.md +42 -0
- data/contrib/zstd/LICENSE-examples +11 -0
- data/contrib/zstd/Makefile +315 -0
- data/contrib/zstd/NEWS +261 -0
- data/contrib/zstd/PATENTS +33 -0
- data/contrib/zstd/README.md +121 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +178 -0
- data/contrib/zstd/circle.yml +75 -0
- data/contrib/zstd/lib/BUCK +186 -0
- data/contrib/zstd/lib/Makefile +163 -0
- data/contrib/zstd/lib/README.md +77 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +7 -4
- data/contrib/zstd/{common → lib/common}/entropy_common.c +19 -23
- data/contrib/zstd/{common → lib/common}/error_private.c +0 -0
- data/contrib/zstd/{common → lib/common}/error_private.h +0 -0
- data/contrib/zstd/{common → lib/common}/fse.h +94 -34
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +18 -19
- data/contrib/zstd/{common → lib/common}/huf.h +52 -20
- data/contrib/zstd/{common → lib/common}/mem.h +17 -13
- data/contrib/zstd/lib/common/pool.c +194 -0
- data/contrib/zstd/lib/common/pool.h +56 -0
- data/contrib/zstd/lib/common/threading.c +80 -0
- data/contrib/zstd/lib/common/threading.h +104 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +3 -1
- data/contrib/zstd/{common → lib/common}/xxhash.h +11 -15
- data/contrib/zstd/{common → lib/common}/zstd_common.c +1 -11
- data/contrib/zstd/{common → lib/common}/zstd_errors.h +16 -2
- data/contrib/zstd/{common → lib/common}/zstd_internal.h +17 -1
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +138 -91
- data/contrib/zstd/{compress → lib/compress}/huf_compress.c +218 -67
- data/contrib/zstd/{compress → lib/compress}/zstd_compress.c +231 -108
- data/contrib/zstd/{compress → lib/compress}/zstd_opt.h +44 -25
- data/contrib/zstd/lib/compress/zstdmt_compress.c +739 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +78 -0
- data/contrib/zstd/{decompress → lib/decompress}/huf_decompress.c +28 -23
- data/contrib/zstd/{decompress → lib/decompress}/zstd_decompress.c +814 -176
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +60 -39
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +145 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +74 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1029 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +0 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +68 -18
- data/contrib/zstd/lib/dictBuilder/zdict.h +201 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +122 -7
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +34 -3
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +45 -12
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +45 -12
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +56 -33
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +45 -18
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +7 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +43 -16
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +7 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +57 -23
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +8 -0
- data/contrib/zstd/lib/libzstd.pc.in +14 -0
- data/contrib/zstd/{zstd.h → lib/zstd.h} +206 -71
- data/ext/depend +2 -0
- data/ext/extconf.rb +4 -4
- data/ext/extzstd.c +1 -1
- data/ext/zstd_common.c +5 -5
- data/ext/zstd_compress.c +3 -3
- data/ext/zstd_decompress.c +2 -2
- data/ext/zstd_dictbuilder.c +2 -2
- data/ext/zstd_legacy_v01.c +1 -1
- data/ext/zstd_legacy_v02.c +1 -1
- data/ext/zstd_legacy_v03.c +1 -1
- data/ext/zstd_legacy_v04.c +1 -1
- data/ext/zstd_legacy_v05.c +1 -1
- data/ext/zstd_legacy_v06.c +1 -1
- data/ext/zstd_legacy_v07.c +1 -1
- data/gemstub.rb +9 -5
- data/lib/extzstd/version.rb +1 -1
- metadata +73 -51
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
|
@@ -56,6 +56,8 @@
|
|
|
56
56
|
* Error Management
|
|
57
57
|
****************************************************************/
|
|
58
58
|
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
|
59
|
+
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f
|
|
60
|
+
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
|
|
59
61
|
|
|
60
62
|
|
|
61
63
|
/* **************************************************************
|
|
@@ -70,31 +72,73 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
|
|
|
70
72
|
/* *******************************************************
|
|
71
73
|
* HUF : Huffman block compression
|
|
72
74
|
*********************************************************/
|
|
75
|
+
/* HUF_compressWeights() :
|
|
76
|
+
* Same as FSE_compress(), but dedicated to huff0's weights compression.
|
|
77
|
+
* The use case needs much less stack memory.
|
|
78
|
+
* Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
|
|
79
|
+
*/
|
|
80
|
+
#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
|
|
81
|
+
size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize)
|
|
82
|
+
{
|
|
83
|
+
BYTE* const ostart = (BYTE*) dst;
|
|
84
|
+
BYTE* op = ostart;
|
|
85
|
+
BYTE* const oend = ostart + dstSize;
|
|
86
|
+
|
|
87
|
+
U32 maxSymbolValue = HUF_TABLELOG_MAX;
|
|
88
|
+
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
|
|
89
|
+
|
|
90
|
+
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
|
|
91
|
+
BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
|
|
92
|
+
|
|
93
|
+
U32 count[HUF_TABLELOG_MAX+1];
|
|
94
|
+
S16 norm[HUF_TABLELOG_MAX+1];
|
|
95
|
+
|
|
96
|
+
/* init conditions */
|
|
97
|
+
if (wtSize <= 1) return 0; /* Not compressible */
|
|
98
|
+
|
|
99
|
+
/* Scan input and build symbol stats */
|
|
100
|
+
{ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) );
|
|
101
|
+
if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
|
|
102
|
+
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
|
|
106
|
+
CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) );
|
|
107
|
+
|
|
108
|
+
/* Write table description header */
|
|
109
|
+
{ CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
|
|
110
|
+
op += hSize;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/* Compress */
|
|
114
|
+
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) );
|
|
115
|
+
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) );
|
|
116
|
+
if (cSize == 0) return 0; /* not enough space for compressed data */
|
|
117
|
+
op += cSize;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return op-ostart;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
|
|
73
124
|
struct HUF_CElt_s {
|
|
74
125
|
U16 val;
|
|
75
126
|
BYTE nbBits;
|
|
76
127
|
}; /* typedef'd to HUF_CElt within "huf.h" */
|
|
77
128
|
|
|
78
|
-
typedef struct nodeElt_s {
|
|
79
|
-
U32 count;
|
|
80
|
-
U16 parent;
|
|
81
|
-
BYTE byte;
|
|
82
|
-
BYTE nbBits;
|
|
83
|
-
} nodeElt;
|
|
84
|
-
|
|
85
129
|
/*! HUF_writeCTable() :
|
|
86
|
-
`CTable` :
|
|
130
|
+
`CTable` : Huffman tree to save, using huf representation.
|
|
87
131
|
@return : size of saved CTable */
|
|
88
132
|
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
|
89
133
|
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
|
|
90
134
|
{
|
|
91
|
-
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
|
|
135
|
+
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
|
|
92
136
|
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
|
|
93
137
|
BYTE* op = (BYTE*)dst;
|
|
94
138
|
U32 n;
|
|
95
139
|
|
|
96
140
|
/* check conditions */
|
|
97
|
-
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(
|
|
141
|
+
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
|
|
98
142
|
|
|
99
143
|
/* convert to weight */
|
|
100
144
|
bitsToWeight[0] = 0;
|
|
@@ -103,38 +147,33 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
|
|
103
147
|
for (n=0; n<maxSymbolValue; n++)
|
|
104
148
|
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
|
|
105
149
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
if ((
|
|
109
|
-
op[0] = (BYTE)
|
|
110
|
-
return
|
|
111
|
-
|
|
112
|
-
}
|
|
150
|
+
/* attempt weights compression by FSE */
|
|
151
|
+
{ CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) );
|
|
152
|
+
if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
|
|
153
|
+
op[0] = (BYTE)hSize;
|
|
154
|
+
return hSize+1;
|
|
155
|
+
} }
|
|
113
156
|
|
|
114
|
-
/* raw values */
|
|
115
|
-
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */
|
|
157
|
+
/* write raw values as 4-bits (max : 15) */
|
|
158
|
+
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
|
|
116
159
|
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
|
|
117
160
|
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
|
|
118
|
-
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
|
|
161
|
+
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
|
|
119
162
|
for (n=0; n<maxSymbolValue; n+=2)
|
|
120
163
|
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
|
|
121
164
|
return ((maxSymbolValue+1)/2) + 1;
|
|
122
|
-
|
|
123
165
|
}
|
|
124
166
|
|
|
125
167
|
|
|
126
168
|
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
|
|
127
169
|
{
|
|
128
|
-
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
|
|
170
|
+
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
|
|
129
171
|
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
|
130
172
|
U32 tableLog = 0;
|
|
131
|
-
size_t readSize;
|
|
132
173
|
U32 nbSymbols = 0;
|
|
133
|
-
/*memset(huffWeight, 0, sizeof(huffWeight));*/ /* is not necessary, even though some analyzer complain ... */
|
|
134
174
|
|
|
135
175
|
/* get symbol weights */
|
|
136
|
-
readSize
|
|
137
|
-
if (HUF_isError(readSize)) return readSize;
|
|
176
|
+
CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize));
|
|
138
177
|
|
|
139
178
|
/* check result */
|
|
140
179
|
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
|
@@ -174,6 +213,13 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
|
|
|
174
213
|
}
|
|
175
214
|
|
|
176
215
|
|
|
216
|
+
typedef struct nodeElt_s {
|
|
217
|
+
U32 count;
|
|
218
|
+
U16 parent;
|
|
219
|
+
BYTE byte;
|
|
220
|
+
BYTE nbBits;
|
|
221
|
+
} nodeElt;
|
|
222
|
+
|
|
177
223
|
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
|
178
224
|
{
|
|
179
225
|
const U32 largestBits = huffNode[lastNonNull].nbBits;
|
|
@@ -279,20 +325,26 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
|
|
|
279
325
|
}
|
|
280
326
|
|
|
281
327
|
|
|
328
|
+
/** HUF_buildCTable_wksp() :
|
|
329
|
+
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
|
330
|
+
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
|
|
331
|
+
*/
|
|
282
332
|
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
|
|
283
|
-
|
|
333
|
+
typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1];
|
|
334
|
+
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
|
|
284
335
|
{
|
|
285
|
-
nodeElt huffNode0
|
|
286
|
-
nodeElt* huffNode = huffNode0
|
|
336
|
+
nodeElt* const huffNode0 = (nodeElt*)workSpace;
|
|
337
|
+
nodeElt* const huffNode = huffNode0+1;
|
|
287
338
|
U32 n, nonNullRank;
|
|
288
339
|
int lowS, lowN;
|
|
289
340
|
U16 nodeNb = STARTNODE;
|
|
290
341
|
U32 nodeRoot;
|
|
291
342
|
|
|
292
343
|
/* safety checks */
|
|
344
|
+
if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */
|
|
293
345
|
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
|
|
294
346
|
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
|
|
295
|
-
memset(huffNode0, 0, sizeof(
|
|
347
|
+
memset(huffNode0, 0, sizeof(huffNodeTable));
|
|
296
348
|
|
|
297
349
|
/* sort, decreasing order */
|
|
298
350
|
HUF_sort(huffNode, count, maxSymbolValue);
|
|
@@ -305,7 +357,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
|
|
|
305
357
|
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
|
|
306
358
|
nodeNb++; lowS-=2;
|
|
307
359
|
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
|
|
308
|
-
huffNode0[0].count = (U32)(1U<<31);
|
|
360
|
+
huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */
|
|
309
361
|
|
|
310
362
|
/* create parents */
|
|
311
363
|
while (nodeNb <= nodeRoot) {
|
|
@@ -348,6 +400,34 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
|
|
|
348
400
|
return maxNbBits;
|
|
349
401
|
}
|
|
350
402
|
|
|
403
|
+
/** HUF_buildCTable() :
|
|
404
|
+
* Note : count is used before tree is written, so they can safely overlap
|
|
405
|
+
*/
|
|
406
|
+
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
|
|
407
|
+
{
|
|
408
|
+
huffNodeTable nodeTable;
|
|
409
|
+
return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
|
|
413
|
+
{
|
|
414
|
+
size_t nbBits = 0;
|
|
415
|
+
int s;
|
|
416
|
+
for (s = 0; s <= (int)maxSymbolValue; ++s) {
|
|
417
|
+
nbBits += CTable[s].nbBits * count[s];
|
|
418
|
+
}
|
|
419
|
+
return nbBits >> 3;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
|
|
423
|
+
int bad = 0;
|
|
424
|
+
int s;
|
|
425
|
+
for (s = 0; s <= (int)maxSymbolValue; ++s) {
|
|
426
|
+
bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
|
|
427
|
+
}
|
|
428
|
+
return !bad;
|
|
429
|
+
}
|
|
430
|
+
|
|
351
431
|
static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
|
|
352
432
|
{
|
|
353
433
|
BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
|
|
@@ -375,8 +455,8 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
|
|
|
375
455
|
|
|
376
456
|
/* init */
|
|
377
457
|
if (dstSize < 8) return 0; /* not enough space to compress */
|
|
378
|
-
{ size_t const
|
|
379
|
-
if (HUF_isError(
|
|
458
|
+
{ size_t const initErr = BIT_initCStream(&bitC, op, oend-op);
|
|
459
|
+
if (HUF_isError(initErr)) return 0; }
|
|
380
460
|
|
|
381
461
|
n = srcSize & ~3; /* join to mod 4 */
|
|
382
462
|
switch (srcSize & 3)
|
|
@@ -419,32 +499,28 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
|
|
|
419
499
|
if (srcSize < 12) return 0; /* no saving possible : too small input */
|
|
420
500
|
op += 6; /* jumpTable */
|
|
421
501
|
|
|
422
|
-
{
|
|
423
|
-
if (HUF_isError(cSize)) return cSize;
|
|
502
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
|
|
424
503
|
if (cSize==0) return 0;
|
|
425
504
|
MEM_writeLE16(ostart, (U16)cSize);
|
|
426
505
|
op += cSize;
|
|
427
506
|
}
|
|
428
507
|
|
|
429
508
|
ip += segmentSize;
|
|
430
|
-
{
|
|
431
|
-
if (HUF_isError(cSize)) return cSize;
|
|
509
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
|
|
432
510
|
if (cSize==0) return 0;
|
|
433
511
|
MEM_writeLE16(ostart+2, (U16)cSize);
|
|
434
512
|
op += cSize;
|
|
435
513
|
}
|
|
436
514
|
|
|
437
515
|
ip += segmentSize;
|
|
438
|
-
{
|
|
439
|
-
if (HUF_isError(cSize)) return cSize;
|
|
516
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
|
|
440
517
|
if (cSize==0) return 0;
|
|
441
518
|
MEM_writeLE16(ostart+4, (U16)cSize);
|
|
442
519
|
op += cSize;
|
|
443
520
|
}
|
|
444
521
|
|
|
445
522
|
ip += segmentSize;
|
|
446
|
-
{
|
|
447
|
-
if (HUF_isError(cSize)) return cSize;
|
|
523
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) );
|
|
448
524
|
if (cSize==0) return 0;
|
|
449
525
|
op += cSize;
|
|
450
526
|
}
|
|
@@ -453,20 +529,43 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
|
|
|
453
529
|
}
|
|
454
530
|
|
|
455
531
|
|
|
532
|
+
static size_t HUF_compressCTable_internal(
|
|
533
|
+
BYTE* const ostart, BYTE* op, BYTE* const oend,
|
|
534
|
+
const void* src, size_t srcSize,
|
|
535
|
+
unsigned singleStream, const HUF_CElt* CTable)
|
|
536
|
+
{
|
|
537
|
+
size_t const cSize = singleStream ?
|
|
538
|
+
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) :
|
|
539
|
+
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
|
|
540
|
+
if (HUF_isError(cSize)) { return cSize; }
|
|
541
|
+
if (cSize==0) { return 0; } /* uncompressible */
|
|
542
|
+
op += cSize;
|
|
543
|
+
/* check compressibility */
|
|
544
|
+
if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
|
|
545
|
+
return op-ostart;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
/* `workSpace` must a table of at least 1024 unsigned */
|
|
456
550
|
static size_t HUF_compress_internal (
|
|
457
551
|
void* dst, size_t dstSize,
|
|
458
552
|
const void* src, size_t srcSize,
|
|
459
553
|
unsigned maxSymbolValue, unsigned huffLog,
|
|
460
|
-
unsigned singleStream
|
|
554
|
+
unsigned singleStream,
|
|
555
|
+
void* workSpace, size_t wkspSize,
|
|
556
|
+
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat)
|
|
461
557
|
{
|
|
462
558
|
BYTE* const ostart = (BYTE*)dst;
|
|
463
559
|
BYTE* const oend = ostart + dstSize;
|
|
464
560
|
BYTE* op = ostart;
|
|
465
561
|
|
|
466
|
-
U32 count
|
|
467
|
-
|
|
562
|
+
U32* count;
|
|
563
|
+
size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
|
|
564
|
+
HUF_CElt* CTable;
|
|
565
|
+
size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
|
|
468
566
|
|
|
469
567
|
/* checks & inits */
|
|
568
|
+
if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) return ERROR(GENERIC);
|
|
470
569
|
if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
|
|
471
570
|
if (!dstSize) return 0; /* cannot fit within dst budget */
|
|
472
571
|
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
|
|
@@ -474,59 +573,111 @@ static size_t HUF_compress_internal (
|
|
|
474
573
|
if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
|
|
475
574
|
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
|
|
476
575
|
|
|
576
|
+
count = (U32*)workSpace;
|
|
577
|
+
workSpace = (BYTE*)workSpace + countSize;
|
|
578
|
+
wkspSize -= countSize;
|
|
579
|
+
CTable = (HUF_CElt*)workSpace;
|
|
580
|
+
workSpace = (BYTE*)workSpace + CTableSize;
|
|
581
|
+
wkspSize -= CTableSize;
|
|
582
|
+
|
|
583
|
+
/* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
|
|
584
|
+
if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
|
|
585
|
+
return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
|
|
586
|
+
}
|
|
587
|
+
|
|
477
588
|
/* Scan input and build symbol stats */
|
|
478
|
-
{
|
|
479
|
-
if (HUF_isError(largest)) return largest;
|
|
589
|
+
{ CHECK_V_F(largest, FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
|
|
480
590
|
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
|
|
481
591
|
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
|
|
482
592
|
}
|
|
483
593
|
|
|
594
|
+
/* Check validity of previous table */
|
|
595
|
+
if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) {
|
|
596
|
+
*repeat = HUF_repeat_none;
|
|
597
|
+
}
|
|
598
|
+
/* Heuristic : use existing table for small inputs */
|
|
599
|
+
if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
|
|
600
|
+
return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
|
|
601
|
+
}
|
|
602
|
+
|
|
484
603
|
/* Build Huffman Tree */
|
|
485
604
|
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
|
|
486
|
-
{
|
|
487
|
-
if (HUF_isError(maxBits)) return maxBits;
|
|
605
|
+
{ CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) );
|
|
488
606
|
huffLog = (U32)maxBits;
|
|
607
|
+
/* Zero the unused symbols so we can check it for validity */
|
|
608
|
+
memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
|
|
489
609
|
}
|
|
490
610
|
|
|
491
611
|
/* Write table description header */
|
|
492
|
-
{
|
|
493
|
-
if
|
|
494
|
-
if (
|
|
612
|
+
{ CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog) );
|
|
613
|
+
/* Check if using the previous table will be beneficial */
|
|
614
|
+
if (repeat && *repeat != HUF_repeat_none) {
|
|
615
|
+
size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
|
|
616
|
+
size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue);
|
|
617
|
+
if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
|
|
618
|
+
return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
/* Use the new table */
|
|
622
|
+
if (hSize + 12ul >= srcSize) { return 0; }
|
|
495
623
|
op += hSize;
|
|
624
|
+
if (repeat) { *repeat = HUF_repeat_none; }
|
|
625
|
+
if (oldHufTable) { memcpy(oldHufTable, CTable, CTableSize); } /* Save the new table */
|
|
496
626
|
}
|
|
627
|
+
return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
|
|
628
|
+
}
|
|
497
629
|
|
|
498
|
-
/* Compress */
|
|
499
|
-
{ size_t const cSize = (singleStream) ?
|
|
500
|
-
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */
|
|
501
|
-
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
|
|
502
|
-
if (HUF_isError(cSize)) return cSize;
|
|
503
|
-
if (cSize==0) return 0; /* uncompressible */
|
|
504
|
-
op += cSize;
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
/* check compressibility */
|
|
508
|
-
if ((size_t)(op-ostart) >= srcSize-1)
|
|
509
|
-
return 0;
|
|
510
630
|
|
|
511
|
-
|
|
631
|
+
size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
|
|
632
|
+
const void* src, size_t srcSize,
|
|
633
|
+
unsigned maxSymbolValue, unsigned huffLog,
|
|
634
|
+
void* workSpace, size_t wkspSize)
|
|
635
|
+
{
|
|
636
|
+
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
|
|
512
637
|
}
|
|
513
638
|
|
|
639
|
+
size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
|
|
640
|
+
const void* src, size_t srcSize,
|
|
641
|
+
unsigned maxSymbolValue, unsigned huffLog,
|
|
642
|
+
void* workSpace, size_t wkspSize,
|
|
643
|
+
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
|
|
644
|
+
{
|
|
645
|
+
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
|
|
646
|
+
}
|
|
514
647
|
|
|
515
648
|
size_t HUF_compress1X (void* dst, size_t dstSize,
|
|
516
649
|
const void* src, size_t srcSize,
|
|
517
650
|
unsigned maxSymbolValue, unsigned huffLog)
|
|
518
651
|
{
|
|
519
|
-
|
|
652
|
+
unsigned workSpace[1024];
|
|
653
|
+
return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
|
|
657
|
+
const void* src, size_t srcSize,
|
|
658
|
+
unsigned maxSymbolValue, unsigned huffLog,
|
|
659
|
+
void* workSpace, size_t wkspSize)
|
|
660
|
+
{
|
|
661
|
+
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
|
|
665
|
+
const void* src, size_t srcSize,
|
|
666
|
+
unsigned maxSymbolValue, unsigned huffLog,
|
|
667
|
+
void* workSpace, size_t wkspSize,
|
|
668
|
+
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
|
|
669
|
+
{
|
|
670
|
+
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
|
|
520
671
|
}
|
|
521
672
|
|
|
522
673
|
size_t HUF_compress2 (void* dst, size_t dstSize,
|
|
523
674
|
const void* src, size_t srcSize,
|
|
524
675
|
unsigned maxSymbolValue, unsigned huffLog)
|
|
525
676
|
{
|
|
526
|
-
|
|
677
|
+
unsigned workSpace[1024];
|
|
678
|
+
return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
|
|
527
679
|
}
|
|
528
680
|
|
|
529
|
-
|
|
530
681
|
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
|
531
682
|
{
|
|
532
683
|
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);
|