extzstd 0.1 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/HISTORY.ja +5 -0
- data/README.md +5 -5
- data/contrib/zstd/CONTRIBUTING.md +42 -0
- data/contrib/zstd/LICENSE-examples +11 -0
- data/contrib/zstd/Makefile +315 -0
- data/contrib/zstd/NEWS +261 -0
- data/contrib/zstd/PATENTS +33 -0
- data/contrib/zstd/README.md +121 -41
- data/contrib/zstd/TESTING.md +44 -0
- data/contrib/zstd/appveyor.yml +178 -0
- data/contrib/zstd/circle.yml +75 -0
- data/contrib/zstd/lib/BUCK +186 -0
- data/contrib/zstd/lib/Makefile +163 -0
- data/contrib/zstd/lib/README.md +77 -0
- data/contrib/zstd/{common → lib/common}/bitstream.h +7 -4
- data/contrib/zstd/{common → lib/common}/entropy_common.c +19 -23
- data/contrib/zstd/{common → lib/common}/error_private.c +0 -0
- data/contrib/zstd/{common → lib/common}/error_private.h +0 -0
- data/contrib/zstd/{common → lib/common}/fse.h +94 -34
- data/contrib/zstd/{common → lib/common}/fse_decompress.c +18 -19
- data/contrib/zstd/{common → lib/common}/huf.h +52 -20
- data/contrib/zstd/{common → lib/common}/mem.h +17 -13
- data/contrib/zstd/lib/common/pool.c +194 -0
- data/contrib/zstd/lib/common/pool.h +56 -0
- data/contrib/zstd/lib/common/threading.c +80 -0
- data/contrib/zstd/lib/common/threading.h +104 -0
- data/contrib/zstd/{common → lib/common}/xxhash.c +3 -1
- data/contrib/zstd/{common → lib/common}/xxhash.h +11 -15
- data/contrib/zstd/{common → lib/common}/zstd_common.c +1 -11
- data/contrib/zstd/{common → lib/common}/zstd_errors.h +16 -2
- data/contrib/zstd/{common → lib/common}/zstd_internal.h +17 -1
- data/contrib/zstd/{compress → lib/compress}/fse_compress.c +138 -91
- data/contrib/zstd/{compress → lib/compress}/huf_compress.c +218 -67
- data/contrib/zstd/{compress → lib/compress}/zstd_compress.c +231 -108
- data/contrib/zstd/{compress → lib/compress}/zstd_opt.h +44 -25
- data/contrib/zstd/lib/compress/zstdmt_compress.c +739 -0
- data/contrib/zstd/lib/compress/zstdmt_compress.h +78 -0
- data/contrib/zstd/{decompress → lib/decompress}/huf_decompress.c +28 -23
- data/contrib/zstd/{decompress → lib/decompress}/zstd_decompress.c +814 -176
- data/contrib/zstd/{common → lib/deprecated}/zbuff.h +60 -39
- data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +145 -0
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +74 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +1029 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +0 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
- data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +68 -18
- data/contrib/zstd/lib/dictBuilder/zdict.h +201 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +122 -7
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +34 -3
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +45 -12
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +45 -12
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +56 -33
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +8 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +45 -18
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +7 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +43 -16
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +7 -0
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +57 -23
- data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +8 -0
- data/contrib/zstd/lib/libzstd.pc.in +14 -0
- data/contrib/zstd/{zstd.h → lib/zstd.h} +206 -71
- data/ext/depend +2 -0
- data/ext/extconf.rb +4 -4
- data/ext/extzstd.c +1 -1
- data/ext/zstd_common.c +5 -5
- data/ext/zstd_compress.c +3 -3
- data/ext/zstd_decompress.c +2 -2
- data/ext/zstd_dictbuilder.c +2 -2
- data/ext/zstd_legacy_v01.c +1 -1
- data/ext/zstd_legacy_v02.c +1 -1
- data/ext/zstd_legacy_v03.c +1 -1
- data/ext/zstd_legacy_v04.c +1 -1
- data/ext/zstd_legacy_v05.c +1 -1
- data/ext/zstd_legacy_v06.c +1 -1
- data/ext/zstd_legacy_v07.c +1 -1
- data/gemstub.rb +9 -5
- data/lib/extzstd/version.rb +1 -1
- metadata +73 -51
- data/contrib/zstd/compress/zbuff_compress.c +0 -319
- data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
- data/contrib/zstd/dictBuilder/zdict.h +0 -111
@@ -56,6 +56,8 @@
|
|
56
56
|
* Error Management
|
57
57
|
****************************************************************/
|
58
58
|
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
59
|
+
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f
|
60
|
+
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
|
59
61
|
|
60
62
|
|
61
63
|
/* **************************************************************
|
@@ -70,31 +72,73 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
|
|
70
72
|
/* *******************************************************
|
71
73
|
* HUF : Huffman block compression
|
72
74
|
*********************************************************/
|
75
|
+
/* HUF_compressWeights() :
|
76
|
+
* Same as FSE_compress(), but dedicated to huff0's weights compression.
|
77
|
+
* The use case needs much less stack memory.
|
78
|
+
* Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
|
79
|
+
*/
|
80
|
+
#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
|
81
|
+
size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize)
|
82
|
+
{
|
83
|
+
BYTE* const ostart = (BYTE*) dst;
|
84
|
+
BYTE* op = ostart;
|
85
|
+
BYTE* const oend = ostart + dstSize;
|
86
|
+
|
87
|
+
U32 maxSymbolValue = HUF_TABLELOG_MAX;
|
88
|
+
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
|
89
|
+
|
90
|
+
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
|
91
|
+
BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
|
92
|
+
|
93
|
+
U32 count[HUF_TABLELOG_MAX+1];
|
94
|
+
S16 norm[HUF_TABLELOG_MAX+1];
|
95
|
+
|
96
|
+
/* init conditions */
|
97
|
+
if (wtSize <= 1) return 0; /* Not compressible */
|
98
|
+
|
99
|
+
/* Scan input and build symbol stats */
|
100
|
+
{ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) );
|
101
|
+
if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
|
102
|
+
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
|
103
|
+
}
|
104
|
+
|
105
|
+
tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
|
106
|
+
CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) );
|
107
|
+
|
108
|
+
/* Write table description header */
|
109
|
+
{ CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
|
110
|
+
op += hSize;
|
111
|
+
}
|
112
|
+
|
113
|
+
/* Compress */
|
114
|
+
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) );
|
115
|
+
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) );
|
116
|
+
if (cSize == 0) return 0; /* not enough space for compressed data */
|
117
|
+
op += cSize;
|
118
|
+
}
|
119
|
+
|
120
|
+
return op-ostart;
|
121
|
+
}
|
122
|
+
|
123
|
+
|
73
124
|
struct HUF_CElt_s {
|
74
125
|
U16 val;
|
75
126
|
BYTE nbBits;
|
76
127
|
}; /* typedef'd to HUF_CElt within "huf.h" */
|
77
128
|
|
78
|
-
typedef struct nodeElt_s {
|
79
|
-
U32 count;
|
80
|
-
U16 parent;
|
81
|
-
BYTE byte;
|
82
|
-
BYTE nbBits;
|
83
|
-
} nodeElt;
|
84
|
-
|
85
129
|
/*! HUF_writeCTable() :
|
86
|
-
`CTable` :
|
130
|
+
`CTable` : Huffman tree to save, using huf representation.
|
87
131
|
@return : size of saved CTable */
|
88
132
|
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
89
133
|
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
|
90
134
|
{
|
91
|
-
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
|
135
|
+
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
|
92
136
|
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
|
93
137
|
BYTE* op = (BYTE*)dst;
|
94
138
|
U32 n;
|
95
139
|
|
96
140
|
/* check conditions */
|
97
|
-
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(
|
141
|
+
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
|
98
142
|
|
99
143
|
/* convert to weight */
|
100
144
|
bitsToWeight[0] = 0;
|
@@ -103,38 +147,33 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
|
103
147
|
for (n=0; n<maxSymbolValue; n++)
|
104
148
|
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
|
105
149
|
|
106
|
-
|
107
|
-
|
108
|
-
if ((
|
109
|
-
op[0] = (BYTE)
|
110
|
-
return
|
111
|
-
|
112
|
-
}
|
150
|
+
/* attempt weights compression by FSE */
|
151
|
+
{ CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) );
|
152
|
+
if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
|
153
|
+
op[0] = (BYTE)hSize;
|
154
|
+
return hSize+1;
|
155
|
+
} }
|
113
156
|
|
114
|
-
/* raw values */
|
115
|
-
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */
|
157
|
+
/* write raw values as 4-bits (max : 15) */
|
158
|
+
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
|
116
159
|
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
|
117
160
|
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
|
118
|
-
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
|
161
|
+
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
|
119
162
|
for (n=0; n<maxSymbolValue; n+=2)
|
120
163
|
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
|
121
164
|
return ((maxSymbolValue+1)/2) + 1;
|
122
|
-
|
123
165
|
}
|
124
166
|
|
125
167
|
|
126
168
|
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
|
127
169
|
{
|
128
|
-
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
|
170
|
+
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
|
129
171
|
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
130
172
|
U32 tableLog = 0;
|
131
|
-
size_t readSize;
|
132
173
|
U32 nbSymbols = 0;
|
133
|
-
/*memset(huffWeight, 0, sizeof(huffWeight));*/ /* is not necessary, even though some analyzer complain ... */
|
134
174
|
|
135
175
|
/* get symbol weights */
|
136
|
-
readSize
|
137
|
-
if (HUF_isError(readSize)) return readSize;
|
176
|
+
CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize));
|
138
177
|
|
139
178
|
/* check result */
|
140
179
|
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
@@ -174,6 +213,13 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
|
|
174
213
|
}
|
175
214
|
|
176
215
|
|
216
|
+
typedef struct nodeElt_s {
|
217
|
+
U32 count;
|
218
|
+
U16 parent;
|
219
|
+
BYTE byte;
|
220
|
+
BYTE nbBits;
|
221
|
+
} nodeElt;
|
222
|
+
|
177
223
|
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
178
224
|
{
|
179
225
|
const U32 largestBits = huffNode[lastNonNull].nbBits;
|
@@ -279,20 +325,26 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
|
|
279
325
|
}
|
280
326
|
|
281
327
|
|
328
|
+
/** HUF_buildCTable_wksp() :
|
329
|
+
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
|
330
|
+
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
|
331
|
+
*/
|
282
332
|
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
|
283
|
-
|
333
|
+
typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1];
|
334
|
+
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
|
284
335
|
{
|
285
|
-
nodeElt huffNode0
|
286
|
-
nodeElt* huffNode = huffNode0
|
336
|
+
nodeElt* const huffNode0 = (nodeElt*)workSpace;
|
337
|
+
nodeElt* const huffNode = huffNode0+1;
|
287
338
|
U32 n, nonNullRank;
|
288
339
|
int lowS, lowN;
|
289
340
|
U16 nodeNb = STARTNODE;
|
290
341
|
U32 nodeRoot;
|
291
342
|
|
292
343
|
/* safety checks */
|
344
|
+
if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */
|
293
345
|
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
|
294
346
|
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
|
295
|
-
memset(huffNode0, 0, sizeof(
|
347
|
+
memset(huffNode0, 0, sizeof(huffNodeTable));
|
296
348
|
|
297
349
|
/* sort, decreasing order */
|
298
350
|
HUF_sort(huffNode, count, maxSymbolValue);
|
@@ -305,7 +357,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
|
|
305
357
|
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
|
306
358
|
nodeNb++; lowS-=2;
|
307
359
|
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
|
308
|
-
huffNode0[0].count = (U32)(1U<<31);
|
360
|
+
huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */
|
309
361
|
|
310
362
|
/* create parents */
|
311
363
|
while (nodeNb <= nodeRoot) {
|
@@ -348,6 +400,34 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
|
|
348
400
|
return maxNbBits;
|
349
401
|
}
|
350
402
|
|
403
|
+
/** HUF_buildCTable() :
|
404
|
+
* Note : count is used before tree is written, so they can safely overlap
|
405
|
+
*/
|
406
|
+
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
|
407
|
+
{
|
408
|
+
huffNodeTable nodeTable;
|
409
|
+
return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
|
410
|
+
}
|
411
|
+
|
412
|
+
static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
|
413
|
+
{
|
414
|
+
size_t nbBits = 0;
|
415
|
+
int s;
|
416
|
+
for (s = 0; s <= (int)maxSymbolValue; ++s) {
|
417
|
+
nbBits += CTable[s].nbBits * count[s];
|
418
|
+
}
|
419
|
+
return nbBits >> 3;
|
420
|
+
}
|
421
|
+
|
422
|
+
static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
|
423
|
+
int bad = 0;
|
424
|
+
int s;
|
425
|
+
for (s = 0; s <= (int)maxSymbolValue; ++s) {
|
426
|
+
bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
|
427
|
+
}
|
428
|
+
return !bad;
|
429
|
+
}
|
430
|
+
|
351
431
|
static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
|
352
432
|
{
|
353
433
|
BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
|
@@ -375,8 +455,8 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
|
|
375
455
|
|
376
456
|
/* init */
|
377
457
|
if (dstSize < 8) return 0; /* not enough space to compress */
|
378
|
-
{ size_t const
|
379
|
-
if (HUF_isError(
|
458
|
+
{ size_t const initErr = BIT_initCStream(&bitC, op, oend-op);
|
459
|
+
if (HUF_isError(initErr)) return 0; }
|
380
460
|
|
381
461
|
n = srcSize & ~3; /* join to mod 4 */
|
382
462
|
switch (srcSize & 3)
|
@@ -419,32 +499,28 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
|
|
419
499
|
if (srcSize < 12) return 0; /* no saving possible : too small input */
|
420
500
|
op += 6; /* jumpTable */
|
421
501
|
|
422
|
-
{
|
423
|
-
if (HUF_isError(cSize)) return cSize;
|
502
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
|
424
503
|
if (cSize==0) return 0;
|
425
504
|
MEM_writeLE16(ostart, (U16)cSize);
|
426
505
|
op += cSize;
|
427
506
|
}
|
428
507
|
|
429
508
|
ip += segmentSize;
|
430
|
-
{
|
431
|
-
if (HUF_isError(cSize)) return cSize;
|
509
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
|
432
510
|
if (cSize==0) return 0;
|
433
511
|
MEM_writeLE16(ostart+2, (U16)cSize);
|
434
512
|
op += cSize;
|
435
513
|
}
|
436
514
|
|
437
515
|
ip += segmentSize;
|
438
|
-
{
|
439
|
-
if (HUF_isError(cSize)) return cSize;
|
516
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
|
440
517
|
if (cSize==0) return 0;
|
441
518
|
MEM_writeLE16(ostart+4, (U16)cSize);
|
442
519
|
op += cSize;
|
443
520
|
}
|
444
521
|
|
445
522
|
ip += segmentSize;
|
446
|
-
{
|
447
|
-
if (HUF_isError(cSize)) return cSize;
|
523
|
+
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) );
|
448
524
|
if (cSize==0) return 0;
|
449
525
|
op += cSize;
|
450
526
|
}
|
@@ -453,20 +529,43 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
|
|
453
529
|
}
|
454
530
|
|
455
531
|
|
532
|
+
static size_t HUF_compressCTable_internal(
|
533
|
+
BYTE* const ostart, BYTE* op, BYTE* const oend,
|
534
|
+
const void* src, size_t srcSize,
|
535
|
+
unsigned singleStream, const HUF_CElt* CTable)
|
536
|
+
{
|
537
|
+
size_t const cSize = singleStream ?
|
538
|
+
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) :
|
539
|
+
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
|
540
|
+
if (HUF_isError(cSize)) { return cSize; }
|
541
|
+
if (cSize==0) { return 0; } /* uncompressible */
|
542
|
+
op += cSize;
|
543
|
+
/* check compressibility */
|
544
|
+
if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
|
545
|
+
return op-ostart;
|
546
|
+
}
|
547
|
+
|
548
|
+
|
549
|
+
/* `workSpace` must a table of at least 1024 unsigned */
|
456
550
|
static size_t HUF_compress_internal (
|
457
551
|
void* dst, size_t dstSize,
|
458
552
|
const void* src, size_t srcSize,
|
459
553
|
unsigned maxSymbolValue, unsigned huffLog,
|
460
|
-
unsigned singleStream
|
554
|
+
unsigned singleStream,
|
555
|
+
void* workSpace, size_t wkspSize,
|
556
|
+
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat)
|
461
557
|
{
|
462
558
|
BYTE* const ostart = (BYTE*)dst;
|
463
559
|
BYTE* const oend = ostart + dstSize;
|
464
560
|
BYTE* op = ostart;
|
465
561
|
|
466
|
-
U32 count
|
467
|
-
|
562
|
+
U32* count;
|
563
|
+
size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
|
564
|
+
HUF_CElt* CTable;
|
565
|
+
size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
|
468
566
|
|
469
567
|
/* checks & inits */
|
568
|
+
if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) return ERROR(GENERIC);
|
470
569
|
if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
|
471
570
|
if (!dstSize) return 0; /* cannot fit within dst budget */
|
472
571
|
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
|
@@ -474,59 +573,111 @@ static size_t HUF_compress_internal (
|
|
474
573
|
if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
|
475
574
|
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
|
476
575
|
|
576
|
+
count = (U32*)workSpace;
|
577
|
+
workSpace = (BYTE*)workSpace + countSize;
|
578
|
+
wkspSize -= countSize;
|
579
|
+
CTable = (HUF_CElt*)workSpace;
|
580
|
+
workSpace = (BYTE*)workSpace + CTableSize;
|
581
|
+
wkspSize -= CTableSize;
|
582
|
+
|
583
|
+
/* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
|
584
|
+
if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
|
585
|
+
return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
|
586
|
+
}
|
587
|
+
|
477
588
|
/* Scan input and build symbol stats */
|
478
|
-
{
|
479
|
-
if (HUF_isError(largest)) return largest;
|
589
|
+
{ CHECK_V_F(largest, FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
|
480
590
|
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
|
481
591
|
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
|
482
592
|
}
|
483
593
|
|
594
|
+
/* Check validity of previous table */
|
595
|
+
if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) {
|
596
|
+
*repeat = HUF_repeat_none;
|
597
|
+
}
|
598
|
+
/* Heuristic : use existing table for small inputs */
|
599
|
+
if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
|
600
|
+
return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
|
601
|
+
}
|
602
|
+
|
484
603
|
/* Build Huffman Tree */
|
485
604
|
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
|
486
|
-
{
|
487
|
-
if (HUF_isError(maxBits)) return maxBits;
|
605
|
+
{ CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) );
|
488
606
|
huffLog = (U32)maxBits;
|
607
|
+
/* Zero the unused symbols so we can check it for validity */
|
608
|
+
memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
|
489
609
|
}
|
490
610
|
|
491
611
|
/* Write table description header */
|
492
|
-
{
|
493
|
-
if
|
494
|
-
if (
|
612
|
+
{ CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog) );
|
613
|
+
/* Check if using the previous table will be beneficial */
|
614
|
+
if (repeat && *repeat != HUF_repeat_none) {
|
615
|
+
size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
|
616
|
+
size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue);
|
617
|
+
if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
|
618
|
+
return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
|
619
|
+
}
|
620
|
+
}
|
621
|
+
/* Use the new table */
|
622
|
+
if (hSize + 12ul >= srcSize) { return 0; }
|
495
623
|
op += hSize;
|
624
|
+
if (repeat) { *repeat = HUF_repeat_none; }
|
625
|
+
if (oldHufTable) { memcpy(oldHufTable, CTable, CTableSize); } /* Save the new table */
|
496
626
|
}
|
627
|
+
return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
|
628
|
+
}
|
497
629
|
|
498
|
-
/* Compress */
|
499
|
-
{ size_t const cSize = (singleStream) ?
|
500
|
-
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */
|
501
|
-
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
|
502
|
-
if (HUF_isError(cSize)) return cSize;
|
503
|
-
if (cSize==0) return 0; /* uncompressible */
|
504
|
-
op += cSize;
|
505
|
-
}
|
506
|
-
|
507
|
-
/* check compressibility */
|
508
|
-
if ((size_t)(op-ostart) >= srcSize-1)
|
509
|
-
return 0;
|
510
630
|
|
511
|
-
|
631
|
+
size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
|
632
|
+
const void* src, size_t srcSize,
|
633
|
+
unsigned maxSymbolValue, unsigned huffLog,
|
634
|
+
void* workSpace, size_t wkspSize)
|
635
|
+
{
|
636
|
+
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
|
512
637
|
}
|
513
638
|
|
639
|
+
size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
|
640
|
+
const void* src, size_t srcSize,
|
641
|
+
unsigned maxSymbolValue, unsigned huffLog,
|
642
|
+
void* workSpace, size_t wkspSize,
|
643
|
+
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
|
644
|
+
{
|
645
|
+
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
|
646
|
+
}
|
514
647
|
|
515
648
|
size_t HUF_compress1X (void* dst, size_t dstSize,
|
516
649
|
const void* src, size_t srcSize,
|
517
650
|
unsigned maxSymbolValue, unsigned huffLog)
|
518
651
|
{
|
519
|
-
|
652
|
+
unsigned workSpace[1024];
|
653
|
+
return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
|
654
|
+
}
|
655
|
+
|
656
|
+
size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
|
657
|
+
const void* src, size_t srcSize,
|
658
|
+
unsigned maxSymbolValue, unsigned huffLog,
|
659
|
+
void* workSpace, size_t wkspSize)
|
660
|
+
{
|
661
|
+
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
|
662
|
+
}
|
663
|
+
|
664
|
+
size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
|
665
|
+
const void* src, size_t srcSize,
|
666
|
+
unsigned maxSymbolValue, unsigned huffLog,
|
667
|
+
void* workSpace, size_t wkspSize,
|
668
|
+
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
|
669
|
+
{
|
670
|
+
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
|
520
671
|
}
|
521
672
|
|
522
673
|
size_t HUF_compress2 (void* dst, size_t dstSize,
|
523
674
|
const void* src, size_t srcSize,
|
524
675
|
unsigned maxSymbolValue, unsigned huffLog)
|
525
676
|
{
|
526
|
-
|
677
|
+
unsigned workSpace[1024];
|
678
|
+
return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
|
527
679
|
}
|
528
680
|
|
529
|
-
|
530
681
|
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
531
682
|
{
|
532
683
|
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);
|