extzstd 0.1 → 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja +5 -0
  3. data/README.md +5 -5
  4. data/contrib/zstd/CONTRIBUTING.md +42 -0
  5. data/contrib/zstd/LICENSE-examples +11 -0
  6. data/contrib/zstd/Makefile +315 -0
  7. data/contrib/zstd/NEWS +261 -0
  8. data/contrib/zstd/PATENTS +33 -0
  9. data/contrib/zstd/README.md +121 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +178 -0
  12. data/contrib/zstd/circle.yml +75 -0
  13. data/contrib/zstd/lib/BUCK +186 -0
  14. data/contrib/zstd/lib/Makefile +163 -0
  15. data/contrib/zstd/lib/README.md +77 -0
  16. data/contrib/zstd/{common → lib/common}/bitstream.h +7 -4
  17. data/contrib/zstd/{common → lib/common}/entropy_common.c +19 -23
  18. data/contrib/zstd/{common → lib/common}/error_private.c +0 -0
  19. data/contrib/zstd/{common → lib/common}/error_private.h +0 -0
  20. data/contrib/zstd/{common → lib/common}/fse.h +94 -34
  21. data/contrib/zstd/{common → lib/common}/fse_decompress.c +18 -19
  22. data/contrib/zstd/{common → lib/common}/huf.h +52 -20
  23. data/contrib/zstd/{common → lib/common}/mem.h +17 -13
  24. data/contrib/zstd/lib/common/pool.c +194 -0
  25. data/contrib/zstd/lib/common/pool.h +56 -0
  26. data/contrib/zstd/lib/common/threading.c +80 -0
  27. data/contrib/zstd/lib/common/threading.h +104 -0
  28. data/contrib/zstd/{common → lib/common}/xxhash.c +3 -1
  29. data/contrib/zstd/{common → lib/common}/xxhash.h +11 -15
  30. data/contrib/zstd/{common → lib/common}/zstd_common.c +1 -11
  31. data/contrib/zstd/{common → lib/common}/zstd_errors.h +16 -2
  32. data/contrib/zstd/{common → lib/common}/zstd_internal.h +17 -1
  33. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +138 -91
  34. data/contrib/zstd/{compress → lib/compress}/huf_compress.c +218 -67
  35. data/contrib/zstd/{compress → lib/compress}/zstd_compress.c +231 -108
  36. data/contrib/zstd/{compress → lib/compress}/zstd_opt.h +44 -25
  37. data/contrib/zstd/lib/compress/zstdmt_compress.c +739 -0
  38. data/contrib/zstd/lib/compress/zstdmt_compress.h +78 -0
  39. data/contrib/zstd/{decompress → lib/decompress}/huf_decompress.c +28 -23
  40. data/contrib/zstd/{decompress → lib/decompress}/zstd_decompress.c +814 -176
  41. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +60 -39
  42. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  43. data/contrib/zstd/lib/deprecated/zbuff_compress.c +145 -0
  44. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +74 -0
  45. data/contrib/zstd/lib/dictBuilder/cover.c +1029 -0
  46. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +0 -0
  47. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  48. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +68 -18
  49. data/contrib/zstd/lib/dictBuilder/zdict.h +201 -0
  50. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +122 -7
  51. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +34 -3
  52. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +8 -0
  53. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +45 -12
  54. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +8 -0
  55. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +45 -12
  56. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +8 -0
  57. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +56 -33
  58. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +8 -0
  59. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +45 -18
  60. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +7 -0
  61. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +43 -16
  62. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +7 -0
  63. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +57 -23
  64. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +8 -0
  65. data/contrib/zstd/lib/libzstd.pc.in +14 -0
  66. data/contrib/zstd/{zstd.h → lib/zstd.h} +206 -71
  67. data/ext/depend +2 -0
  68. data/ext/extconf.rb +4 -4
  69. data/ext/extzstd.c +1 -1
  70. data/ext/zstd_common.c +5 -5
  71. data/ext/zstd_compress.c +3 -3
  72. data/ext/zstd_decompress.c +2 -2
  73. data/ext/zstd_dictbuilder.c +2 -2
  74. data/ext/zstd_legacy_v01.c +1 -1
  75. data/ext/zstd_legacy_v02.c +1 -1
  76. data/ext/zstd_legacy_v03.c +1 -1
  77. data/ext/zstd_legacy_v04.c +1 -1
  78. data/ext/zstd_legacy_v05.c +1 -1
  79. data/ext/zstd_legacy_v06.c +1 -1
  80. data/ext/zstd_legacy_v07.c +1 -1
  81. data/gemstub.rb +9 -5
  82. data/lib/extzstd/version.rb +1 -1
  83. metadata +73 -51
  84. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  85. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  86. data/contrib/zstd/dictBuilder/zdict.h +0 -111
@@ -56,6 +56,8 @@
56
56
  * Error Management
57
57
  ****************************************************************/
58
58
  #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
59
+ #define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f
60
+ #define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
59
61
 
60
62
 
61
63
  /* **************************************************************
@@ -70,31 +72,73 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
70
72
  /* *******************************************************
71
73
  * HUF : Huffman block compression
72
74
  *********************************************************/
75
+ /* HUF_compressWeights() :
76
+ * Same as FSE_compress(), but dedicated to huff0's weights compression.
77
+ * The use case needs much less stack memory.
78
+ * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
79
+ */
80
+ #define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
81
+ size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize)
82
+ {
83
+ BYTE* const ostart = (BYTE*) dst;
84
+ BYTE* op = ostart;
85
+ BYTE* const oend = ostart + dstSize;
86
+
87
+ U32 maxSymbolValue = HUF_TABLELOG_MAX;
88
+ U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
89
+
90
+ FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
91
+ BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
92
+
93
+ U32 count[HUF_TABLELOG_MAX+1];
94
+ S16 norm[HUF_TABLELOG_MAX+1];
95
+
96
+ /* init conditions */
97
+ if (wtSize <= 1) return 0; /* Not compressible */
98
+
99
+ /* Scan input and build symbol stats */
100
+ { CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) );
101
+ if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
102
+ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
103
+ }
104
+
105
+ tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
106
+ CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) );
107
+
108
+ /* Write table description header */
109
+ { CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
110
+ op += hSize;
111
+ }
112
+
113
+ /* Compress */
114
+ CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) );
115
+ { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) );
116
+ if (cSize == 0) return 0; /* not enough space for compressed data */
117
+ op += cSize;
118
+ }
119
+
120
+ return op-ostart;
121
+ }
122
+
123
+
73
124
  struct HUF_CElt_s {
74
125
  U16 val;
75
126
  BYTE nbBits;
76
127
  }; /* typedef'd to HUF_CElt within "huf.h" */
77
128
 
78
- typedef struct nodeElt_s {
79
- U32 count;
80
- U16 parent;
81
- BYTE byte;
82
- BYTE nbBits;
83
- } nodeElt;
84
-
85
129
  /*! HUF_writeCTable() :
86
- `CTable` : huffman tree to save, using huf representation.
130
+ `CTable` : Huffman tree to save, using huf representation.
87
131
  @return : size of saved CTable */
88
132
  size_t HUF_writeCTable (void* dst, size_t maxDstSize,
89
133
  const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
90
134
  {
91
- BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
135
+ BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
92
136
  BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
93
137
  BYTE* op = (BYTE*)dst;
94
138
  U32 n;
95
139
 
96
140
  /* check conditions */
97
- if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
141
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
98
142
 
99
143
  /* convert to weight */
100
144
  bitsToWeight[0] = 0;
@@ -103,38 +147,33 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
103
147
  for (n=0; n<maxSymbolValue; n++)
104
148
  huffWeight[n] = bitsToWeight[CTable[n].nbBits];
105
149
 
106
- { size_t const size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue);
107
- if (FSE_isError(size)) return size;
108
- if ((size>1) & (size < maxSymbolValue/2)) { /* FSE compressed */
109
- op[0] = (BYTE)size;
110
- return size+1;
111
- }
112
- }
150
+ /* attempt weights compression by FSE */
151
+ { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) );
152
+ if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
153
+ op[0] = (BYTE)hSize;
154
+ return hSize+1;
155
+ } }
113
156
 
114
- /* raw values */
115
- if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */
157
+ /* write raw values as 4-bits (max : 15) */
158
+ if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
116
159
  if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
117
160
  op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
118
- huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
161
+ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
119
162
  for (n=0; n<maxSymbolValue; n+=2)
120
163
  op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
121
164
  return ((maxSymbolValue+1)/2) + 1;
122
-
123
165
  }
124
166
 
125
167
 
126
168
  size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
127
169
  {
128
- BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
170
+ BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
129
171
  U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
130
172
  U32 tableLog = 0;
131
- size_t readSize;
132
173
  U32 nbSymbols = 0;
133
- /*memset(huffWeight, 0, sizeof(huffWeight));*/ /* is not necessary, even though some analyzer complain ... */
134
174
 
135
175
  /* get symbol weights */
136
- readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
137
- if (HUF_isError(readSize)) return readSize;
176
+ CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize));
138
177
 
139
178
  /* check result */
140
179
  if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
@@ -174,6 +213,13 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
174
213
  }
175
214
 
176
215
 
216
+ typedef struct nodeElt_s {
217
+ U32 count;
218
+ U16 parent;
219
+ BYTE byte;
220
+ BYTE nbBits;
221
+ } nodeElt;
222
+
177
223
  static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
178
224
  {
179
225
  const U32 largestBits = huffNode[lastNonNull].nbBits;
@@ -279,20 +325,26 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
279
325
  }
280
326
 
281
327
 
328
+ /** HUF_buildCTable_wksp() :
329
+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
330
+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
331
+ */
282
332
  #define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
283
- size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
333
+ typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1];
334
+ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
284
335
  {
285
- nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1];
286
- nodeElt* huffNode = huffNode0 + 1;
336
+ nodeElt* const huffNode0 = (nodeElt*)workSpace;
337
+ nodeElt* const huffNode = huffNode0+1;
287
338
  U32 n, nonNullRank;
288
339
  int lowS, lowN;
289
340
  U16 nodeNb = STARTNODE;
290
341
  U32 nodeRoot;
291
342
 
292
343
  /* safety checks */
344
+ if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */
293
345
  if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
294
346
  if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
295
- memset(huffNode0, 0, sizeof(huffNode0));
347
+ memset(huffNode0, 0, sizeof(huffNodeTable));
296
348
 
297
349
  /* sort, decreasing order */
298
350
  HUF_sort(huffNode, count, maxSymbolValue);
@@ -305,7 +357,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
305
357
  huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
306
358
  nodeNb++; lowS-=2;
307
359
  for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
308
- huffNode0[0].count = (U32)(1U<<31);
360
+ huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */
309
361
 
310
362
  /* create parents */
311
363
  while (nodeNb <= nodeRoot) {
@@ -348,6 +400,34 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
348
400
  return maxNbBits;
349
401
  }
350
402
 
403
+ /** HUF_buildCTable() :
404
+ * Note : count is used before tree is written, so they can safely overlap
405
+ */
406
+ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
407
+ {
408
+ huffNodeTable nodeTable;
409
+ return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
410
+ }
411
+
412
+ static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
413
+ {
414
+ size_t nbBits = 0;
415
+ int s;
416
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
417
+ nbBits += CTable[s].nbBits * count[s];
418
+ }
419
+ return nbBits >> 3;
420
+ }
421
+
422
+ static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
423
+ int bad = 0;
424
+ int s;
425
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
426
+ bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
427
+ }
428
+ return !bad;
429
+ }
430
+
351
431
  static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
352
432
  {
353
433
  BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
@@ -375,8 +455,8 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
375
455
 
376
456
  /* init */
377
457
  if (dstSize < 8) return 0; /* not enough space to compress */
378
- { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op);
379
- if (HUF_isError(errorCode)) return 0; }
458
+ { size_t const initErr = BIT_initCStream(&bitC, op, oend-op);
459
+ if (HUF_isError(initErr)) return 0; }
380
460
 
381
461
  n = srcSize & ~3; /* join to mod 4 */
382
462
  switch (srcSize & 3)
@@ -419,32 +499,28 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
419
499
  if (srcSize < 12) return 0; /* no saving possible : too small input */
420
500
  op += 6; /* jumpTable */
421
501
 
422
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
423
- if (HUF_isError(cSize)) return cSize;
502
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
424
503
  if (cSize==0) return 0;
425
504
  MEM_writeLE16(ostart, (U16)cSize);
426
505
  op += cSize;
427
506
  }
428
507
 
429
508
  ip += segmentSize;
430
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
431
- if (HUF_isError(cSize)) return cSize;
509
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
432
510
  if (cSize==0) return 0;
433
511
  MEM_writeLE16(ostart+2, (U16)cSize);
434
512
  op += cSize;
435
513
  }
436
514
 
437
515
  ip += segmentSize;
438
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
439
- if (HUF_isError(cSize)) return cSize;
516
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
440
517
  if (cSize==0) return 0;
441
518
  MEM_writeLE16(ostart+4, (U16)cSize);
442
519
  op += cSize;
443
520
  }
444
521
 
445
522
  ip += segmentSize;
446
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
447
- if (HUF_isError(cSize)) return cSize;
523
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) );
448
524
  if (cSize==0) return 0;
449
525
  op += cSize;
450
526
  }
@@ -453,20 +529,43 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
453
529
  }
454
530
 
455
531
 
532
+ static size_t HUF_compressCTable_internal(
533
+ BYTE* const ostart, BYTE* op, BYTE* const oend,
534
+ const void* src, size_t srcSize,
535
+ unsigned singleStream, const HUF_CElt* CTable)
536
+ {
537
+ size_t const cSize = singleStream ?
538
+ HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) :
539
+ HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
540
+ if (HUF_isError(cSize)) { return cSize; }
541
+ if (cSize==0) { return 0; } /* uncompressible */
542
+ op += cSize;
543
+ /* check compressibility */
544
+ if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
545
+ return op-ostart;
546
+ }
547
+
548
+
549
+ /* `workSpace` must a table of at least 1024 unsigned */
456
550
  static size_t HUF_compress_internal (
457
551
  void* dst, size_t dstSize,
458
552
  const void* src, size_t srcSize,
459
553
  unsigned maxSymbolValue, unsigned huffLog,
460
- unsigned singleStream)
554
+ unsigned singleStream,
555
+ void* workSpace, size_t wkspSize,
556
+ HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat)
461
557
  {
462
558
  BYTE* const ostart = (BYTE*)dst;
463
559
  BYTE* const oend = ostart + dstSize;
464
560
  BYTE* op = ostart;
465
561
 
466
- U32 count[HUF_SYMBOLVALUE_MAX+1];
467
- HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
562
+ U32* count;
563
+ size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
564
+ HUF_CElt* CTable;
565
+ size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
468
566
 
469
567
  /* checks & inits */
568
+ if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) return ERROR(GENERIC);
470
569
  if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
471
570
  if (!dstSize) return 0; /* cannot fit within dst budget */
472
571
  if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
@@ -474,59 +573,111 @@ static size_t HUF_compress_internal (
474
573
  if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
475
574
  if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
476
575
 
576
+ count = (U32*)workSpace;
577
+ workSpace = (BYTE*)workSpace + countSize;
578
+ wkspSize -= countSize;
579
+ CTable = (HUF_CElt*)workSpace;
580
+ workSpace = (BYTE*)workSpace + CTableSize;
581
+ wkspSize -= CTableSize;
582
+
583
+ /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
584
+ if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
585
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
586
+ }
587
+
477
588
  /* Scan input and build symbol stats */
478
- { size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
479
- if (HUF_isError(largest)) return largest;
589
+ { CHECK_V_F(largest, FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
480
590
  if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
481
591
  if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
482
592
  }
483
593
 
594
+ /* Check validity of previous table */
595
+ if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) {
596
+ *repeat = HUF_repeat_none;
597
+ }
598
+ /* Heuristic : use existing table for small inputs */
599
+ if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
600
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
601
+ }
602
+
484
603
  /* Build Huffman Tree */
485
604
  huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
486
- { size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
487
- if (HUF_isError(maxBits)) return maxBits;
605
+ { CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) );
488
606
  huffLog = (U32)maxBits;
607
+ /* Zero the unused symbols so we can check it for validity */
608
+ memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
489
609
  }
490
610
 
491
611
  /* Write table description header */
492
- { size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
493
- if (HUF_isError(hSize)) return hSize;
494
- if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */
612
+ { CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog) );
613
+ /* Check if using the previous table will be beneficial */
614
+ if (repeat && *repeat != HUF_repeat_none) {
615
+ size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
616
+ size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue);
617
+ if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
618
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
619
+ }
620
+ }
621
+ /* Use the new table */
622
+ if (hSize + 12ul >= srcSize) { return 0; }
495
623
  op += hSize;
624
+ if (repeat) { *repeat = HUF_repeat_none; }
625
+ if (oldHufTable) { memcpy(oldHufTable, CTable, CTableSize); } /* Save the new table */
496
626
  }
627
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
628
+ }
497
629
 
498
- /* Compress */
499
- { size_t const cSize = (singleStream) ?
500
- HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */
501
- HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
502
- if (HUF_isError(cSize)) return cSize;
503
- if (cSize==0) return 0; /* uncompressible */
504
- op += cSize;
505
- }
506
-
507
- /* check compressibility */
508
- if ((size_t)(op-ostart) >= srcSize-1)
509
- return 0;
510
630
 
511
- return op-ostart;
631
+ size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
632
+ const void* src, size_t srcSize,
633
+ unsigned maxSymbolValue, unsigned huffLog,
634
+ void* workSpace, size_t wkspSize)
635
+ {
636
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
512
637
  }
513
638
 
639
+ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
640
+ const void* src, size_t srcSize,
641
+ unsigned maxSymbolValue, unsigned huffLog,
642
+ void* workSpace, size_t wkspSize,
643
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
644
+ {
645
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
646
+ }
514
647
 
515
648
  size_t HUF_compress1X (void* dst, size_t dstSize,
516
649
  const void* src, size_t srcSize,
517
650
  unsigned maxSymbolValue, unsigned huffLog)
518
651
  {
519
- return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1);
652
+ unsigned workSpace[1024];
653
+ return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
654
+ }
655
+
656
+ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
657
+ const void* src, size_t srcSize,
658
+ unsigned maxSymbolValue, unsigned huffLog,
659
+ void* workSpace, size_t wkspSize)
660
+ {
661
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
662
+ }
663
+
664
+ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
665
+ const void* src, size_t srcSize,
666
+ unsigned maxSymbolValue, unsigned huffLog,
667
+ void* workSpace, size_t wkspSize,
668
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
669
+ {
670
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
520
671
  }
521
672
 
522
673
  size_t HUF_compress2 (void* dst, size_t dstSize,
523
674
  const void* src, size_t srcSize,
524
675
  unsigned maxSymbolValue, unsigned huffLog)
525
676
  {
526
- return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
677
+ unsigned workSpace[1024];
678
+ return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
527
679
  }
528
680
 
529
-
530
681
  size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
531
682
  {
532
683
  return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);