extzstd 0.1 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja +5 -0
  3. data/README.md +5 -5
  4. data/contrib/zstd/CONTRIBUTING.md +42 -0
  5. data/contrib/zstd/LICENSE-examples +11 -0
  6. data/contrib/zstd/Makefile +315 -0
  7. data/contrib/zstd/NEWS +261 -0
  8. data/contrib/zstd/PATENTS +33 -0
  9. data/contrib/zstd/README.md +121 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +178 -0
  12. data/contrib/zstd/circle.yml +75 -0
  13. data/contrib/zstd/lib/BUCK +186 -0
  14. data/contrib/zstd/lib/Makefile +163 -0
  15. data/contrib/zstd/lib/README.md +77 -0
  16. data/contrib/zstd/{common → lib/common}/bitstream.h +7 -4
  17. data/contrib/zstd/{common → lib/common}/entropy_common.c +19 -23
  18. data/contrib/zstd/{common → lib/common}/error_private.c +0 -0
  19. data/contrib/zstd/{common → lib/common}/error_private.h +0 -0
  20. data/contrib/zstd/{common → lib/common}/fse.h +94 -34
  21. data/contrib/zstd/{common → lib/common}/fse_decompress.c +18 -19
  22. data/contrib/zstd/{common → lib/common}/huf.h +52 -20
  23. data/contrib/zstd/{common → lib/common}/mem.h +17 -13
  24. data/contrib/zstd/lib/common/pool.c +194 -0
  25. data/contrib/zstd/lib/common/pool.h +56 -0
  26. data/contrib/zstd/lib/common/threading.c +80 -0
  27. data/contrib/zstd/lib/common/threading.h +104 -0
  28. data/contrib/zstd/{common → lib/common}/xxhash.c +3 -1
  29. data/contrib/zstd/{common → lib/common}/xxhash.h +11 -15
  30. data/contrib/zstd/{common → lib/common}/zstd_common.c +1 -11
  31. data/contrib/zstd/{common → lib/common}/zstd_errors.h +16 -2
  32. data/contrib/zstd/{common → lib/common}/zstd_internal.h +17 -1
  33. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +138 -91
  34. data/contrib/zstd/{compress → lib/compress}/huf_compress.c +218 -67
  35. data/contrib/zstd/{compress → lib/compress}/zstd_compress.c +231 -108
  36. data/contrib/zstd/{compress → lib/compress}/zstd_opt.h +44 -25
  37. data/contrib/zstd/lib/compress/zstdmt_compress.c +739 -0
  38. data/contrib/zstd/lib/compress/zstdmt_compress.h +78 -0
  39. data/contrib/zstd/{decompress → lib/decompress}/huf_decompress.c +28 -23
  40. data/contrib/zstd/{decompress → lib/decompress}/zstd_decompress.c +814 -176
  41. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +60 -39
  42. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  43. data/contrib/zstd/lib/deprecated/zbuff_compress.c +145 -0
  44. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +74 -0
  45. data/contrib/zstd/lib/dictBuilder/cover.c +1029 -0
  46. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +0 -0
  47. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  48. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +68 -18
  49. data/contrib/zstd/lib/dictBuilder/zdict.h +201 -0
  50. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +122 -7
  51. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +34 -3
  52. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +8 -0
  53. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +45 -12
  54. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +8 -0
  55. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +45 -12
  56. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +8 -0
  57. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +56 -33
  58. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +8 -0
  59. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +45 -18
  60. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +7 -0
  61. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +43 -16
  62. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +7 -0
  63. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +57 -23
  64. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +8 -0
  65. data/contrib/zstd/lib/libzstd.pc.in +14 -0
  66. data/contrib/zstd/{zstd.h → lib/zstd.h} +206 -71
  67. data/ext/depend +2 -0
  68. data/ext/extconf.rb +4 -4
  69. data/ext/extzstd.c +1 -1
  70. data/ext/zstd_common.c +5 -5
  71. data/ext/zstd_compress.c +3 -3
  72. data/ext/zstd_decompress.c +2 -2
  73. data/ext/zstd_dictbuilder.c +2 -2
  74. data/ext/zstd_legacy_v01.c +1 -1
  75. data/ext/zstd_legacy_v02.c +1 -1
  76. data/ext/zstd_legacy_v03.c +1 -1
  77. data/ext/zstd_legacy_v04.c +1 -1
  78. data/ext/zstd_legacy_v05.c +1 -1
  79. data/ext/zstd_legacy_v06.c +1 -1
  80. data/ext/zstd_legacy_v07.c +1 -1
  81. data/gemstub.rb +9 -5
  82. data/lib/extzstd/version.rb +1 -1
  83. metadata +73 -51
  84. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  85. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  86. data/contrib/zstd/dictBuilder/zdict.h +0 -111
@@ -56,6 +56,8 @@
56
56
  * Error Management
57
57
  ****************************************************************/
58
58
  #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
59
+ #define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f
60
+ #define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
59
61
 
60
62
 
61
63
  /* **************************************************************
@@ -70,31 +72,73 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
70
72
  /* *******************************************************
71
73
  * HUF : Huffman block compression
72
74
  *********************************************************/
75
+ /* HUF_compressWeights() :
76
+ * Same as FSE_compress(), but dedicated to huff0's weights compression.
77
+ * The use case needs much less stack memory.
78
+ * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
79
+ */
80
+ #define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
81
+ size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize)
82
+ {
83
+ BYTE* const ostart = (BYTE*) dst;
84
+ BYTE* op = ostart;
85
+ BYTE* const oend = ostart + dstSize;
86
+
87
+ U32 maxSymbolValue = HUF_TABLELOG_MAX;
88
+ U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
89
+
90
+ FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
91
+ BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
92
+
93
+ U32 count[HUF_TABLELOG_MAX+1];
94
+ S16 norm[HUF_TABLELOG_MAX+1];
95
+
96
+ /* init conditions */
97
+ if (wtSize <= 1) return 0; /* Not compressible */
98
+
99
+ /* Scan input and build symbol stats */
100
+ { CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) );
101
+ if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
102
+ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
103
+ }
104
+
105
+ tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
106
+ CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) );
107
+
108
+ /* Write table description header */
109
+ { CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
110
+ op += hSize;
111
+ }
112
+
113
+ /* Compress */
114
+ CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) );
115
+ { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) );
116
+ if (cSize == 0) return 0; /* not enough space for compressed data */
117
+ op += cSize;
118
+ }
119
+
120
+ return op-ostart;
121
+ }
122
+
123
+
73
124
  struct HUF_CElt_s {
74
125
  U16 val;
75
126
  BYTE nbBits;
76
127
  }; /* typedef'd to HUF_CElt within "huf.h" */
77
128
 
78
- typedef struct nodeElt_s {
79
- U32 count;
80
- U16 parent;
81
- BYTE byte;
82
- BYTE nbBits;
83
- } nodeElt;
84
-
85
129
  /*! HUF_writeCTable() :
86
- `CTable` : huffman tree to save, using huf representation.
130
+ `CTable` : Huffman tree to save, using huf representation.
87
131
  @return : size of saved CTable */
88
132
  size_t HUF_writeCTable (void* dst, size_t maxDstSize,
89
133
  const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
90
134
  {
91
- BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
135
+ BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
92
136
  BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
93
137
  BYTE* op = (BYTE*)dst;
94
138
  U32 n;
95
139
 
96
140
  /* check conditions */
97
- if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
141
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
98
142
 
99
143
  /* convert to weight */
100
144
  bitsToWeight[0] = 0;
@@ -103,38 +147,33 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
103
147
  for (n=0; n<maxSymbolValue; n++)
104
148
  huffWeight[n] = bitsToWeight[CTable[n].nbBits];
105
149
 
106
- { size_t const size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue);
107
- if (FSE_isError(size)) return size;
108
- if ((size>1) & (size < maxSymbolValue/2)) { /* FSE compressed */
109
- op[0] = (BYTE)size;
110
- return size+1;
111
- }
112
- }
150
+ /* attempt weights compression by FSE */
151
+ { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) );
152
+ if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
153
+ op[0] = (BYTE)hSize;
154
+ return hSize+1;
155
+ } }
113
156
 
114
- /* raw values */
115
- if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */
157
+ /* write raw values as 4-bits (max : 15) */
158
+ if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
116
159
  if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
117
160
  op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
118
- huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
161
+ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
119
162
  for (n=0; n<maxSymbolValue; n+=2)
120
163
  op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
121
164
  return ((maxSymbolValue+1)/2) + 1;
122
-
123
165
  }
124
166
 
125
167
 
126
168
  size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
127
169
  {
128
- BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
170
+ BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
129
171
  U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
130
172
  U32 tableLog = 0;
131
- size_t readSize;
132
173
  U32 nbSymbols = 0;
133
- /*memset(huffWeight, 0, sizeof(huffWeight));*/ /* is not necessary, even though some analyzer complain ... */
134
174
 
135
175
  /* get symbol weights */
136
- readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
137
- if (HUF_isError(readSize)) return readSize;
176
+ CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize));
138
177
 
139
178
  /* check result */
140
179
  if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
@@ -174,6 +213,13 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
174
213
  }
175
214
 
176
215
 
216
+ typedef struct nodeElt_s {
217
+ U32 count;
218
+ U16 parent;
219
+ BYTE byte;
220
+ BYTE nbBits;
221
+ } nodeElt;
222
+
177
223
  static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
178
224
  {
179
225
  const U32 largestBits = huffNode[lastNonNull].nbBits;
@@ -279,20 +325,26 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
279
325
  }
280
326
 
281
327
 
328
+ /** HUF_buildCTable_wksp() :
329
+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
330
+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
331
+ */
282
332
  #define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
283
- size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
333
+ typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1];
334
+ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
284
335
  {
285
- nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1];
286
- nodeElt* huffNode = huffNode0 + 1;
336
+ nodeElt* const huffNode0 = (nodeElt*)workSpace;
337
+ nodeElt* const huffNode = huffNode0+1;
287
338
  U32 n, nonNullRank;
288
339
  int lowS, lowN;
289
340
  U16 nodeNb = STARTNODE;
290
341
  U32 nodeRoot;
291
342
 
292
343
  /* safety checks */
344
+ if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */
293
345
  if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
294
346
  if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
295
- memset(huffNode0, 0, sizeof(huffNode0));
347
+ memset(huffNode0, 0, sizeof(huffNodeTable));
296
348
 
297
349
  /* sort, decreasing order */
298
350
  HUF_sort(huffNode, count, maxSymbolValue);
@@ -305,7 +357,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
305
357
  huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
306
358
  nodeNb++; lowS-=2;
307
359
  for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
308
- huffNode0[0].count = (U32)(1U<<31);
360
+ huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */
309
361
 
310
362
  /* create parents */
311
363
  while (nodeNb <= nodeRoot) {
@@ -348,6 +400,34 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
348
400
  return maxNbBits;
349
401
  }
350
402
 
403
+ /** HUF_buildCTable() :
404
+ * Note : count is used before tree is written, so they can safely overlap
405
+ */
406
+ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
407
+ {
408
+ huffNodeTable nodeTable;
409
+ return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
410
+ }
411
+
412
+ static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
413
+ {
414
+ size_t nbBits = 0;
415
+ int s;
416
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
417
+ nbBits += CTable[s].nbBits * count[s];
418
+ }
419
+ return nbBits >> 3;
420
+ }
421
+
422
+ static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
423
+ int bad = 0;
424
+ int s;
425
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
426
+ bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
427
+ }
428
+ return !bad;
429
+ }
430
+
351
431
  static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
352
432
  {
353
433
  BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
@@ -375,8 +455,8 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
375
455
 
376
456
  /* init */
377
457
  if (dstSize < 8) return 0; /* not enough space to compress */
378
- { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op);
379
- if (HUF_isError(errorCode)) return 0; }
458
+ { size_t const initErr = BIT_initCStream(&bitC, op, oend-op);
459
+ if (HUF_isError(initErr)) return 0; }
380
460
 
381
461
  n = srcSize & ~3; /* join to mod 4 */
382
462
  switch (srcSize & 3)
@@ -419,32 +499,28 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
419
499
  if (srcSize < 12) return 0; /* no saving possible : too small input */
420
500
  op += 6; /* jumpTable */
421
501
 
422
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
423
- if (HUF_isError(cSize)) return cSize;
502
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
424
503
  if (cSize==0) return 0;
425
504
  MEM_writeLE16(ostart, (U16)cSize);
426
505
  op += cSize;
427
506
  }
428
507
 
429
508
  ip += segmentSize;
430
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
431
- if (HUF_isError(cSize)) return cSize;
509
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
432
510
  if (cSize==0) return 0;
433
511
  MEM_writeLE16(ostart+2, (U16)cSize);
434
512
  op += cSize;
435
513
  }
436
514
 
437
515
  ip += segmentSize;
438
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
439
- if (HUF_isError(cSize)) return cSize;
516
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
440
517
  if (cSize==0) return 0;
441
518
  MEM_writeLE16(ostart+4, (U16)cSize);
442
519
  op += cSize;
443
520
  }
444
521
 
445
522
  ip += segmentSize;
446
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
447
- if (HUF_isError(cSize)) return cSize;
523
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) );
448
524
  if (cSize==0) return 0;
449
525
  op += cSize;
450
526
  }
@@ -453,20 +529,43 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
453
529
  }
454
530
 
455
531
 
532
+ static size_t HUF_compressCTable_internal(
533
+ BYTE* const ostart, BYTE* op, BYTE* const oend,
534
+ const void* src, size_t srcSize,
535
+ unsigned singleStream, const HUF_CElt* CTable)
536
+ {
537
+ size_t const cSize = singleStream ?
538
+ HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) :
539
+ HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
540
+ if (HUF_isError(cSize)) { return cSize; }
541
+ if (cSize==0) { return 0; } /* uncompressible */
542
+ op += cSize;
543
+ /* check compressibility */
544
+ if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
545
+ return op-ostart;
546
+ }
547
+
548
+
549
+ /* `workSpace` must a table of at least 1024 unsigned */
456
550
  static size_t HUF_compress_internal (
457
551
  void* dst, size_t dstSize,
458
552
  const void* src, size_t srcSize,
459
553
  unsigned maxSymbolValue, unsigned huffLog,
460
- unsigned singleStream)
554
+ unsigned singleStream,
555
+ void* workSpace, size_t wkspSize,
556
+ HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat)
461
557
  {
462
558
  BYTE* const ostart = (BYTE*)dst;
463
559
  BYTE* const oend = ostart + dstSize;
464
560
  BYTE* op = ostart;
465
561
 
466
- U32 count[HUF_SYMBOLVALUE_MAX+1];
467
- HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
562
+ U32* count;
563
+ size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
564
+ HUF_CElt* CTable;
565
+ size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
468
566
 
469
567
  /* checks & inits */
568
+ if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) return ERROR(GENERIC);
470
569
  if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
471
570
  if (!dstSize) return 0; /* cannot fit within dst budget */
472
571
  if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
@@ -474,59 +573,111 @@ static size_t HUF_compress_internal (
474
573
  if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
475
574
  if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
476
575
 
576
+ count = (U32*)workSpace;
577
+ workSpace = (BYTE*)workSpace + countSize;
578
+ wkspSize -= countSize;
579
+ CTable = (HUF_CElt*)workSpace;
580
+ workSpace = (BYTE*)workSpace + CTableSize;
581
+ wkspSize -= CTableSize;
582
+
583
+ /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
584
+ if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
585
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
586
+ }
587
+
477
588
  /* Scan input and build symbol stats */
478
- { size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
479
- if (HUF_isError(largest)) return largest;
589
+ { CHECK_V_F(largest, FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
480
590
  if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
481
591
  if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
482
592
  }
483
593
 
594
+ /* Check validity of previous table */
595
+ if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) {
596
+ *repeat = HUF_repeat_none;
597
+ }
598
+ /* Heuristic : use existing table for small inputs */
599
+ if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
600
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
601
+ }
602
+
484
603
  /* Build Huffman Tree */
485
604
  huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
486
- { size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
487
- if (HUF_isError(maxBits)) return maxBits;
605
+ { CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) );
488
606
  huffLog = (U32)maxBits;
607
+ /* Zero the unused symbols so we can check it for validity */
608
+ memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
489
609
  }
490
610
 
491
611
  /* Write table description header */
492
- { size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
493
- if (HUF_isError(hSize)) return hSize;
494
- if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */
612
+ { CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog) );
613
+ /* Check if using the previous table will be beneficial */
614
+ if (repeat && *repeat != HUF_repeat_none) {
615
+ size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
616
+ size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue);
617
+ if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
618
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
619
+ }
620
+ }
621
+ /* Use the new table */
622
+ if (hSize + 12ul >= srcSize) { return 0; }
495
623
  op += hSize;
624
+ if (repeat) { *repeat = HUF_repeat_none; }
625
+ if (oldHufTable) { memcpy(oldHufTable, CTable, CTableSize); } /* Save the new table */
496
626
  }
627
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
628
+ }
497
629
 
498
- /* Compress */
499
- { size_t const cSize = (singleStream) ?
500
- HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */
501
- HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
502
- if (HUF_isError(cSize)) return cSize;
503
- if (cSize==0) return 0; /* uncompressible */
504
- op += cSize;
505
- }
506
-
507
- /* check compressibility */
508
- if ((size_t)(op-ostart) >= srcSize-1)
509
- return 0;
510
630
 
511
- return op-ostart;
631
+ size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
632
+ const void* src, size_t srcSize,
633
+ unsigned maxSymbolValue, unsigned huffLog,
634
+ void* workSpace, size_t wkspSize)
635
+ {
636
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
512
637
  }
513
638
 
639
+ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
640
+ const void* src, size_t srcSize,
641
+ unsigned maxSymbolValue, unsigned huffLog,
642
+ void* workSpace, size_t wkspSize,
643
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
644
+ {
645
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
646
+ }
514
647
 
515
648
  size_t HUF_compress1X (void* dst, size_t dstSize,
516
649
  const void* src, size_t srcSize,
517
650
  unsigned maxSymbolValue, unsigned huffLog)
518
651
  {
519
- return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1);
652
+ unsigned workSpace[1024];
653
+ return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
654
+ }
655
+
656
+ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
657
+ const void* src, size_t srcSize,
658
+ unsigned maxSymbolValue, unsigned huffLog,
659
+ void* workSpace, size_t wkspSize)
660
+ {
661
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
662
+ }
663
+
664
+ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
665
+ const void* src, size_t srcSize,
666
+ unsigned maxSymbolValue, unsigned huffLog,
667
+ void* workSpace, size_t wkspSize,
668
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
669
+ {
670
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
520
671
  }
521
672
 
522
673
  size_t HUF_compress2 (void* dst, size_t dstSize,
523
674
  const void* src, size_t srcSize,
524
675
  unsigned maxSymbolValue, unsigned huffLog)
525
676
  {
526
- return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
677
+ unsigned workSpace[1024];
678
+ return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
527
679
  }
528
680
 
529
-
530
681
  size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
531
682
  {
532
683
  return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);