zstd-ruby 1.5.5.0 → 1.5.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/README.md +2 -2
  4. data/ext/zstdruby/extconf.rb +2 -0
  5. data/ext/zstdruby/libzstd/common/allocations.h +1 -1
  6. data/ext/zstdruby/libzstd/common/bitstream.h +49 -29
  7. data/ext/zstdruby/libzstd/common/compiler.h +114 -22
  8. data/ext/zstdruby/libzstd/common/cpu.h +36 -0
  9. data/ext/zstdruby/libzstd/common/debug.c +6 -0
  10. data/ext/zstdruby/libzstd/common/debug.h +20 -11
  11. data/ext/zstdruby/libzstd/common/error_private.h +45 -36
  12. data/ext/zstdruby/libzstd/common/fse.h +3 -2
  13. data/ext/zstdruby/libzstd/common/fse_decompress.c +19 -17
  14. data/ext/zstdruby/libzstd/common/huf.h +14 -1
  15. data/ext/zstdruby/libzstd/common/mem.h +0 -9
  16. data/ext/zstdruby/libzstd/common/pool.c +1 -1
  17. data/ext/zstdruby/libzstd/common/pool.h +1 -1
  18. data/ext/zstdruby/libzstd/common/portability_macros.h +2 -0
  19. data/ext/zstdruby/libzstd/common/threading.c +8 -2
  20. data/ext/zstdruby/libzstd/common/xxhash.c +5 -11
  21. data/ext/zstdruby/libzstd/common/xxhash.h +2341 -1007
  22. data/ext/zstdruby/libzstd/common/zstd_internal.h +5 -5
  23. data/ext/zstdruby/libzstd/compress/fse_compress.c +8 -7
  24. data/ext/zstdruby/libzstd/compress/huf_compress.c +54 -25
  25. data/ext/zstdruby/libzstd/compress/zstd_compress.c +282 -161
  26. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +29 -27
  27. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +224 -113
  28. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +19 -13
  29. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +17 -5
  30. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +11 -0
  31. data/ext/zstdruby/libzstd/compress/zstd_fast.c +14 -6
  32. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +129 -87
  33. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +103 -28
  34. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +8 -2
  35. data/ext/zstdruby/libzstd/compress/zstd_opt.c +216 -112
  36. data/ext/zstdruby/libzstd/compress/zstd_opt.h +31 -7
  37. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +94 -79
  38. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +188 -126
  39. data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +38 -19
  40. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +84 -32
  41. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +231 -208
  42. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +1 -1
  43. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +2 -0
  44. data/ext/zstdruby/libzstd/dictBuilder/cover.c +16 -12
  45. data/ext/zstdruby/libzstd/dictBuilder/cover.h +2 -8
  46. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +2 -2
  47. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +12 -6
  48. data/ext/zstdruby/libzstd/zstd.h +129 -60
  49. data/ext/zstdruby/streaming_compress.c +23 -3
  50. data/ext/zstdruby/streaming_decompress.c +23 -3
  51. data/lib/zstd-ruby/version.rb +1 -1
  52. metadata +2 -2
@@ -178,7 +178,7 @@ static void ZSTD_copy8(void* dst, const void* src) {
178
178
  ZSTD_memcpy(dst, src, 8);
179
179
  #endif
180
180
  }
181
- #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
181
+ #define COPY8(d,s) do { ZSTD_copy8(d,s); d+=8; s+=8; } while (0)
182
182
 
183
183
  /* Need to use memmove here since the literal buffer can now be located within
184
184
  the dst buffer. In circumstances where the op "catches up" to where the
@@ -198,7 +198,7 @@ static void ZSTD_copy16(void* dst, const void* src) {
198
198
  ZSTD_memcpy(dst, copy16_buf, 16);
199
199
  #endif
200
200
  }
201
- #define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
201
+ #define COPY16(d,s) do { ZSTD_copy16(d,s); d+=16; s+=16; } while (0)
202
202
 
203
203
  #define WILDCOPY_OVERLENGTH 32
204
204
  #define WILDCOPY_VECLEN 16
@@ -227,7 +227,7 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
227
227
  if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
228
228
  /* Handle short offset copies. */
229
229
  do {
230
- COPY8(op, ip)
230
+ COPY8(op, ip);
231
231
  } while (op < oend);
232
232
  } else {
233
233
  assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN);
@@ -366,13 +366,13 @@ typedef struct {
366
366
 
367
367
  /*! ZSTD_getcBlockSize() :
368
368
  * Provides the size of compressed block from block header `src` */
369
- /* Used by: decompress, fullbench (does not get its definition from here) */
369
+ /* Used by: decompress, fullbench */
370
370
  size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
371
371
  blockProperties_t* bpPtr);
372
372
 
373
373
  /*! ZSTD_decodeSeqHeaders() :
374
374
  * decode sequence header from src */
375
- /* Used by: decompress, fullbench (does not get its definition from here) */
375
+ /* Used by: zstd_decompress_block, fullbench */
376
376
  size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
377
377
  const void* src, size_t srcSize);
378
378
 
@@ -25,7 +25,7 @@
25
25
  #include "../common/error_private.h"
26
26
  #define ZSTD_DEPS_NEED_MALLOC
27
27
  #define ZSTD_DEPS_NEED_MATH64
28
- #include "../common/zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */
28
+ #include "../common/zstd_deps.h" /* ZSTD_memset */
29
29
  #include "../common/bits.h" /* ZSTD_highbit32 */
30
30
 
31
31
 
@@ -225,8 +225,8 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
225
225
  size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog
226
226
  + 4 /* bitCount initialized at 4 */
227
227
  + 2 /* first two symbols may use one additional bit each */) / 8)
228
- + 1 /* round up to whole nb bytes */
229
- + 2 /* additional two bytes for bitstream flush */;
228
+ + 1 /* round up to whole nb bytes */
229
+ + 2 /* additional two bytes for bitstream flush */;
230
230
  return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
231
231
  }
232
232
 
@@ -255,7 +255,7 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
255
255
  /* Init */
256
256
  remaining = tableSize+1; /* +1 for extra accuracy */
257
257
  threshold = tableSize;
258
- nbBits = tableLog+1;
258
+ nbBits = (int)tableLog+1;
259
259
 
260
260
  while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */
261
261
  if (previousIs0) {
@@ -274,7 +274,7 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
274
274
  }
275
275
  while (symbol >= start+3) {
276
276
  start+=3;
277
- bitStream += 3 << bitCount;
277
+ bitStream += 3U << bitCount;
278
278
  bitCount += 2;
279
279
  }
280
280
  bitStream += (symbol-start) << bitCount;
@@ -294,7 +294,7 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
294
294
  count++; /* +1 for extra accuracy */
295
295
  if (count>=threshold)
296
296
  count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
297
- bitStream += count << bitCount;
297
+ bitStream += (U32)count << bitCount;
298
298
  bitCount += nbBits;
299
299
  bitCount -= (count<max);
300
300
  previousIs0 = (count==1);
@@ -322,7 +322,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize,
322
322
  out[1] = (BYTE)(bitStream>>8);
323
323
  out+= (bitCount+7) /8;
324
324
 
325
- return (out-ostart);
325
+ assert(out >= ostart);
326
+ return (size_t)(out-ostart);
326
327
  }
327
328
 
328
329
 
@@ -220,6 +220,25 @@ static void HUF_setValue(HUF_CElt* elt, size_t value)
220
220
  }
221
221
  }
222
222
 
223
+ HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable)
224
+ {
225
+ HUF_CTableHeader header;
226
+ ZSTD_memcpy(&header, ctable, sizeof(header));
227
+ return header;
228
+ }
229
+
230
+ static void HUF_writeCTableHeader(HUF_CElt* ctable, U32 tableLog, U32 maxSymbolValue)
231
+ {
232
+ HUF_CTableHeader header;
233
+ HUF_STATIC_ASSERT(sizeof(ctable[0]) == sizeof(header));
234
+ ZSTD_memset(&header, 0, sizeof(header));
235
+ assert(tableLog < 256);
236
+ header.tableLog = (BYTE)tableLog;
237
+ assert(maxSymbolValue < 256);
238
+ header.maxSymbolValue = (BYTE)maxSymbolValue;
239
+ ZSTD_memcpy(ctable, &header, sizeof(header));
240
+ }
241
+
223
242
  typedef struct {
224
243
  HUF_CompressWeightsWksp wksp;
225
244
  BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
@@ -237,6 +256,9 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
237
256
 
238
257
  HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp));
239
258
 
259
+ assert(HUF_readCTableHeader(CTable).maxSymbolValue == maxSymbolValue);
260
+ assert(HUF_readCTableHeader(CTable).tableLog == huffLog);
261
+
240
262
  /* check conditions */
241
263
  if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC);
242
264
  if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
@@ -283,7 +305,9 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
283
305
  if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
284
306
  if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall);
285
307
 
286
- CTable[0] = tableLog;
308
+ *maxSymbolValuePtr = nbSymbols - 1;
309
+
310
+ HUF_writeCTableHeader(CTable, tableLog, *maxSymbolValuePtr);
287
311
 
288
312
  /* Prepare base value per rank */
289
313
  { U32 n, nextRankStart = 0;
@@ -315,7 +339,6 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
315
339
  { U32 n; for (n=0; n<nbSymbols; n++) HUF_setValue(ct + n, valPerRank[HUF_getNbBits(ct[n])]++); }
316
340
  }
317
341
 
318
- *maxSymbolValuePtr = nbSymbols - 1;
319
342
  return readSize;
320
343
  }
321
344
 
@@ -323,6 +346,8 @@ U32 HUF_getNbBitsFromCTable(HUF_CElt const* CTable, U32 symbolValue)
323
346
  {
324
347
  const HUF_CElt* const ct = CTable + 1;
325
348
  assert(symbolValue <= HUF_SYMBOLVALUE_MAX);
349
+ if (symbolValue > HUF_readCTableHeader(CTable).maxSymbolValue)
350
+ return 0;
326
351
  return (U32)HUF_getNbBits(ct[symbolValue]);
327
352
  }
328
353
 
@@ -723,7 +748,8 @@ static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, i
723
748
  HUF_setNbBits(ct + huffNode[n].byte, huffNode[n].nbBits); /* push nbBits per symbol, symbol order */
724
749
  for (n=0; n<alphabetSize; n++)
725
750
  HUF_setValue(ct + n, valPerRank[HUF_getNbBits(ct[n])]++); /* assign value within rank, symbol order */
726
- CTable[0] = maxNbBits;
751
+
752
+ HUF_writeCTableHeader(CTable, maxNbBits, maxSymbolValue);
727
753
  }
728
754
 
729
755
  size_t
@@ -776,13 +802,20 @@ size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count,
776
802
  }
777
803
 
778
804
  int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
779
- HUF_CElt const* ct = CTable + 1;
780
- int bad = 0;
781
- int s;
782
- for (s = 0; s <= (int)maxSymbolValue; ++s) {
783
- bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0);
784
- }
785
- return !bad;
805
+ HUF_CTableHeader header = HUF_readCTableHeader(CTable);
806
+ HUF_CElt const* ct = CTable + 1;
807
+ int bad = 0;
808
+ int s;
809
+
810
+ assert(header.tableLog <= HUF_TABLELOG_ABSOLUTEMAX);
811
+
812
+ if (header.maxSymbolValue < maxSymbolValue)
813
+ return 0;
814
+
815
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
816
+ bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0);
817
+ }
818
+ return !bad;
786
819
  }
787
820
 
788
821
  size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
@@ -1024,17 +1057,17 @@ HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize,
1024
1057
  const void* src, size_t srcSize,
1025
1058
  const HUF_CElt* CTable)
1026
1059
  {
1027
- U32 const tableLog = (U32)CTable[0];
1060
+ U32 const tableLog = HUF_readCTableHeader(CTable).tableLog;
1028
1061
  HUF_CElt const* ct = CTable + 1;
1029
1062
  const BYTE* ip = (const BYTE*) src;
1030
1063
  BYTE* const ostart = (BYTE*)dst;
1031
1064
  BYTE* const oend = ostart + dstSize;
1032
- BYTE* op = ostart;
1033
1065
  HUF_CStream_t bitC;
1034
1066
 
1035
1067
  /* init */
1036
1068
  if (dstSize < 8) return 0; /* not enough space to compress */
1037
- { size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op));
1069
+ { BYTE* op = ostart;
1070
+ size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op));
1038
1071
  if (HUF_isError(initErr)) return 0; }
1039
1072
 
1040
1073
  if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11)
@@ -1255,7 +1288,7 @@ unsigned HUF_optimalTableLog(
1255
1288
 
1256
1289
  { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
1257
1290
  size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
1258
- size_t maxBits, hSize, newSize;
1291
+ size_t hSize, newSize;
1259
1292
  const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
1260
1293
  const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
1261
1294
  size_t optSize = ((size_t) ~0) - 1;
@@ -1266,12 +1299,14 @@ unsigned HUF_optimalTableLog(
1266
1299
  /* Search until size increases */
1267
1300
  for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
1268
1301
  DEBUGLOG(7, "checking for huffLog=%u", optLogGuess);
1269
- maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
1270
- if (ERR_isError(maxBits)) continue;
1271
1302
 
1272
- if (maxBits < optLogGuess && optLogGuess > minTableLog) break;
1303
+ { size_t maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
1304
+ if (ERR_isError(maxBits)) continue;
1305
+
1306
+ if (maxBits < optLogGuess && optLogGuess > minTableLog) break;
1273
1307
 
1274
- hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
1308
+ hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
1309
+ }
1275
1310
 
1276
1311
  if (ERR_isError(hSize)) continue;
1277
1312
 
@@ -1372,12 +1407,6 @@ HUF_compress_internal (void* dst, size_t dstSize,
1372
1407
  huffLog = (U32)maxBits;
1373
1408
  DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1));
1374
1409
  }
1375
- /* Zero unused symbols in CTable, so we can check it for validity */
1376
- {
1377
- size_t const ctableSize = HUF_CTABLE_SIZE_ST(maxSymbolValue);
1378
- size_t const unusedSize = sizeof(table->CTable) - ctableSize * sizeof(HUF_CElt);
1379
- ZSTD_memset(table->CTable + ctableSize, 0, unusedSize);
1380
- }
1381
1410
 
1382
1411
  /* Write table description header */
1383
1412
  { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog,
@@ -1420,7 +1449,7 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
1420
1449
  /* HUF_compress4X_repeat():
1421
1450
  * compress input using 4 streams.
1422
1451
  * consider skipping quickly
1423
- * re-use an existing huffman compression table */
1452
+ * reuse an existing huffman compression table */
1424
1453
  size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
1425
1454
  const void* src, size_t srcSize,
1426
1455
  unsigned maxSymbolValue, unsigned huffLog,