zstd-ruby 1.4.4.0 → 1.4.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/ext/zstdruby/libzstd/Makefile +123 -58
  4. data/ext/zstdruby/libzstd/README.md +34 -14
  5. data/ext/zstdruby/libzstd/common/bitstream.h +31 -37
  6. data/ext/zstdruby/libzstd/common/compiler.h +19 -3
  7. data/ext/zstdruby/libzstd/common/cpu.h +1 -1
  8. data/ext/zstdruby/libzstd/common/debug.c +11 -31
  9. data/ext/zstdruby/libzstd/common/debug.h +11 -31
  10. data/ext/zstdruby/libzstd/common/entropy_common.c +13 -33
  11. data/ext/zstdruby/libzstd/common/error_private.c +2 -1
  12. data/ext/zstdruby/libzstd/common/error_private.h +6 -2
  13. data/ext/zstdruby/libzstd/common/fse.h +11 -31
  14. data/ext/zstdruby/libzstd/common/fse_decompress.c +12 -37
  15. data/ext/zstdruby/libzstd/common/huf.h +15 -33
  16. data/ext/zstdruby/libzstd/common/mem.h +1 -1
  17. data/ext/zstdruby/libzstd/common/pool.c +1 -1
  18. data/ext/zstdruby/libzstd/common/pool.h +2 -2
  19. data/ext/zstdruby/libzstd/common/threading.c +4 -3
  20. data/ext/zstdruby/libzstd/common/threading.h +4 -3
  21. data/ext/zstdruby/libzstd/common/xxhash.c +15 -33
  22. data/ext/zstdruby/libzstd/common/xxhash.h +11 -31
  23. data/ext/zstdruby/libzstd/common/zstd_common.c +1 -1
  24. data/ext/zstdruby/libzstd/common/zstd_errors.h +2 -1
  25. data/ext/zstdruby/libzstd/common/zstd_internal.h +112 -15
  26. data/ext/zstdruby/libzstd/compress/fse_compress.c +17 -40
  27. data/ext/zstdruby/libzstd/compress/hist.c +15 -35
  28. data/ext/zstdruby/libzstd/compress/hist.h +12 -32
  29. data/ext/zstdruby/libzstd/compress/huf_compress.c +92 -92
  30. data/ext/zstdruby/libzstd/compress/zstd_compress.c +450 -275
  31. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +136 -14
  32. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +10 -6
  33. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +1 -1
  34. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +24 -20
  35. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +10 -3
  36. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +845 -0
  37. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
  38. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +3 -13
  39. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +11 -8
  40. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
  41. data/ext/zstdruby/libzstd/compress/zstd_fast.c +36 -24
  42. data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
  43. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +34 -11
  44. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +1 -1
  45. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +27 -5
  46. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +7 -2
  47. data/ext/zstdruby/libzstd/compress/zstd_opt.c +38 -84
  48. data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
  49. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +48 -21
  50. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +2 -2
  51. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +76 -62
  52. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +12 -8
  53. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
  54. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +264 -148
  55. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +312 -203
  56. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +3 -3
  57. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +18 -4
  58. data/ext/zstdruby/libzstd/deprecated/zbuff.h +3 -3
  59. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
  60. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
  61. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
  62. data/ext/zstdruby/libzstd/dictBuilder/cover.c +5 -5
  63. data/ext/zstdruby/libzstd/dictBuilder/cover.h +14 -4
  64. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +14 -4
  65. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +33 -9
  66. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +51 -28
  67. data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
  68. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +4 -4
  69. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +18 -12
  70. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +1 -1
  71. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +10 -6
  72. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +1 -1
  73. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +10 -6
  74. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +1 -1
  75. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +13 -7
  76. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +1 -1
  77. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +17 -13
  78. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +2 -2
  79. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +17 -13
  80. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +1 -1
  81. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +22 -14
  82. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +1 -1
  83. data/ext/zstdruby/libzstd/libzstd.pc.in +2 -2
  84. data/ext/zstdruby/libzstd/zstd.h +62 -21
  85. data/lib/zstd-ruby/version.rb +1 -1
  86. metadata +7 -5
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -18,7 +18,7 @@
18
18
  /*-*************************************
19
19
  * Dependencies
20
20
  ***************************************/
21
- #include "zstd_internal.h"
21
+ #include "../common/zstd_internal.h"
22
22
  #include "zstd_cwksp.h"
23
23
  #ifdef ZSTD_MULTITHREAD
24
24
  # include "zstdmt_compress.h"
@@ -166,6 +166,7 @@ typedef struct {
166
166
  typedef struct {
167
167
  ZSTD_window_t window; /* State for the window round buffer management */
168
168
  ldmEntry_t* hashTable;
169
+ U32 loadedDictEnd;
169
170
  BYTE* bucketOffsets; /* Next position in bucket to insert entry */
170
171
  U64 hashPower; /* Used to compute the rolling hash.
171
172
  * Depends on ldmParams.minMatchLength */
@@ -249,6 +250,7 @@ struct ZSTD_CCtx_s {
249
250
  size_t staticSize;
250
251
  SeqCollector seqCollector;
251
252
  int isFirstBlock;
253
+ int initialized;
252
254
 
253
255
  seqStore_t seqStore; /* sequences storage ptrs */
254
256
  ldmState_t ldmState; /* long distance matching state */
@@ -324,6 +326,31 @@ MEM_STATIC U32 ZSTD_MLcode(U32 mlBase)
324
326
  return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
325
327
  }
326
328
 
329
+ typedef struct repcodes_s {
330
+ U32 rep[3];
331
+ } repcodes_t;
332
+
333
+ MEM_STATIC repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0)
334
+ {
335
+ repcodes_t newReps;
336
+ if (offset >= ZSTD_REP_NUM) { /* full offset */
337
+ newReps.rep[2] = rep[1];
338
+ newReps.rep[1] = rep[0];
339
+ newReps.rep[0] = offset - ZSTD_REP_MOVE;
340
+ } else { /* repcode */
341
+ U32 const repCode = offset + ll0;
342
+ if (repCode > 0) { /* note : if repCode==0, no change */
343
+ U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
344
+ newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2];
345
+ newReps.rep[1] = rep[0];
346
+ newReps.rep[0] = currentOffset;
347
+ } else { /* repCode == 0 */
348
+ memcpy(&newReps, rep, sizeof(newReps));
349
+ }
350
+ }
351
+ return newReps;
352
+ }
353
+
327
354
  /* ZSTD_cParam_withinBounds:
328
355
  * @return 1 if value is within cParam bounds,
329
356
  * 0 otherwise */
@@ -336,6 +363,30 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
336
363
  return 1;
337
364
  }
338
365
 
366
+ /* ZSTD_noCompressBlock() :
367
+ * Writes uncompressed block to dst buffer from given src.
368
+ * Returns the size of the block */
369
+ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
370
+ {
371
+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3);
372
+ RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity,
373
+ dstSize_tooSmall, "dst buf too small for uncompressed block");
374
+ MEM_writeLE24(dst, cBlockHeader24);
375
+ memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
376
+ return ZSTD_blockHeaderSize + srcSize;
377
+ }
378
+
379
+ MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock)
380
+ {
381
+ BYTE* const op = (BYTE*)dst;
382
+ U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3);
383
+ RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, "");
384
+ MEM_writeLE24(op, cBlockHeader);
385
+ op[3] = src;
386
+ return 4;
387
+ }
388
+
389
+
339
390
  /* ZSTD_minGain() :
340
391
  * minimum compression required
341
392
  * to generate a compress block or a compressed literals section.
@@ -348,6 +399,21 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
348
399
  return (srcSize >> minlog) + 2;
349
400
  }
350
401
 
402
+ MEM_STATIC int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams)
403
+ {
404
+ switch (cctxParams->literalCompressionMode) {
405
+ case ZSTD_lcm_huffman:
406
+ return 0;
407
+ case ZSTD_lcm_uncompressed:
408
+ return 1;
409
+ default:
410
+ assert(0 /* impossible: pre-validated */);
411
+ /* fall-through */
412
+ case ZSTD_lcm_auto:
413
+ return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
414
+ }
415
+ }
416
+
351
417
  /*! ZSTD_safecopyLiterals() :
352
418
  * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w.
353
419
  * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single
@@ -433,8 +499,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
433
499
  if (MEM_64bits()) {
434
500
  # if defined(_MSC_VER) && defined(_WIN64)
435
501
  unsigned long r = 0;
436
- _BitScanForward64( &r, (U64)val );
437
- return (unsigned)(r>>3);
502
+ return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0;
438
503
  # elif defined(__GNUC__) && (__GNUC__ >= 4)
439
504
  return (__builtin_ctzll((U64)val) >> 3);
440
505
  # else
@@ -451,8 +516,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
451
516
  } else { /* 32 bits */
452
517
  # if defined(_MSC_VER)
453
518
  unsigned long r=0;
454
- _BitScanForward( &r, (U32)val );
455
- return (unsigned)(r>>3);
519
+ return _BitScanForward( &r, (U32)val ) ? (unsigned)(r >> 3) : 0;
456
520
  # elif defined(__GNUC__) && (__GNUC__ >= 3)
457
521
  return (__builtin_ctz((U32)val) >> 3);
458
522
  # else
@@ -467,8 +531,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
467
531
  if (MEM_64bits()) {
468
532
  # if defined(_MSC_VER) && defined(_WIN64)
469
533
  unsigned long r = 0;
470
- _BitScanReverse64( &r, val );
471
- return (unsigned)(r>>3);
534
+ return _BitScanReverse64( &r, val ) ? (unsigned)(r >> 3) : 0;
472
535
  # elif defined(__GNUC__) && (__GNUC__ >= 4)
473
536
  return (__builtin_clzll(val) >> 3);
474
537
  # else
@@ -482,8 +545,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
482
545
  } else { /* 32 bits */
483
546
  # if defined(_MSC_VER)
484
547
  unsigned long r = 0;
485
- _BitScanReverse( &r, (unsigned long)val );
486
- return (unsigned)(r>>3);
548
+ return _BitScanReverse( &r, (unsigned long)val ) ? (unsigned)(r >> 3) : 0;
487
549
  # elif defined(__GNUC__) && (__GNUC__ >= 3)
488
550
  return (__builtin_clz((U32)val) >> 3);
489
551
  # else
@@ -730,7 +792,10 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
730
792
  */
731
793
  U32 const cycleMask = (1U << cycleLog) - 1;
732
794
  U32 const current = (U32)((BYTE const*)src - window->base);
733
- U32 const newCurrent = (current & cycleMask) + maxDist;
795
+ U32 const currentCycle0 = current & cycleMask;
796
+ /* Exclude zero so that newCurrent - maxDist >= 1. */
797
+ U32 const currentCycle1 = currentCycle0 == 0 ? (1U << cycleLog) : currentCycle0;
798
+ U32 const newCurrent = currentCycle1 + maxDist;
734
799
  U32 const correction = current - newCurrent;
735
800
  assert((maxDist & cycleMask) == 0);
736
801
  assert(current > newCurrent);
@@ -739,8 +804,17 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
739
804
 
740
805
  window->base += correction;
741
806
  window->dictBase += correction;
742
- window->lowLimit -= correction;
743
- window->dictLimit -= correction;
807
+ if (window->lowLimit <= correction) window->lowLimit = 1;
808
+ else window->lowLimit -= correction;
809
+ if (window->dictLimit <= correction) window->dictLimit = 1;
810
+ else window->dictLimit -= correction;
811
+
812
+ /* Ensure we can still reference the full window. */
813
+ assert(newCurrent >= maxDist);
814
+ assert(newCurrent - maxDist >= 1);
815
+ /* Ensure that lowLimit and dictLimit didn't underflow. */
816
+ assert(window->lowLimit <= newCurrent);
817
+ assert(window->dictLimit <= newCurrent);
744
818
 
745
819
  DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
746
820
  window->lowLimit);
@@ -844,6 +918,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
844
918
  } } }
845
919
  }
846
920
 
921
+ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
922
+ memset(window, 0, sizeof(*window));
923
+ window->base = (BYTE const*)"";
924
+ window->dictBase = (BYTE const*)"";
925
+ window->dictLimit = 1; /* start from 1, so that 1st position is valid */
926
+ window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
927
+ window->nextSrc = window->base + 1; /* see issue #1241 */
928
+ }
929
+
847
930
  /**
848
931
  * ZSTD_window_update():
849
932
  * Updates the window by appending [src, src + srcSize) to the window.
@@ -857,6 +940,10 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
857
940
  BYTE const* const ip = (BYTE const*)src;
858
941
  U32 contiguous = 1;
859
942
  DEBUGLOG(5, "ZSTD_window_update");
943
+ if (srcSize == 0)
944
+ return contiguous;
945
+ assert(window->base != NULL);
946
+ assert(window->dictBase != NULL);
860
947
  /* Check if blocks follow each other */
861
948
  if (src != window->nextSrc) {
862
949
  /* not contiguous */
@@ -867,7 +954,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
867
954
  window->dictLimit = (U32)distanceFromBase;
868
955
  window->dictBase = window->base;
869
956
  window->base = ip - distanceFromBase;
870
- // ms->nextToUpdate = window->dictLimit;
957
+ /* ms->nextToUpdate = window->dictLimit; */
871
958
  if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */
872
959
  contiguous = 0;
873
960
  }
@@ -883,6 +970,9 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
883
970
  return contiguous;
884
971
  }
885
972
 
973
+ /**
974
+ * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix.
975
+ */
886
976
  MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog)
887
977
  {
888
978
  U32 const maxDistance = 1U << windowLog;
@@ -893,6 +983,19 @@ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current
893
983
  return matchLowest;
894
984
  }
895
985
 
986
+ /**
987
+ * Returns the lowest allowed match index in the prefix.
988
+ */
989
+ MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog)
990
+ {
991
+ U32 const maxDistance = 1U << windowLog;
992
+ U32 const lowestValid = ms->window.dictLimit;
993
+ U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid;
994
+ U32 const isDictionary = (ms->loadedDictEnd != 0);
995
+ U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
996
+ return matchLowest;
997
+ }
998
+
896
999
 
897
1000
 
898
1001
  /* debug functions */
@@ -931,6 +1034,21 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
931
1034
  }
932
1035
  #endif
933
1036
 
1037
+ /* ===============================================================
1038
+ * Shared internal declarations
1039
+ * These prototypes may be called from sources not in lib/compress
1040
+ * =============================================================== */
1041
+
1042
+ /* ZSTD_loadCEntropy() :
1043
+ * dict : must point at beginning of a valid zstd dictionary.
1044
+ * return : size of dictionary header (size of magic number + dict ID + entropy tables)
1045
+ * assumptions : magic number supposed already checked
1046
+ * and dictSize >= 8 */
1047
+ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
1048
+ short* offcodeNCount, unsigned* offcodeMaxValue,
1049
+ const void* const dict, size_t dictSize);
1050
+
1051
+ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
934
1052
 
935
1053
  /* ==============================================================
936
1054
  * Private declarations
@@ -940,6 +1058,7 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
940
1058
  /* ZSTD_getCParamsFromCCtxParams() :
941
1059
  * cParams are built depending on compressionLevel, src size hints,
942
1060
  * LDM and manually set compression parameters.
1061
+ * Note: srcSizeHint == 0 means 0!
943
1062
  */
944
1063
  ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
945
1064
  const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize);
@@ -999,5 +1118,8 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity);
999
1118
  */
1000
1119
  size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq);
1001
1120
 
1121
+ /** ZSTD_cycleLog() :
1122
+ * condition for correct operation : hashLog > 1 */
1123
+ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
1002
1124
 
1003
1125
  #endif /* ZSTD_COMPRESS_H */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -18,7 +18,7 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
18
18
  BYTE* const ostart = (BYTE* const)dst;
19
19
  U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
20
20
 
21
- RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall);
21
+ RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
22
22
 
23
23
  switch(flSize)
24
24
  {
@@ -36,6 +36,7 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
36
36
  }
37
37
 
38
38
  memcpy(ostart + flSize, src, srcSize);
39
+ DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
39
40
  return srcSize + flSize;
40
41
  }
41
42
 
@@ -62,6 +63,7 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
62
63
  }
63
64
 
64
65
  ostart[flSize] = *(const BYTE*)src;
66
+ DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1);
65
67
  return flSize+1;
66
68
  }
67
69
 
@@ -80,8 +82,8 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
80
82
  symbolEncodingType_e hType = set_compressed;
81
83
  size_t cLitSize;
82
84
 
83
- DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i)",
84
- disableLiteralCompression);
85
+ DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)",
86
+ disableLiteralCompression, (U32)srcSize);
85
87
 
86
88
  /* Prepare nextEntropy assuming reusing the existing table */
87
89
  memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
@@ -102,14 +104,15 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
102
104
  cLitSize = singleStream ?
103
105
  HUF_compress1X_repeat(
104
106
  ostart+lhSize, dstCapacity-lhSize, src, srcSize,
105
- 255, 11, entropyWorkspace, entropyWorkspaceSize,
107
+ HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
106
108
  (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) :
107
109
  HUF_compress4X_repeat(
108
110
  ostart+lhSize, dstCapacity-lhSize, src, srcSize,
109
- 255, 11, entropyWorkspace, entropyWorkspaceSize,
111
+ HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
110
112
  (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2);
111
113
  if (repeat != HUF_repeat_none) {
112
114
  /* reused the existing table */
115
+ DEBUGLOG(5, "Reusing previous huffman table");
113
116
  hType = set_repeat;
114
117
  }
115
118
  }
@@ -150,5 +153,6 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
150
153
  default: /* not possible : lhSize is {3,4,5} */
151
154
  assert(0);
152
155
  }
156
+ DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize));
153
157
  return lhSize+cLitSize;
154
158
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -60,7 +60,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
60
60
  BYTE wksp[FSE_NCOUNTBOUND];
61
61
  S16 norm[MaxSeq + 1];
62
62
  const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
63
- FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max));
63
+ FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max), "");
64
64
  return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
65
65
  }
66
66
 
@@ -86,7 +86,7 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
86
86
  * Returns the cost in bits of encoding the distribution in count using ctable.
87
87
  * Returns an error if ctable cannot represent all the symbols in count.
88
88
  */
89
- static size_t ZSTD_fseBitCost(
89
+ size_t ZSTD_fseBitCost(
90
90
  FSE_CTable const* ctable,
91
91
  unsigned const* count,
92
92
  unsigned const max)
@@ -96,18 +96,22 @@ static size_t ZSTD_fseBitCost(
96
96
  unsigned s;
97
97
  FSE_CState_t cstate;
98
98
  FSE_initCState(&cstate, ctable);
99
- RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC,
100
- "Repeat FSE_CTable has maxSymbolValue %u < %u",
99
+ if (ZSTD_getFSEMaxSymbolValue(ctable) < max) {
100
+ DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u",
101
101
  ZSTD_getFSEMaxSymbolValue(ctable), max);
102
+ return ERROR(GENERIC);
103
+ }
102
104
  for (s = 0; s <= max; ++s) {
103
105
  unsigned const tableLog = cstate.stateLog;
104
106
  unsigned const badCost = (tableLog + 1) << kAccuracyLog;
105
107
  unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog);
106
108
  if (count[s] == 0)
107
109
  continue;
108
- RETURN_ERROR_IF(bitCost >= badCost, GENERIC,
109
- "Repeat FSE_CTable has Prob[%u] == 0", s);
110
- cost += count[s] * bitCost;
110
+ if (bitCost >= badCost) {
111
+ DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s);
112
+ return ERROR(GENERIC);
113
+ }
114
+ cost += (size_t)count[s] * bitCost;
111
115
  }
112
116
  return cost >> kAccuracyLog;
113
117
  }
@@ -117,15 +121,15 @@ static size_t ZSTD_fseBitCost(
117
121
  * table described by norm. The max symbol support by norm is assumed >= max.
118
122
  * norm must be valid for every symbol with non-zero probability in count.
119
123
  */
120
- static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
121
- unsigned const* count, unsigned const max)
124
+ size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
125
+ unsigned const* count, unsigned const max)
122
126
  {
123
127
  unsigned const shift = 8 - accuracyLog;
124
128
  size_t cost = 0;
125
129
  unsigned s;
126
130
  assert(accuracyLog <= 8);
127
131
  for (s = 0; s <= max; ++s) {
128
- unsigned const normAcc = norm[s] != -1 ? norm[s] : 1;
132
+ unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1;
129
133
  unsigned const norm256 = normAcc << shift;
130
134
  assert(norm256 > 0);
131
135
  assert(norm256 < 256);
@@ -230,15 +234,15 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
230
234
 
231
235
  switch (type) {
232
236
  case set_rle:
233
- FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max));
234
- RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall);
237
+ FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), "");
238
+ RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space");
235
239
  *op = codeTable[0];
236
240
  return 1;
237
241
  case set_repeat:
238
242
  memcpy(nextCTable, prevCTable, prevCTableSize);
239
243
  return 0;
240
244
  case set_basic:
241
- FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */
245
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */
242
246
  return 0;
243
247
  case set_compressed: {
244
248
  S16 norm[MaxSeq + 1];
@@ -249,14 +253,14 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
249
253
  nbSeq_1--;
250
254
  }
251
255
  assert(nbSeq_1 > 1);
252
- FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
256
+ FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max), "");
253
257
  { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
254
- FORWARD_IF_ERROR(NCountSize);
255
- FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize));
258
+ FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed");
259
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize), "");
256
260
  return NCountSize;
257
261
  }
258
262
  }
259
- default: assert(0); RETURN_ERROR(GENERIC);
263
+ default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach");
260
264
  }
261
265
  }
262
266
 
@@ -290,7 +294,7 @@ ZSTD_encodeSequences_body(
290
294
  if (MEM_32bits()) BIT_flushBits(&blockStream);
291
295
  if (longOffsets) {
292
296
  U32 const ofBits = ofCodeTable[nbSeq-1];
293
- int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
297
+ unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
294
298
  if (extraBits) {
295
299
  BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits);
296
300
  BIT_flushBits(&blockStream);
@@ -327,7 +331,7 @@ ZSTD_encodeSequences_body(
327
331
  BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
328
332
  if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream);
329
333
  if (longOffsets) {
330
- int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
334
+ unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
331
335
  if (extraBits) {
332
336
  BIT_addBits(&blockStream, sequences[n].offset, extraBits);
333
337
  BIT_flushBits(&blockStream); /* (7)*/