zstdlib 0.6.0-x86-mingw32 → 0.7.0-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +5 -0
  3. data/ext/zstdlib/extconf.rb +1 -1
  4. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/bitstream.h +31 -37
  5. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/compiler.h +19 -3
  6. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/cpu.h +1 -1
  7. data/ext/zstdlib/zstd-1.4.5/lib/common/debug.c +24 -0
  8. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/debug.h +11 -31
  9. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/entropy_common.c +13 -33
  10. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/error_private.c +2 -1
  11. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/error_private.h +6 -2
  12. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/fse.h +11 -31
  13. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/fse_decompress.c +12 -37
  14. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/huf.h +15 -33
  15. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/mem.h +1 -1
  16. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/pool.c +1 -1
  17. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/pool.h +2 -2
  18. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/threading.c +4 -3
  19. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/threading.h +4 -3
  20. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/xxhash.c +15 -33
  21. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/xxhash.h +11 -31
  22. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_common.c +1 -1
  23. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_errors.h +2 -1
  24. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_internal.h +112 -15
  25. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/fse_compress.c +17 -40
  26. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/hist.c +15 -35
  27. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/hist.h +12 -32
  28. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/huf_compress.c +92 -92
  29. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress.c +450 -275
  30. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_internal.h +136 -14
  31. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_literals.c +10 -6
  32. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_literals.h +1 -1
  33. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_sequences.c +24 -20
  34. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_sequences.h +10 -3
  35. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_compress_superblock.c +845 -0
  36. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_compress_superblock.h +32 -0
  37. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_cwksp.h +3 -13
  38. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_double_fast.c +11 -8
  39. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_double_fast.h +2 -2
  40. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_fast.c +36 -24
  41. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_fast.h +2 -2
  42. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_lazy.c +34 -11
  43. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_lazy.h +1 -1
  44. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_ldm.c +27 -5
  45. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_ldm.h +7 -2
  46. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_opt.c +38 -84
  47. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_opt.h +1 -1
  48. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstdmt_compress.c +48 -21
  49. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstdmt_compress.h +2 -2
  50. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/huf_decompress.c +76 -62
  51. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_ddict.c +12 -8
  52. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_ddict.h +2 -2
  53. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress.c +264 -148
  54. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_block.c +312 -203
  55. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_block.h +3 -3
  56. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_internal.h +18 -4
  57. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/zstd.h +62 -21
  58. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzclose.c +0 -0
  59. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzcompatibility.h +1 -1
  60. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzguts.h +0 -0
  61. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzlib.c +0 -0
  62. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzread.c +0 -0
  63. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzwrite.c +0 -0
  64. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/zstd_zlibwrapper.c +1 -1
  65. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  66. data/lib/2.2/zstdlib.so +0 -0
  67. data/lib/2.3/zstdlib.so +0 -0
  68. data/lib/2.4/zstdlib.so +0 -0
  69. data/lib/2.5/zstdlib.so +0 -0
  70. data/lib/2.6/zstdlib.so +0 -0
  71. data/lib/2.7/zstdlib.so +0 -0
  72. metadata +64 -62
  73. data/ext/zstdlib/zstd-1.4.4/lib/common/debug.c +0 -44
@@ -1,10 +1,11 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
6
6
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
7
  * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #ifndef ZSTD_LDM_H
@@ -15,7 +16,7 @@ extern "C" {
15
16
  #endif
16
17
 
17
18
  #include "zstd_compress_internal.h" /* ldmParams_t, U32 */
18
- #include "zstd.h" /* ZSTD_CCtx, size_t */
19
+ #include "../zstd.h" /* ZSTD_CCtx, size_t */
19
20
 
20
21
  /*-*************************************
21
22
  * Long distance matching
@@ -23,6 +24,10 @@ extern "C" {
23
24
 
24
25
  #define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
25
26
 
27
+ void ZSTD_ldm_fillHashTable(
28
+ ldmState_t* state, const BYTE* ip,
29
+ const BYTE* iend, ldmParams_t const* params);
30
+
26
31
  /**
27
32
  * ZSTD_ldm_generateSequences():
28
33
  *
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -249,40 +249,6 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
249
249
  }
250
250
  }
251
251
 
252
- /* ZSTD_litLengthContribution() :
253
- * @return ( cost(litlength) - cost(0) )
254
- * this value can then be added to rawLiteralsCost()
255
- * to provide a cost which is directly comparable to a match ending at same position */
256
- static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr, int optLevel)
257
- {
258
- if (optPtr->priceType >= zop_predef) return (int)WEIGHT(litLength, optLevel);
259
-
260
- /* dynamic statistics */
261
- { U32 const llCode = ZSTD_LLcode(litLength);
262
- int const contribution = (int)(LL_bits[llCode] * BITCOST_MULTIPLIER)
263
- + (int)WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */
264
- - (int)WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
265
- #if 1
266
- return contribution;
267
- #else
268
- return MAX(0, contribution); /* sometimes better, sometimes not ... */
269
- #endif
270
- }
271
- }
272
-
273
- /* ZSTD_literalsContribution() :
274
- * creates a fake cost for the literals part of a sequence
275
- * which can be compared to the ending cost of a match
276
- * should a new match start at this position */
277
- static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLength,
278
- const optState_t* const optPtr,
279
- int optLevel)
280
- {
281
- int const contribution = (int)ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel)
282
- + ZSTD_litLengthContribution(litLength, optPtr, optLevel);
283
- return contribution;
284
- }
285
-
286
252
  /* ZSTD_getMatchPrice() :
287
253
  * Provides the cost of the match part (offset + matchLength) of a sequence
288
254
  * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
@@ -603,7 +569,10 @@ U32 ZSTD_insertBtAndGetAllMatches (
603
569
  U32 repLen = 0;
604
570
  assert(current >= dictLimit);
605
571
  if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < current-dictLimit) { /* equivalent to `current > repIndex >= dictLimit` */
606
- if (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch)) {
572
+ /* We must validate the repcode offset because when we're using a dictionary the
573
+ * valid offset range shrinks when the dictionary goes out of bounds.
574
+ */
575
+ if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) {
607
576
  repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch;
608
577
  }
609
578
  } else { /* repIndex < dictLimit || repIndex >= current */
@@ -799,30 +768,6 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
799
768
  /*-*******************************
800
769
  * Optimal parser
801
770
  *********************************/
802
- typedef struct repcodes_s {
803
- U32 rep[3];
804
- } repcodes_t;
805
-
806
- static repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0)
807
- {
808
- repcodes_t newReps;
809
- if (offset >= ZSTD_REP_NUM) { /* full offset */
810
- newReps.rep[2] = rep[1];
811
- newReps.rep[1] = rep[0];
812
- newReps.rep[0] = offset - ZSTD_REP_MOVE;
813
- } else { /* repcode */
814
- U32 const repCode = offset + ll0;
815
- if (repCode > 0) { /* note : if repCode==0, no change */
816
- U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
817
- newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2];
818
- newReps.rep[1] = rep[0];
819
- newReps.rep[0] = currentOffset;
820
- } else { /* repCode == 0 */
821
- memcpy(&newReps, rep, sizeof(newReps));
822
- }
823
- }
824
- return newReps;
825
- }
826
771
 
827
772
 
828
773
  static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
@@ -839,7 +784,7 @@ listStats(const U32* table, int lastEltID)
839
784
  int enb;
840
785
  for (enb=0; enb < nbElts; enb++) {
841
786
  (void)table;
842
- //RAWLOG(2, "%3i:%3i, ", enb, table[enb]);
787
+ /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */
843
788
  RAWLOG(2, "%4i,", table[enb]);
844
789
  }
845
790
  RAWLOG(2, " \n");
@@ -894,7 +839,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
894
839
  { U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
895
840
  opt[0].mlen = 0; /* means is_a_literal */
896
841
  opt[0].litlen = litlen;
897
- opt[0].price = ZSTD_literalsContribution(anchor, litlen, optStatePtr, optLevel);
842
+ /* We don't need to include the actual price of the literals because
843
+ * it is static for the duration of the forward pass, and is included
844
+ * in every price. We include the literal length to avoid negative
845
+ * prices when we subtract the previous literal length.
846
+ */
847
+ opt[0].price = ZSTD_litLengthPrice(litlen, optStatePtr, optLevel);
898
848
 
899
849
  /* large match -> immediate encoding */
900
850
  { U32 const maxML = matches[nbMatches-1].len;
@@ -923,7 +873,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
923
873
  for (matchNb = 0; matchNb < nbMatches; matchNb++) {
924
874
  U32 const offset = matches[matchNb].off;
925
875
  U32 const end = matches[matchNb].len;
926
- repcodes_t const repHistory = ZSTD_updateRep(rep, offset, ll0);
927
876
  for ( ; pos <= end ; pos++ ) {
928
877
  U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel);
929
878
  U32 const sequencePrice = literalsPrice + matchPrice;
@@ -933,8 +882,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
933
882
  opt[pos].off = offset;
934
883
  opt[pos].litlen = litlen;
935
884
  opt[pos].price = sequencePrice;
936
- ZSTD_STATIC_ASSERT(sizeof(opt[pos].rep) == sizeof(repHistory));
937
- memcpy(opt[pos].rep, &repHistory, sizeof(repHistory));
938
885
  } }
939
886
  last_pos = pos-1;
940
887
  }
@@ -961,7 +908,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
961
908
  opt[cur].off = 0;
962
909
  opt[cur].litlen = litlen;
963
910
  opt[cur].price = price;
964
- memcpy(opt[cur].rep, opt[cur-1].rep, sizeof(opt[cur].rep));
965
911
  } else {
966
912
  DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)",
967
913
  inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price),
@@ -969,6 +915,21 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
969
915
  }
970
916
  }
971
917
 
918
+ /* Set the repcodes of the current position. We must do it here
919
+ * because we rely on the repcodes of the 2nd to last sequence being
920
+ * correct to set the next chunks repcodes during the backward
921
+ * traversal.
922
+ */
923
+ ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t));
924
+ assert(cur >= opt[cur].mlen);
925
+ if (opt[cur].mlen != 0) {
926
+ U32 const prev = cur - opt[cur].mlen;
927
+ repcodes_t newReps = ZSTD_updateRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0);
928
+ memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t));
929
+ } else {
930
+ memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t));
931
+ }
932
+
972
933
  /* last match must start at a minimum distance of 8 from oend */
973
934
  if (inr > ilimit) continue;
974
935
 
@@ -1009,7 +970,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
1009
970
  /* set prices using matches found at position == cur */
1010
971
  for (matchNb = 0; matchNb < nbMatches; matchNb++) {
1011
972
  U32 const offset = matches[matchNb].off;
1012
- repcodes_t const repHistory = ZSTD_updateRep(opt[cur].rep, offset, ll0);
1013
973
  U32 const lastML = matches[matchNb].len;
1014
974
  U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch;
1015
975
  U32 mlen;
@@ -1029,8 +989,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
1029
989
  opt[pos].off = offset;
1030
990
  opt[pos].litlen = litlen;
1031
991
  opt[pos].price = price;
1032
- ZSTD_STATIC_ASSERT(sizeof(opt[pos].rep) == sizeof(repHistory));
1033
- memcpy(opt[pos].rep, &repHistory, sizeof(repHistory));
1034
992
  } else {
1035
993
  DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)",
1036
994
  pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price));
@@ -1046,6 +1004,17 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
1046
1004
  _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
1047
1005
  assert(opt[0].mlen == 0);
1048
1006
 
1007
+ /* Set the next chunk's repcodes based on the repcodes of the beginning
1008
+ * of the last match, and the last sequence. This avoids us having to
1009
+ * update them while traversing the sequences.
1010
+ */
1011
+ if (lastSequence.mlen != 0) {
1012
+ repcodes_t reps = ZSTD_updateRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0);
1013
+ memcpy(rep, &reps, sizeof(reps));
1014
+ } else {
1015
+ memcpy(rep, opt[cur].rep, sizeof(repcodes_t));
1016
+ }
1017
+
1049
1018
  { U32 const storeEnd = cur + 1;
1050
1019
  U32 storeStart = storeEnd;
1051
1020
  U32 seqPos = cur;
@@ -1082,20 +1051,6 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
1082
1051
  continue; /* will finish */
1083
1052
  }
1084
1053
 
1085
- /* repcodes update : like ZSTD_updateRep(), but update in place */
1086
- if (offCode >= ZSTD_REP_NUM) { /* full offset */
1087
- rep[2] = rep[1];
1088
- rep[1] = rep[0];
1089
- rep[0] = offCode - ZSTD_REP_MOVE;
1090
- } else { /* repcode */
1091
- U32 const repCode = offCode + (llen==0);
1092
- if (repCode) { /* note : if repCode==0, no change */
1093
- U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
1094
- if (repCode >= 2) rep[2] = rep[1];
1095
- rep[1] = rep[0];
1096
- rep[0] = currentOffset;
1097
- } }
1098
-
1099
1054
  assert(anchor + llen <= iend);
1100
1055
  ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen);
1101
1056
  ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen-MINMATCH);
@@ -1104,7 +1059,6 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
1104
1059
  } }
1105
1060
  ZSTD_setBasePrices(optStatePtr, optLevel);
1106
1061
  }
1107
-
1108
1062
  } /* while (ip < ilimit) */
1109
1063
 
1110
1064
  /* Return the last literals size */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -22,9 +22,9 @@
22
22
  /* ====== Dependencies ====== */
23
23
  #include <string.h> /* memcpy, memset */
24
24
  #include <limits.h> /* INT_MAX, UINT_MAX */
25
- #include "mem.h" /* MEM_STATIC */
26
- #include "pool.h" /* threadpool */
27
- #include "threading.h" /* mutex */
25
+ #include "../common/mem.h" /* MEM_STATIC */
26
+ #include "../common/pool.h" /* threadpool */
27
+ #include "../common/threading.h" /* mutex */
28
28
  #include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
29
29
  #include "zstd_ldm.h"
30
30
  #include "zstdmt_compress.h"
@@ -461,7 +461,13 @@ typedef struct {
461
461
  ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */
462
462
  } serialState_t;
463
463
 
464
- static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool* seqPool, ZSTD_CCtx_params params, size_t jobSize)
464
+ static int
465
+ ZSTDMT_serialState_reset(serialState_t* serialState,
466
+ ZSTDMT_seqPool* seqPool,
467
+ ZSTD_CCtx_params params,
468
+ size_t jobSize,
469
+ const void* dict, size_t const dictSize,
470
+ ZSTD_dictContentType_e dictContentType)
465
471
  {
466
472
  /* Adjust parameters */
467
473
  if (params.ldmParams.enableLdm) {
@@ -490,8 +496,7 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
490
496
  /* Size the seq pool tables */
491
497
  ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize));
492
498
  /* Reset the window */
493
- ZSTD_window_clear(&serialState->ldmState.window);
494
- serialState->ldmWindow = serialState->ldmState.window;
499
+ ZSTD_window_init(&serialState->ldmState.window);
495
500
  /* Resize tables and output space if necessary. */
496
501
  if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) {
497
502
  ZSTD_free(serialState->ldmState.hashTable, cMem);
@@ -506,7 +511,24 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
506
511
  /* Zero the tables */
507
512
  memset(serialState->ldmState.hashTable, 0, hashSize);
508
513
  memset(serialState->ldmState.bucketOffsets, 0, bucketSize);
514
+
515
+ /* Update window state and fill hash table with dict */
516
+ serialState->ldmState.loadedDictEnd = 0;
517
+ if (dictSize > 0) {
518
+ if (dictContentType == ZSTD_dct_rawContent) {
519
+ BYTE const* const dictEnd = (const BYTE*)dict + dictSize;
520
+ ZSTD_window_update(&serialState->ldmState.window, dict, dictSize);
521
+ ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, &params.ldmParams);
522
+ serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base);
523
+ } else {
524
+ /* don't even load anything */
525
+ }
526
+ }
527
+
528
+ /* Initialize serialState's copy of ldmWindow. */
529
+ serialState->ldmWindow = serialState->ldmState.window;
509
530
  }
531
+
510
532
  serialState->params = params;
511
533
  serialState->params.jobSize = (U32)jobSize;
512
534
  return 0;
@@ -1054,7 +1076,7 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(const ZSTD_CCtx_params* params)
1054
1076
  static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers)
1055
1077
  {
1056
1078
  if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation);
1057
- FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) );
1079
+ FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , "");
1058
1080
  mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, nbWorkers);
1059
1081
  if (mtctx->bufPool == NULL) return ERROR(memory_allocation);
1060
1082
  mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers);
@@ -1076,7 +1098,7 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
1076
1098
  DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)",
1077
1099
  compressionLevel);
1078
1100
  mtctx->params.compressionLevel = compressionLevel;
1079
- { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, 0, 0);
1101
+ { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1080
1102
  cParams.windowLog = saved_wlog;
1081
1103
  mtctx->params.cParams = cParams;
1082
1104
  }
@@ -1235,7 +1257,8 @@ ZSTDMT_computeNbJobs(const ZSTD_CCtx_params* params, size_t srcSize, unsigned nb
1235
1257
  /* ZSTDMT_compress_advanced_internal() :
1236
1258
  * This is a blocking function : it will only give back control to caller after finishing its compression job.
1237
1259
  */
1238
- static size_t ZSTDMT_compress_advanced_internal(
1260
+ static size_t
1261
+ ZSTDMT_compress_advanced_internal(
1239
1262
  ZSTDMT_CCtx* mtctx,
1240
1263
  void* dst, size_t dstCapacity,
1241
1264
  const void* src, size_t srcSize,
@@ -1267,10 +1290,11 @@ static size_t ZSTDMT_compress_advanced_internal(
1267
1290
 
1268
1291
  assert(avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), required to compress directly into Dst (no additional buffer) */
1269
1292
  ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgJobSize) );
1270
- if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize))
1293
+ /* LDM doesn't even try to load the dictionary in single-ingestion mode */
1294
+ if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize, NULL, 0, ZSTD_dct_auto))
1271
1295
  return ERROR(memory_allocation);
1272
1296
 
1273
- FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) ); /* only expands if necessary */
1297
+ FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) , ""); /* only expands if necessary */
1274
1298
 
1275
1299
  { unsigned u;
1276
1300
  for (u=0; u<nbJobs; u++) {
@@ -1403,7 +1427,7 @@ size_t ZSTDMT_initCStream_internal(
1403
1427
 
1404
1428
  /* init */
1405
1429
  if (params.nbWorkers != mtctx->params.nbWorkers)
1406
- FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) );
1430
+ FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , "");
1407
1431
 
1408
1432
  if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
1409
1433
  if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX;
@@ -1500,7 +1524,8 @@ size_t ZSTDMT_initCStream_internal(
1500
1524
  mtctx->allJobsCompleted = 0;
1501
1525
  mtctx->consumed = 0;
1502
1526
  mtctx->produced = 0;
1503
- if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize))
1527
+ if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize,
1528
+ dict, dictSize, dictContentType))
1504
1529
  return ERROR(memory_allocation);
1505
1530
  return 0;
1506
1531
  }
@@ -1714,9 +1739,11 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
1714
1739
  assert(mtctx->doneJobID < mtctx->nextJobID);
1715
1740
  assert(cSize >= mtctx->jobs[wJobID].dstFlushed);
1716
1741
  assert(mtctx->jobs[wJobID].dstBuff.start != NULL);
1717
- memcpy((char*)output->dst + output->pos,
1718
- (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed,
1719
- toFlush);
1742
+ if (toFlush > 0) {
1743
+ memcpy((char*)output->dst + output->pos,
1744
+ (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed,
1745
+ toFlush);
1746
+ }
1720
1747
  output->pos += toFlush;
1721
1748
  mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */
1722
1749
 
@@ -1786,7 +1813,7 @@ static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range)
1786
1813
  BYTE const* const bufferStart = (BYTE const*)buffer.start;
1787
1814
  BYTE const* const bufferEnd = bufferStart + buffer.capacity;
1788
1815
  BYTE const* const rangeStart = (BYTE const*)range.start;
1789
- BYTE const* const rangeEnd = rangeStart + range.size;
1816
+ BYTE const* const rangeEnd = range.size != 0 ? rangeStart + range.size : rangeStart;
1790
1817
 
1791
1818
  if (rangeStart == NULL || bufferStart == NULL)
1792
1819
  return 0;
@@ -2060,7 +2087,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
2060
2087
  || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */
2061
2088
  size_t const jobSize = mtctx->inBuff.filled;
2062
2089
  assert(mtctx->inBuff.filled <= mtctx->targetSectionSize);
2063
- FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) );
2090
+ FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , "");
2064
2091
  }
2065
2092
 
2066
2093
  /* check for potential compressed data ready to be flushed */
@@ -2074,7 +2101,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
2074
2101
 
2075
2102
  size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
2076
2103
  {
2077
- FORWARD_IF_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) );
2104
+ FORWARD_IF_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) , "");
2078
2105
 
2079
2106
  /* recommended next input size : fill current input buffer */
2080
2107
  return mtctx->targetSectionSize - mtctx->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */
@@ -2091,7 +2118,7 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* ou
2091
2118
  || ((endFrame==ZSTD_e_end) && !mtctx->frameEnded)) { /* need a last 0-size block to end frame */
2092
2119
  DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job (%u bytes, end:%u)",
2093
2120
  (U32)srcSize, (U32)endFrame);
2094
- FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) );
2121
+ FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) , "");
2095
2122
  }
2096
2123
 
2097
2124
  /* check if there is any data available to flush */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -40,7 +40,7 @@
40
40
  /* === Dependencies === */
41
41
  #include <stddef.h> /* size_t */
42
42
  #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
43
- #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
43
+ #include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
44
44
 
45
45
 
46
46
  /* === Constants === */