zstdlib 0.6.0-x86-mingw32 → 0.7.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +5 -0
  3. data/ext/zstdlib/extconf.rb +1 -1
  4. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/bitstream.h +31 -37
  5. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/compiler.h +19 -3
  6. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/cpu.h +1 -1
  7. data/ext/zstdlib/zstd-1.4.5/lib/common/debug.c +24 -0
  8. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/debug.h +11 -31
  9. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/entropy_common.c +13 -33
  10. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/error_private.c +2 -1
  11. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/error_private.h +6 -2
  12. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/fse.h +11 -31
  13. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/fse_decompress.c +12 -37
  14. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/huf.h +15 -33
  15. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/mem.h +1 -1
  16. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/pool.c +1 -1
  17. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/pool.h +2 -2
  18. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/threading.c +4 -3
  19. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/threading.h +4 -3
  20. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/xxhash.c +15 -33
  21. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/xxhash.h +11 -31
  22. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_common.c +1 -1
  23. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_errors.h +2 -1
  24. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_internal.h +112 -15
  25. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/fse_compress.c +17 -40
  26. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/hist.c +15 -35
  27. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/hist.h +12 -32
  28. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/huf_compress.c +92 -92
  29. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress.c +450 -275
  30. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_internal.h +136 -14
  31. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_literals.c +10 -6
  32. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_literals.h +1 -1
  33. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_sequences.c +24 -20
  34. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_sequences.h +10 -3
  35. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_compress_superblock.c +845 -0
  36. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_compress_superblock.h +32 -0
  37. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_cwksp.h +3 -13
  38. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_double_fast.c +11 -8
  39. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_double_fast.h +2 -2
  40. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_fast.c +36 -24
  41. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_fast.h +2 -2
  42. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_lazy.c +34 -11
  43. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_lazy.h +1 -1
  44. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_ldm.c +27 -5
  45. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_ldm.h +7 -2
  46. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_opt.c +38 -84
  47. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_opt.h +1 -1
  48. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstdmt_compress.c +48 -21
  49. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstdmt_compress.h +2 -2
  50. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/huf_decompress.c +76 -62
  51. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_ddict.c +12 -8
  52. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_ddict.h +2 -2
  53. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress.c +264 -148
  54. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_block.c +312 -203
  55. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_block.h +3 -3
  56. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_internal.h +18 -4
  57. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/zstd.h +62 -21
  58. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzclose.c +0 -0
  59. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzcompatibility.h +1 -1
  60. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzguts.h +0 -0
  61. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzlib.c +0 -0
  62. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzread.c +0 -0
  63. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzwrite.c +0 -0
  64. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/zstd_zlibwrapper.c +1 -1
  65. data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  66. data/lib/2.2/zstdlib.so +0 -0
  67. data/lib/2.3/zstdlib.so +0 -0
  68. data/lib/2.4/zstdlib.so +0 -0
  69. data/lib/2.5/zstdlib.so +0 -0
  70. data/lib/2.6/zstdlib.so +0 -0
  71. data/lib/2.7/zstdlib.so +0 -0
  72. metadata +64 -62
  73. data/ext/zstdlib/zstd-1.4.4/lib/common/debug.c +0 -44
@@ -1,10 +1,11 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
6
6
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
7
  * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #ifndef ZSTD_LDM_H
@@ -15,7 +16,7 @@ extern "C" {
15
16
  #endif
16
17
 
17
18
  #include "zstd_compress_internal.h" /* ldmParams_t, U32 */
18
- #include "zstd.h" /* ZSTD_CCtx, size_t */
19
+ #include "../zstd.h" /* ZSTD_CCtx, size_t */
19
20
 
20
21
  /*-*************************************
21
22
  * Long distance matching
@@ -23,6 +24,10 @@ extern "C" {
23
24
 
24
25
  #define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
25
26
 
27
+ void ZSTD_ldm_fillHashTable(
28
+ ldmState_t* state, const BYTE* ip,
29
+ const BYTE* iend, ldmParams_t const* params);
30
+
26
31
  /**
27
32
  * ZSTD_ldm_generateSequences():
28
33
  *
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -249,40 +249,6 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
249
249
  }
250
250
  }
251
251
 
252
- /* ZSTD_litLengthContribution() :
253
- * @return ( cost(litlength) - cost(0) )
254
- * this value can then be added to rawLiteralsCost()
255
- * to provide a cost which is directly comparable to a match ending at same position */
256
- static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr, int optLevel)
257
- {
258
- if (optPtr->priceType >= zop_predef) return (int)WEIGHT(litLength, optLevel);
259
-
260
- /* dynamic statistics */
261
- { U32 const llCode = ZSTD_LLcode(litLength);
262
- int const contribution = (int)(LL_bits[llCode] * BITCOST_MULTIPLIER)
263
- + (int)WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */
264
- - (int)WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
265
- #if 1
266
- return contribution;
267
- #else
268
- return MAX(0, contribution); /* sometimes better, sometimes not ... */
269
- #endif
270
- }
271
- }
272
-
273
- /* ZSTD_literalsContribution() :
274
- * creates a fake cost for the literals part of a sequence
275
- * which can be compared to the ending cost of a match
276
- * should a new match start at this position */
277
- static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLength,
278
- const optState_t* const optPtr,
279
- int optLevel)
280
- {
281
- int const contribution = (int)ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel)
282
- + ZSTD_litLengthContribution(litLength, optPtr, optLevel);
283
- return contribution;
284
- }
285
-
286
252
  /* ZSTD_getMatchPrice() :
287
253
  * Provides the cost of the match part (offset + matchLength) of a sequence
288
254
  * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
@@ -603,7 +569,10 @@ U32 ZSTD_insertBtAndGetAllMatches (
603
569
  U32 repLen = 0;
604
570
  assert(current >= dictLimit);
605
571
  if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < current-dictLimit) { /* equivalent to `current > repIndex >= dictLimit` */
606
- if (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch)) {
572
+ /* We must validate the repcode offset because when we're using a dictionary the
573
+ * valid offset range shrinks when the dictionary goes out of bounds.
574
+ */
575
+ if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) {
607
576
  repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch;
608
577
  }
609
578
  } else { /* repIndex < dictLimit || repIndex >= current */
@@ -799,30 +768,6 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
799
768
  /*-*******************************
800
769
  * Optimal parser
801
770
  *********************************/
802
- typedef struct repcodes_s {
803
- U32 rep[3];
804
- } repcodes_t;
805
-
806
- static repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0)
807
- {
808
- repcodes_t newReps;
809
- if (offset >= ZSTD_REP_NUM) { /* full offset */
810
- newReps.rep[2] = rep[1];
811
- newReps.rep[1] = rep[0];
812
- newReps.rep[0] = offset - ZSTD_REP_MOVE;
813
- } else { /* repcode */
814
- U32 const repCode = offset + ll0;
815
- if (repCode > 0) { /* note : if repCode==0, no change */
816
- U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
817
- newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2];
818
- newReps.rep[1] = rep[0];
819
- newReps.rep[0] = currentOffset;
820
- } else { /* repCode == 0 */
821
- memcpy(&newReps, rep, sizeof(newReps));
822
- }
823
- }
824
- return newReps;
825
- }
826
771
 
827
772
 
828
773
  static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
@@ -839,7 +784,7 @@ listStats(const U32* table, int lastEltID)
839
784
  int enb;
840
785
  for (enb=0; enb < nbElts; enb++) {
841
786
  (void)table;
842
- //RAWLOG(2, "%3i:%3i, ", enb, table[enb]);
787
+ /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */
843
788
  RAWLOG(2, "%4i,", table[enb]);
844
789
  }
845
790
  RAWLOG(2, " \n");
@@ -894,7 +839,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
894
839
  { U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
895
840
  opt[0].mlen = 0; /* means is_a_literal */
896
841
  opt[0].litlen = litlen;
897
- opt[0].price = ZSTD_literalsContribution(anchor, litlen, optStatePtr, optLevel);
842
+ /* We don't need to include the actual price of the literals because
843
+ * it is static for the duration of the forward pass, and is included
844
+ * in every price. We include the literal length to avoid negative
845
+ * prices when we subtract the previous literal length.
846
+ */
847
+ opt[0].price = ZSTD_litLengthPrice(litlen, optStatePtr, optLevel);
898
848
 
899
849
  /* large match -> immediate encoding */
900
850
  { U32 const maxML = matches[nbMatches-1].len;
@@ -923,7 +873,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
923
873
  for (matchNb = 0; matchNb < nbMatches; matchNb++) {
924
874
  U32 const offset = matches[matchNb].off;
925
875
  U32 const end = matches[matchNb].len;
926
- repcodes_t const repHistory = ZSTD_updateRep(rep, offset, ll0);
927
876
  for ( ; pos <= end ; pos++ ) {
928
877
  U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel);
929
878
  U32 const sequencePrice = literalsPrice + matchPrice;
@@ -933,8 +882,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
933
882
  opt[pos].off = offset;
934
883
  opt[pos].litlen = litlen;
935
884
  opt[pos].price = sequencePrice;
936
- ZSTD_STATIC_ASSERT(sizeof(opt[pos].rep) == sizeof(repHistory));
937
- memcpy(opt[pos].rep, &repHistory, sizeof(repHistory));
938
885
  } }
939
886
  last_pos = pos-1;
940
887
  }
@@ -961,7 +908,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
961
908
  opt[cur].off = 0;
962
909
  opt[cur].litlen = litlen;
963
910
  opt[cur].price = price;
964
- memcpy(opt[cur].rep, opt[cur-1].rep, sizeof(opt[cur].rep));
965
911
  } else {
966
912
  DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)",
967
913
  inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price),
@@ -969,6 +915,21 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
969
915
  }
970
916
  }
971
917
 
918
+ /* Set the repcodes of the current position. We must do it here
919
+ * because we rely on the repcodes of the 2nd to last sequence being
920
+ * correct to set the next chunks repcodes during the backward
921
+ * traversal.
922
+ */
923
+ ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t));
924
+ assert(cur >= opt[cur].mlen);
925
+ if (opt[cur].mlen != 0) {
926
+ U32 const prev = cur - opt[cur].mlen;
927
+ repcodes_t newReps = ZSTD_updateRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0);
928
+ memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t));
929
+ } else {
930
+ memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t));
931
+ }
932
+
972
933
  /* last match must start at a minimum distance of 8 from oend */
973
934
  if (inr > ilimit) continue;
974
935
 
@@ -1009,7 +970,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
1009
970
  /* set prices using matches found at position == cur */
1010
971
  for (matchNb = 0; matchNb < nbMatches; matchNb++) {
1011
972
  U32 const offset = matches[matchNb].off;
1012
- repcodes_t const repHistory = ZSTD_updateRep(opt[cur].rep, offset, ll0);
1013
973
  U32 const lastML = matches[matchNb].len;
1014
974
  U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch;
1015
975
  U32 mlen;
@@ -1029,8 +989,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
1029
989
  opt[pos].off = offset;
1030
990
  opt[pos].litlen = litlen;
1031
991
  opt[pos].price = price;
1032
- ZSTD_STATIC_ASSERT(sizeof(opt[pos].rep) == sizeof(repHistory));
1033
- memcpy(opt[pos].rep, &repHistory, sizeof(repHistory));
1034
992
  } else {
1035
993
  DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)",
1036
994
  pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price));
@@ -1046,6 +1004,17 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
1046
1004
  _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
1047
1005
  assert(opt[0].mlen == 0);
1048
1006
 
1007
+ /* Set the next chunk's repcodes based on the repcodes of the beginning
1008
+ * of the last match, and the last sequence. This avoids us having to
1009
+ * update them while traversing the sequences.
1010
+ */
1011
+ if (lastSequence.mlen != 0) {
1012
+ repcodes_t reps = ZSTD_updateRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0);
1013
+ memcpy(rep, &reps, sizeof(reps));
1014
+ } else {
1015
+ memcpy(rep, opt[cur].rep, sizeof(repcodes_t));
1016
+ }
1017
+
1049
1018
  { U32 const storeEnd = cur + 1;
1050
1019
  U32 storeStart = storeEnd;
1051
1020
  U32 seqPos = cur;
@@ -1082,20 +1051,6 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
1082
1051
  continue; /* will finish */
1083
1052
  }
1084
1053
 
1085
- /* repcodes update : like ZSTD_updateRep(), but update in place */
1086
- if (offCode >= ZSTD_REP_NUM) { /* full offset */
1087
- rep[2] = rep[1];
1088
- rep[1] = rep[0];
1089
- rep[0] = offCode - ZSTD_REP_MOVE;
1090
- } else { /* repcode */
1091
- U32 const repCode = offCode + (llen==0);
1092
- if (repCode) { /* note : if repCode==0, no change */
1093
- U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
1094
- if (repCode >= 2) rep[2] = rep[1];
1095
- rep[1] = rep[0];
1096
- rep[0] = currentOffset;
1097
- } }
1098
-
1099
1054
  assert(anchor + llen <= iend);
1100
1055
  ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen);
1101
1056
  ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen-MINMATCH);
@@ -1104,7 +1059,6 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
1104
1059
  } }
1105
1060
  ZSTD_setBasePrices(optStatePtr, optLevel);
1106
1061
  }
1107
-
1108
1062
  } /* while (ip < ilimit) */
1109
1063
 
1110
1064
  /* Return the last literals size */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -22,9 +22,9 @@
22
22
  /* ====== Dependencies ====== */
23
23
  #include <string.h> /* memcpy, memset */
24
24
  #include <limits.h> /* INT_MAX, UINT_MAX */
25
- #include "mem.h" /* MEM_STATIC */
26
- #include "pool.h" /* threadpool */
27
- #include "threading.h" /* mutex */
25
+ #include "../common/mem.h" /* MEM_STATIC */
26
+ #include "../common/pool.h" /* threadpool */
27
+ #include "../common/threading.h" /* mutex */
28
28
  #include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
29
29
  #include "zstd_ldm.h"
30
30
  #include "zstdmt_compress.h"
@@ -461,7 +461,13 @@ typedef struct {
461
461
  ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */
462
462
  } serialState_t;
463
463
 
464
- static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool* seqPool, ZSTD_CCtx_params params, size_t jobSize)
464
+ static int
465
+ ZSTDMT_serialState_reset(serialState_t* serialState,
466
+ ZSTDMT_seqPool* seqPool,
467
+ ZSTD_CCtx_params params,
468
+ size_t jobSize,
469
+ const void* dict, size_t const dictSize,
470
+ ZSTD_dictContentType_e dictContentType)
465
471
  {
466
472
  /* Adjust parameters */
467
473
  if (params.ldmParams.enableLdm) {
@@ -490,8 +496,7 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
490
496
  /* Size the seq pool tables */
491
497
  ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize));
492
498
  /* Reset the window */
493
- ZSTD_window_clear(&serialState->ldmState.window);
494
- serialState->ldmWindow = serialState->ldmState.window;
499
+ ZSTD_window_init(&serialState->ldmState.window);
495
500
  /* Resize tables and output space if necessary. */
496
501
  if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) {
497
502
  ZSTD_free(serialState->ldmState.hashTable, cMem);
@@ -506,7 +511,24 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
506
511
  /* Zero the tables */
507
512
  memset(serialState->ldmState.hashTable, 0, hashSize);
508
513
  memset(serialState->ldmState.bucketOffsets, 0, bucketSize);
514
+
515
+ /* Update window state and fill hash table with dict */
516
+ serialState->ldmState.loadedDictEnd = 0;
517
+ if (dictSize > 0) {
518
+ if (dictContentType == ZSTD_dct_rawContent) {
519
+ BYTE const* const dictEnd = (const BYTE*)dict + dictSize;
520
+ ZSTD_window_update(&serialState->ldmState.window, dict, dictSize);
521
+ ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, &params.ldmParams);
522
+ serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base);
523
+ } else {
524
+ /* don't even load anything */
525
+ }
526
+ }
527
+
528
+ /* Initialize serialState's copy of ldmWindow. */
529
+ serialState->ldmWindow = serialState->ldmState.window;
509
530
  }
531
+
510
532
  serialState->params = params;
511
533
  serialState->params.jobSize = (U32)jobSize;
512
534
  return 0;
@@ -1054,7 +1076,7 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(const ZSTD_CCtx_params* params)
1054
1076
  static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers)
1055
1077
  {
1056
1078
  if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation);
1057
- FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) );
1079
+ FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , "");
1058
1080
  mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, nbWorkers);
1059
1081
  if (mtctx->bufPool == NULL) return ERROR(memory_allocation);
1060
1082
  mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers);
@@ -1076,7 +1098,7 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
1076
1098
  DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)",
1077
1099
  compressionLevel);
1078
1100
  mtctx->params.compressionLevel = compressionLevel;
1079
- { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, 0, 0);
1101
+ { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1080
1102
  cParams.windowLog = saved_wlog;
1081
1103
  mtctx->params.cParams = cParams;
1082
1104
  }
@@ -1235,7 +1257,8 @@ ZSTDMT_computeNbJobs(const ZSTD_CCtx_params* params, size_t srcSize, unsigned nb
1235
1257
  /* ZSTDMT_compress_advanced_internal() :
1236
1258
  * This is a blocking function : it will only give back control to caller after finishing its compression job.
1237
1259
  */
1238
- static size_t ZSTDMT_compress_advanced_internal(
1260
+ static size_t
1261
+ ZSTDMT_compress_advanced_internal(
1239
1262
  ZSTDMT_CCtx* mtctx,
1240
1263
  void* dst, size_t dstCapacity,
1241
1264
  const void* src, size_t srcSize,
@@ -1267,10 +1290,11 @@ static size_t ZSTDMT_compress_advanced_internal(
1267
1290
 
1268
1291
  assert(avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), required to compress directly into Dst (no additional buffer) */
1269
1292
  ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgJobSize) );
1270
- if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize))
1293
+ /* LDM doesn't even try to load the dictionary in single-ingestion mode */
1294
+ if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize, NULL, 0, ZSTD_dct_auto))
1271
1295
  return ERROR(memory_allocation);
1272
1296
 
1273
- FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) ); /* only expands if necessary */
1297
+ FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) , ""); /* only expands if necessary */
1274
1298
 
1275
1299
  { unsigned u;
1276
1300
  for (u=0; u<nbJobs; u++) {
@@ -1403,7 +1427,7 @@ size_t ZSTDMT_initCStream_internal(
1403
1427
 
1404
1428
  /* init */
1405
1429
  if (params.nbWorkers != mtctx->params.nbWorkers)
1406
- FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) );
1430
+ FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , "");
1407
1431
 
1408
1432
  if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
1409
1433
  if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX;
@@ -1500,7 +1524,8 @@ size_t ZSTDMT_initCStream_internal(
1500
1524
  mtctx->allJobsCompleted = 0;
1501
1525
  mtctx->consumed = 0;
1502
1526
  mtctx->produced = 0;
1503
- if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize))
1527
+ if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize,
1528
+ dict, dictSize, dictContentType))
1504
1529
  return ERROR(memory_allocation);
1505
1530
  return 0;
1506
1531
  }
@@ -1714,9 +1739,11 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
1714
1739
  assert(mtctx->doneJobID < mtctx->nextJobID);
1715
1740
  assert(cSize >= mtctx->jobs[wJobID].dstFlushed);
1716
1741
  assert(mtctx->jobs[wJobID].dstBuff.start != NULL);
1717
- memcpy((char*)output->dst + output->pos,
1718
- (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed,
1719
- toFlush);
1742
+ if (toFlush > 0) {
1743
+ memcpy((char*)output->dst + output->pos,
1744
+ (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed,
1745
+ toFlush);
1746
+ }
1720
1747
  output->pos += toFlush;
1721
1748
  mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */
1722
1749
 
@@ -1786,7 +1813,7 @@ static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range)
1786
1813
  BYTE const* const bufferStart = (BYTE const*)buffer.start;
1787
1814
  BYTE const* const bufferEnd = bufferStart + buffer.capacity;
1788
1815
  BYTE const* const rangeStart = (BYTE const*)range.start;
1789
- BYTE const* const rangeEnd = rangeStart + range.size;
1816
+ BYTE const* const rangeEnd = range.size != 0 ? rangeStart + range.size : rangeStart;
1790
1817
 
1791
1818
  if (rangeStart == NULL || bufferStart == NULL)
1792
1819
  return 0;
@@ -2060,7 +2087,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
2060
2087
  || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */
2061
2088
  size_t const jobSize = mtctx->inBuff.filled;
2062
2089
  assert(mtctx->inBuff.filled <= mtctx->targetSectionSize);
2063
- FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) );
2090
+ FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , "");
2064
2091
  }
2065
2092
 
2066
2093
  /* check for potential compressed data ready to be flushed */
@@ -2074,7 +2101,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
2074
2101
 
2075
2102
  size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
2076
2103
  {
2077
- FORWARD_IF_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) );
2104
+ FORWARD_IF_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) , "");
2078
2105
 
2079
2106
  /* recommended next input size : fill current input buffer */
2080
2107
  return mtctx->targetSectionSize - mtctx->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */
@@ -2091,7 +2118,7 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* ou
2091
2118
  || ((endFrame==ZSTD_e_end) && !mtctx->frameEnded)) { /* need a last 0-size block to end frame */
2092
2119
  DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job (%u bytes, end:%u)",
2093
2120
  (U32)srcSize, (U32)endFrame);
2094
- FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) );
2121
+ FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) , "");
2095
2122
  }
2096
2123
 
2097
2124
  /* check if there is any data available to flush */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -40,7 +40,7 @@
40
40
  /* === Dependencies === */
41
41
  #include <stddef.h> /* size_t */
42
42
  #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
43
- #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
43
+ #include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
44
44
 
45
45
 
46
46
  /* === Constants === */