zstd-ruby 1.3.7.0 → 1.3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/ext/zstdruby/libzstd/BUCK +15 -2
  4. data/ext/zstdruby/libzstd/Makefile +37 -2
  5. data/ext/zstdruby/libzstd/README.md +67 -41
  6. data/ext/zstdruby/libzstd/common/bitstream.h +2 -2
  7. data/ext/zstdruby/libzstd/common/compiler.h +19 -12
  8. data/ext/zstdruby/libzstd/common/cpu.h +1 -1
  9. data/ext/zstdruby/libzstd/common/debug.h +22 -11
  10. data/ext/zstdruby/libzstd/common/error_private.c +6 -0
  11. data/ext/zstdruby/libzstd/common/fse.h +2 -2
  12. data/ext/zstdruby/libzstd/common/huf.h +25 -1
  13. data/ext/zstdruby/libzstd/common/pool.c +1 -1
  14. data/ext/zstdruby/libzstd/common/zstd_common.c +3 -1
  15. data/ext/zstdruby/libzstd/common/zstd_errors.h +1 -0
  16. data/ext/zstdruby/libzstd/common/zstd_internal.h +11 -2
  17. data/ext/zstdruby/libzstd/compress/fse_compress.c +3 -3
  18. data/ext/zstdruby/libzstd/compress/hist.c +19 -11
  19. data/ext/zstdruby/libzstd/compress/hist.h +11 -8
  20. data/ext/zstdruby/libzstd/compress/huf_compress.c +33 -31
  21. data/ext/zstdruby/libzstd/compress/zstd_compress.c +621 -371
  22. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +90 -28
  23. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +4 -4
  24. data/ext/zstdruby/libzstd/compress/zstd_fast.c +15 -15
  25. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +25 -18
  26. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +18 -67
  27. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +2 -6
  28. data/ext/zstdruby/libzstd/compress/zstd_opt.c +133 -48
  29. data/ext/zstdruby/libzstd/compress/zstd_opt.h +8 -0
  30. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +229 -73
  31. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +18 -10
  32. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +178 -42
  33. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +240 -0
  34. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +44 -0
  35. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +244 -1680
  36. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +1307 -0
  37. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +59 -0
  38. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +168 -0
  39. data/ext/zstdruby/libzstd/dictBuilder/cover.c +13 -11
  40. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +15 -15
  41. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +28 -28
  42. data/ext/zstdruby/libzstd/dll/libzstd.def +0 -1
  43. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +0 -10
  44. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +15 -15
  45. data/ext/zstdruby/libzstd/zstd.h +1208 -968
  46. data/lib/zstd-ruby/version.rb +1 -1
  47. metadata +7 -2
@@ -37,8 +37,8 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params,
37
37
  params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
38
38
  assert(params->hashLog <= ZSTD_HASHLOG_MAX);
39
39
  }
40
- if (params->hashEveryLog == 0) {
41
- params->hashEveryLog = params->windowLog < params->hashLog
40
+ if (params->hashRateLog == 0) {
41
+ params->hashRateLog = params->windowLog < params->hashLog
42
42
  ? 0
43
43
  : params->windowLog - params->hashLog;
44
44
  }
@@ -119,20 +119,20 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
119
119
  *
120
120
  * Gets the small hash, checksum, and tag from the rollingHash.
121
121
  *
122
- * If the tag matches (1 << ldmParams.hashEveryLog)-1, then
122
+ * If the tag matches (1 << ldmParams.hashRateLog)-1, then
123
123
  * creates an ldmEntry from the offset, and inserts it into the hash table.
124
124
  *
125
125
  * hBits is the length of the small hash, which is the most significant hBits
126
126
  * of rollingHash. The checksum is the next 32 most significant bits, followed
127
- * by ldmParams.hashEveryLog bits that make up the tag. */
127
+ * by ldmParams.hashRateLog bits that make up the tag. */
128
128
  static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
129
129
  U64 const rollingHash,
130
130
  U32 const hBits,
131
131
  U32 const offset,
132
132
  ldmParams_t const ldmParams)
133
133
  {
134
- U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashEveryLog);
135
- U32 const tagMask = ((U32)1 << ldmParams.hashEveryLog) - 1;
134
+ U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashRateLog);
135
+ U32 const tagMask = ((U32)1 << ldmParams.hashRateLog) - 1;
136
136
  if (tag == tagMask) {
137
137
  U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
138
138
  U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
@@ -143,56 +143,6 @@ static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
143
143
  }
144
144
  }
145
145
 
146
- /** ZSTD_ldm_getRollingHash() :
147
- * Get a 64-bit hash using the first len bytes from buf.
148
- *
149
- * Giving bytes s = s_1, s_2, ... s_k, the hash is defined to be
150
- * H(s) = s_1*(a^(k-1)) + s_2*(a^(k-2)) + ... + s_k*(a^0)
151
- *
152
- * where the constant a is defined to be prime8bytes.
153
- *
154
- * The implementation adds an offset to each byte, so
155
- * H(s) = (s_1 + HASH_CHAR_OFFSET)*(a^(k-1)) + ... */
156
- static U64 ZSTD_ldm_getRollingHash(const BYTE* buf, U32 len)
157
- {
158
- U64 ret = 0;
159
- U32 i;
160
- for (i = 0; i < len; i++) {
161
- ret *= prime8bytes;
162
- ret += buf[i] + LDM_HASH_CHAR_OFFSET;
163
- }
164
- return ret;
165
- }
166
-
167
- /** ZSTD_ldm_ipow() :
168
- * Return base^exp. */
169
- static U64 ZSTD_ldm_ipow(U64 base, U64 exp)
170
- {
171
- U64 ret = 1;
172
- while (exp) {
173
- if (exp & 1) { ret *= base; }
174
- exp >>= 1;
175
- base *= base;
176
- }
177
- return ret;
178
- }
179
-
180
- U64 ZSTD_ldm_getHashPower(U32 minMatchLength) {
181
- DEBUGLOG(4, "ZSTD_ldm_getHashPower: mml=%u", minMatchLength);
182
- assert(minMatchLength >= ZSTD_LDM_MINMATCH_MIN);
183
- return ZSTD_ldm_ipow(prime8bytes, minMatchLength - 1);
184
- }
185
-
186
- /** ZSTD_ldm_updateHash() :
187
- * Updates hash by removing toRemove and adding toAdd. */
188
- static U64 ZSTD_ldm_updateHash(U64 hash, BYTE toRemove, BYTE toAdd, U64 hashPower)
189
- {
190
- hash -= ((toRemove + LDM_HASH_CHAR_OFFSET) * hashPower);
191
- hash *= prime8bytes;
192
- hash += toAdd + LDM_HASH_CHAR_OFFSET;
193
- return hash;
194
- }
195
-
196
146
  /** ZSTD_ldm_countBackwardsMatch() :
197
147
  * Returns the number of bytes that match backwards before pIn and pMatch.
198
148
  *
@@ -238,6 +188,7 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
238
188
  case ZSTD_btlazy2:
239
189
  case ZSTD_btopt:
240
190
  case ZSTD_btultra:
191
+ case ZSTD_btultra2:
241
192
  break;
242
193
  default:
243
194
  assert(0); /* not possible : not a valid strategy id */
@@ -261,9 +212,9 @@ static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state,
261
212
  const BYTE* cur = lastHashed + 1;
262
213
 
263
214
  while (cur < iend) {
264
- rollingHash = ZSTD_ldm_updateHash(rollingHash, cur[-1],
265
- cur[ldmParams.minMatchLength-1],
266
- state->hashPower);
215
+ rollingHash = ZSTD_rollingHash_rotate(rollingHash, cur[-1],
216
+ cur[ldmParams.minMatchLength-1],
217
+ state->hashPower);
267
218
  ZSTD_ldm_makeEntryAndInsertByTag(state,
268
219
  rollingHash, hBits,
269
220
  (U32)(cur - base), ldmParams);
@@ -297,8 +248,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
297
248
  U64 const hashPower = ldmState->hashPower;
298
249
  U32 const hBits = params->hashLog - params->bucketSizeLog;
299
250
  U32 const ldmBucketSize = 1U << params->bucketSizeLog;
300
- U32 const hashEveryLog = params->hashEveryLog;
301
- U32 const ldmTagMask = (1U << params->hashEveryLog) - 1;
251
+ U32 const hashRateLog = params->hashRateLog;
252
+ U32 const ldmTagMask = (1U << params->hashRateLog) - 1;
302
253
  /* Prefix and extDict parameters */
303
254
  U32 const dictLimit = ldmState->window.dictLimit;
304
255
  U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
@@ -324,16 +275,16 @@ static size_t ZSTD_ldm_generateSequences_internal(
324
275
  size_t forwardMatchLength = 0, backwardMatchLength = 0;
325
276
  ldmEntry_t* bestEntry = NULL;
326
277
  if (ip != istart) {
327
- rollingHash = ZSTD_ldm_updateHash(rollingHash, lastHashed[0],
328
- lastHashed[minMatchLength],
329
- hashPower);
278
+ rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0],
279
+ lastHashed[minMatchLength],
280
+ hashPower);
330
281
  } else {
331
- rollingHash = ZSTD_ldm_getRollingHash(ip, minMatchLength);
282
+ rollingHash = ZSTD_rollingHash_compute(ip, minMatchLength);
332
283
  }
333
284
  lastHashed = ip;
334
285
 
335
286
  /* Do not insert and do not look for a match */
336
- if (ZSTD_ldm_getTag(rollingHash, hBits, hashEveryLog) != ldmTagMask) {
287
+ if (ZSTD_ldm_getTag(rollingHash, hBits, hashRateLog) != ldmTagMask) {
337
288
  ip++;
338
289
  continue;
339
290
  }
@@ -593,7 +544,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
593
544
  void const* src, size_t srcSize)
594
545
  {
595
546
  const ZSTD_compressionParameters* const cParams = &ms->cParams;
596
- unsigned const minMatch = cParams->searchLength;
547
+ unsigned const minMatch = cParams->minMatch;
597
548
  ZSTD_blockCompressor const blockCompressor =
598
549
  ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms));
599
550
  /* Input bounds */
@@ -21,7 +21,7 @@ extern "C" {
21
21
  * Long distance matching
22
22
  ***************************************/
23
23
 
24
- #define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
24
+ #define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
25
25
 
26
26
  /**
27
27
  * ZSTD_ldm_generateSequences():
@@ -86,12 +86,8 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params);
86
86
  */
87
87
  size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
88
88
 
89
- /** ZSTD_ldm_getTableSize() :
90
- * Return prime8bytes^(minMatchLength-1) */
91
- U64 ZSTD_ldm_getHashPower(U32 minMatchLength);
92
-
93
89
  /** ZSTD_ldm_adjustParameters() :
94
- * If the params->hashEveryLog is not set, set it to its default value based on
90
+ * If the params->hashRateLog is not set, set it to its default value based on
95
91
  * windowLog and params->hashLog.
96
92
  *
97
93
  * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to
@@ -17,6 +17,8 @@
17
17
  #define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */
18
18
  #define ZSTD_MAX_PRICE (1<<30)
19
19
 
20
+ #define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
21
+
20
22
 
21
23
  /*-*************************************
22
24
  * Price functions for optimal parser
@@ -52,11 +54,15 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
52
54
  return weight;
53
55
  }
54
56
 
55
- /* debugging function, @return price in bytes */
57
+ #if (DEBUGLEVEL>=2)
58
+ /* debugging function,
59
+ * @return price in bytes as fractional value
60
+ * for debug messages only */
56
61
  MEM_STATIC double ZSTD_fCost(U32 price)
57
62
  {
58
63
  return (double)price / (BITCOST_MULTIPLIER*8);
59
64
  }
65
+ #endif
60
66
 
61
67
  static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
62
68
  {
@@ -67,29 +73,44 @@ static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
67
73
  }
68
74
 
69
75
 
70
- static U32 ZSTD_downscaleStat(U32* table, U32 lastEltIndex, int malus)
76
+ /* ZSTD_downscaleStat() :
77
+ * reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus)
78
+ * return the resulting sum of elements */
79
+ static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus)
71
80
  {
72
81
  U32 s, sum=0;
82
+ DEBUGLOG(5, "ZSTD_downscaleStat (nbElts=%u)", (unsigned)lastEltIndex+1);
73
83
  assert(ZSTD_FREQ_DIV+malus > 0 && ZSTD_FREQ_DIV+malus < 31);
74
- for (s=0; s<=lastEltIndex; s++) {
84
+ for (s=0; s<lastEltIndex+1; s++) {
75
85
  table[s] = 1 + (table[s] >> (ZSTD_FREQ_DIV+malus));
76
86
  sum += table[s];
77
87
  }
78
88
  return sum;
79
89
  }
80
90
 
81
- static void ZSTD_rescaleFreqs(optState_t* const optPtr,
82
- const BYTE* const src, size_t const srcSize,
83
- int optLevel)
91
+ /* ZSTD_rescaleFreqs() :
92
+ * if first block (detected by optPtr->litLengthSum == 0) : init statistics
93
+ * take hints from dictionary if there is one
94
+ * or init from zero, using src for literals stats, or flat 1 for match symbols
95
+ * otherwise downscale existing stats, to be used as seed for next block.
96
+ */
97
+ static void
98
+ ZSTD_rescaleFreqs(optState_t* const optPtr,
99
+ const BYTE* const src, size_t const srcSize,
100
+ int const optLevel)
84
101
  {
102
+ DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
85
103
  optPtr->priceType = zop_dynamic;
86
104
 
87
105
  if (optPtr->litLengthSum == 0) { /* first block : init */
88
- if (srcSize <= 1024) /* heuristic */
106
+ if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */
107
+ DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef");
89
108
  optPtr->priceType = zop_predef;
109
+ }
90
110
 
91
111
  assert(optPtr->symbolCosts != NULL);
92
- if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */
112
+ if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
113
+ /* huffman table presumed generated by dictionary */
93
114
  optPtr->priceType = zop_dynamic;
94
115
 
95
116
  assert(optPtr->litFreq != NULL);
@@ -208,7 +229,9 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
208
229
 
209
230
  /* dynamic statistics */
210
231
  { U32 const llCode = ZSTD_LLcode(litLength);
211
- return (LL_bits[llCode] * BITCOST_MULTIPLIER) + (optPtr->litLengthSumBasePrice - WEIGHT(optPtr->litLengthFreq[llCode], optLevel));
232
+ return (LL_bits[llCode] * BITCOST_MULTIPLIER)
233
+ + optPtr->litLengthSumBasePrice
234
+ - WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
212
235
  }
213
236
  }
214
237
 
@@ -253,7 +276,7 @@ static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLe
253
276
  FORCE_INLINE_TEMPLATE U32
254
277
  ZSTD_getMatchPrice(U32 const offset,
255
278
  U32 const matchLength,
256
- const optState_t* const optPtr,
279
+ const optState_t* const optPtr,
257
280
  int const optLevel)
258
281
  {
259
282
  U32 price;
@@ -385,7 +408,6 @@ static U32 ZSTD_insertBt1(
385
408
  U32* largerPtr = smallerPtr + 1;
386
409
  U32 dummy32; /* to be nullified at the end */
387
410
  U32 const windowLow = ms->window.lowLimit;
388
- U32 const matchLow = windowLow ? windowLow : 1;
389
411
  U32 matchEndIdx = current+8+1;
390
412
  size_t bestLength = 8;
391
413
  U32 nbCompares = 1U << cParams->searchLog;
@@ -401,7 +423,8 @@ static U32 ZSTD_insertBt1(
401
423
  assert(ip <= iend-8); /* required for h calculation */
402
424
  hashTable[h] = current; /* Update Hash Table */
403
425
 
404
- while (nbCompares-- && (matchIndex >= matchLow)) {
426
+ assert(windowLow > 0);
427
+ while (nbCompares-- && (matchIndex >= windowLow)) {
405
428
  U32* const nextPtr = bt + 2*(matchIndex & btMask);
406
429
  size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
407
430
  assert(matchIndex < current);
@@ -479,7 +502,7 @@ void ZSTD_updateTree_internal(
479
502
  const BYTE* const base = ms->window.base;
480
503
  U32 const target = (U32)(ip - base);
481
504
  U32 idx = ms->nextToUpdate;
482
- DEBUGLOG(5, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
505
+ DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
483
506
  idx, target, dictMode);
484
507
 
485
508
  while(idx < target)
@@ -488,15 +511,18 @@ void ZSTD_updateTree_internal(
488
511
  }
489
512
 
490
513
  void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
491
- ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.searchLength, ZSTD_noDict);
514
+ ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
492
515
  }
493
516
 
494
517
  FORCE_INLINE_TEMPLATE
495
518
  U32 ZSTD_insertBtAndGetAllMatches (
496
519
  ZSTD_matchState_t* ms,
497
520
  const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode,
498
- U32 rep[ZSTD_REP_NUM], U32 const ll0,
499
- ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */)
521
+ U32 rep[ZSTD_REP_NUM],
522
+ U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
523
+ ZSTD_match_t* matches,
524
+ const U32 lengthToBeat,
525
+ U32 const mls /* template */)
500
526
  {
501
527
  const ZSTD_compressionParameters* const cParams = &ms->cParams;
502
528
  U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
@@ -542,6 +568,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
542
568
  DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current);
543
569
 
544
570
  /* check repCode */
571
+ assert(ll0 <= 1); /* necessarily 1 or 0 */
545
572
  { U32 const lastR = ZSTD_REP_NUM + ll0;
546
573
  U32 repCode;
547
574
  for (repCode = ll0; repCode < lastR; repCode++) {
@@ -724,7 +751,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
724
751
  ZSTD_match_t* matches, U32 const lengthToBeat)
725
752
  {
726
753
  const ZSTD_compressionParameters* const cParams = &ms->cParams;
727
- U32 const matchLengthSearch = cParams->searchLength;
754
+ U32 const matchLengthSearch = cParams->minMatch;
728
755
  DEBUGLOG(8, "ZSTD_BtGetAllMatches");
729
756
  if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */
730
757
  ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode);
@@ -774,12 +801,30 @@ static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
774
801
  return sol.litlen + sol.mlen;
775
802
  }
776
803
 
804
+ #if 0 /* debug */
805
+
806
+ static void
807
+ listStats(const U32* table, int lastEltID)
808
+ {
809
+ int const nbElts = lastEltID + 1;
810
+ int enb;
811
+ for (enb=0; enb < nbElts; enb++) {
812
+ (void)table;
813
+ //RAWLOG(2, "%3i:%3i, ", enb, table[enb]);
814
+ RAWLOG(2, "%4i,", table[enb]);
815
+ }
816
+ RAWLOG(2, " \n");
817
+ }
818
+
819
+ #endif
820
+
777
821
  FORCE_INLINE_TEMPLATE size_t
778
822
  ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
779
823
  seqStore_t* seqStore,
780
824
  U32 rep[ZSTD_REP_NUM],
781
- const void* src, size_t srcSize,
782
- const int optLevel, const ZSTD_dictMode_e dictMode)
825
+ const void* src, size_t srcSize,
826
+ const int optLevel,
827
+ const ZSTD_dictMode_e dictMode)
783
828
  {
784
829
  optState_t* const optStatePtr = &ms->opt;
785
830
  const BYTE* const istart = (const BYTE*)src;
@@ -792,14 +837,15 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
792
837
  const ZSTD_compressionParameters* const cParams = &ms->cParams;
793
838
 
794
839
  U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
795
- U32 const minMatch = (cParams->searchLength == 3) ? 3 : 4;
840
+ U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
796
841
 
797
842
  ZSTD_optimal_t* const opt = optStatePtr->priceTable;
798
843
  ZSTD_match_t* const matches = optStatePtr->matchTable;
799
844
  ZSTD_optimal_t lastSequence;
800
845
 
801
846
  /* init */
802
- DEBUGLOG(5, "ZSTD_compressBlock_opt_generic");
847
+ DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
848
+ (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
803
849
  assert(optLevel <= 2);
804
850
  ms->nextToUpdate3 = ms->nextToUpdate;
805
851
  ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
@@ -999,7 +1045,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
999
1045
  U32 const offCode = opt[storePos].off;
1000
1046
  U32 const advance = llen + mlen;
1001
1047
  DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u",
1002
- anchor - istart, llen, mlen);
1048
+ anchor - istart, (unsigned)llen, (unsigned)mlen);
1003
1049
 
1004
1050
  if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */
1005
1051
  assert(storePos == storeEnd); /* must be last sequence */
@@ -1047,11 +1093,11 @@ size_t ZSTD_compressBlock_btopt(
1047
1093
 
1048
1094
 
1049
1095
  /* used in 2-pass strategy */
1050
- static U32 ZSTD_upscaleStat(U32* table, U32 lastEltIndex, int bonus)
1096
+ static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus)
1051
1097
  {
1052
1098
  U32 s, sum=0;
1053
- assert(ZSTD_FREQ_DIV+bonus > 0);
1054
- for (s=0; s<=lastEltIndex; s++) {
1099
+ assert(ZSTD_FREQ_DIV+bonus >= 0);
1100
+ for (s=0; s<lastEltIndex+1; s++) {
1055
1101
  table[s] <<= ZSTD_FREQ_DIV+bonus;
1056
1102
  table[s]--;
1057
1103
  sum += table[s];
@@ -1063,9 +1109,43 @@ static U32 ZSTD_upscaleStat(U32* table, U32 lastEltIndex, int bonus)
1063
1109
  MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
1064
1110
  {
1065
1111
  optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0);
1066
- optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 1);
1067
- optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 1);
1068
- optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 1);
1112
+ optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0);
1113
+ optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0);
1114
+ optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0);
1115
+ }
1116
+
1117
+ /* ZSTD_initStats_ultra():
1118
+ * make a first compression pass, just to seed stats with more accurate starting values.
1119
+ * only works on first block, with no dictionary and no ldm.
1120
+ * this function cannot error, hence its constract must be respected.
1121
+ */
1122
+ static void
1123
+ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
1124
+ seqStore_t* seqStore,
1125
+ U32 rep[ZSTD_REP_NUM],
1126
+ const void* src, size_t srcSize)
1127
+ {
1128
+ U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
1129
+ memcpy(tmpRep, rep, sizeof(tmpRep));
1130
+
1131
+ DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
1132
+ assert(ms->opt.litLengthSum == 0); /* first block */
1133
+ assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
1134
+ assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
1135
+ assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
1136
+
1137
+ ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
1138
+
1139
+ /* invalidate first scan from history */
1140
+ ZSTD_resetSeqStore(seqStore);
1141
+ ms->window.base -= srcSize;
1142
+ ms->window.dictLimit += (U32)srcSize;
1143
+ ms->window.lowLimit = ms->window.dictLimit;
1144
+ ms->nextToUpdate = ms->window.dictLimit;
1145
+ ms->nextToUpdate3 = ms->window.dictLimit;
1146
+
1147
+ /* re-inforce weight of collected statistics */
1148
+ ZSTD_upscaleStats(&ms->opt);
1069
1149
  }
1070
1150
 
1071
1151
  size_t ZSTD_compressBlock_btultra(
@@ -1073,33 +1153,34 @@ size_t ZSTD_compressBlock_btultra(
1073
1153
  const void* src, size_t srcSize)
1074
1154
  {
1075
1155
  DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
1076
- #if 0
1077
- /* 2-pass strategy (disabled)
1156
+ return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
1157
+ }
1158
+
1159
+ size_t ZSTD_compressBlock_btultra2(
1160
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
1161
+ const void* src, size_t srcSize)
1162
+ {
1163
+ U32 const current = (U32)((const BYTE*)src - ms->window.base);
1164
+ DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
1165
+
1166
+ /* 2-pass strategy:
1078
1167
  * this strategy makes a first pass over first block to collect statistics
1079
1168
  * and seed next round's statistics with it.
1169
+ * After 1st pass, function forgets everything, and starts a new block.
1170
+ * Consequently, this can only work if no data has been previously loaded in tables,
1171
+ * aka, no dictionary, no prefix, no ldm preprocessing.
1080
1172
  * The compression ratio gain is generally small (~0.5% on first block),
1081
1173
  * the cost is 2x cpu time on first block. */
1082
1174
  assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
1083
1175
  if ( (ms->opt.litLengthSum==0) /* first block */
1084
- && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
1085
- && (ms->window.dictLimit == ms->window.lowLimit) ) { /* no dictionary */
1086
- U32 tmpRep[ZSTD_REP_NUM];
1087
- DEBUGLOG(5, "ZSTD_compressBlock_btultra: first block: collecting statistics");
1088
- assert(ms->nextToUpdate >= ms->window.dictLimit
1089
- && ms->nextToUpdate <= ms->window.dictLimit + 1);
1090
- memcpy(tmpRep, rep, sizeof(tmpRep));
1091
- ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
1092
- ZSTD_resetSeqStore(seqStore);
1093
- /* invalidate first scan from history */
1094
- ms->window.base -= srcSize;
1095
- ms->window.dictLimit += (U32)srcSize;
1096
- ms->window.lowLimit = ms->window.dictLimit;
1097
- ms->nextToUpdate = ms->window.dictLimit;
1098
- ms->nextToUpdate3 = ms->window.dictLimit;
1099
- /* re-inforce weight of collected statistics */
1100
- ZSTD_upscaleStats(&ms->opt);
1176
+ && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
1177
+ && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
1178
+ && (current == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */
1179
+ && (srcSize > ZSTD_PREDEF_THRESHOLD)
1180
+ ) {
1181
+ ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
1101
1182
  }
1102
- #endif
1183
+
1103
1184
  return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
1104
1185
  }
1105
1186
 
@@ -1130,3 +1211,7 @@ size_t ZSTD_compressBlock_btultra_extDict(
1130
1211
  {
1131
1212
  return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict);
1132
1213
  }
1214
+
1215
+ /* note : no btultra2 variant for extDict nor dictMatchState,
1216
+ * because btultra2 is not meant to work with dictionaries
1217
+ * and is only specific for the first block (no prefix) */