zstdlib 0.3.0 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +30 -1
  3. data/README.md +2 -2
  4. data/Rakefile +1 -1
  5. data/ext/zstdlib/extconf.rb +3 -3
  6. data/ext/zstdlib/ruby/zlib-2.7/zstdlib.c +4895 -0
  7. data/ext/zstdlib/ruby/zlib-3.0/zstdlib.c +4994 -0
  8. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/bitstream.h +59 -51
  9. data/ext/zstdlib/zstd-1.5.0/lib/common/compiler.h +289 -0
  10. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/cpu.h +1 -3
  11. data/ext/zstdlib/zstd-1.5.0/lib/common/debug.c +24 -0
  12. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/debug.h +22 -49
  13. data/ext/zstdlib/zstd-1.5.0/lib/common/entropy_common.c +362 -0
  14. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/error_private.c +3 -1
  15. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/error_private.h +8 -4
  16. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/fse.h +50 -42
  17. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/fse_decompress.c +149 -55
  18. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/huf.h +43 -39
  19. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/mem.h +69 -25
  20. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/pool.c +30 -20
  21. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/pool.h +3 -3
  22. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/threading.c +51 -4
  23. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/threading.h +36 -4
  24. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/xxhash.c +40 -92
  25. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/xxhash.h +12 -32
  26. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/zstd_common.c +10 -10
  27. data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_deps.h +111 -0
  28. data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_internal.h +490 -0
  29. data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_trace.h +154 -0
  30. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/fse_compress.c +47 -63
  31. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/hist.c +41 -63
  32. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/hist.h +13 -33
  33. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/huf_compress.c +332 -193
  34. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress.c +6393 -0
  35. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_internal.h +522 -86
  36. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_literals.c +25 -16
  37. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_literals.h +2 -2
  38. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.c +50 -24
  39. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.h +11 -4
  40. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress_superblock.c +572 -0
  41. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress_superblock.h +32 -0
  42. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_cwksp.h +662 -0
  43. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_double_fast.c +43 -41
  44. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_double_fast.h +2 -2
  45. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_fast.c +85 -80
  46. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_fast.h +2 -2
  47. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.c +2184 -0
  48. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.h +125 -0
  49. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_ldm.c +333 -208
  50. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_ldm.h +15 -3
  51. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_ldm_geartab.h +103 -0
  52. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_opt.c +228 -129
  53. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_opt.h +1 -1
  54. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstdmt_compress.c +151 -440
  55. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstdmt_compress.h +110 -0
  56. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/huf_decompress.c +395 -276
  57. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_ddict.c +20 -16
  58. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_ddict.h +3 -3
  59. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress.c +628 -231
  60. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.c +606 -380
  61. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.h +8 -5
  62. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_internal.h +39 -9
  63. data/ext/zstdlib/zstd-1.5.0/lib/zdict.h +452 -0
  64. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/zstd.h +740 -153
  65. data/ext/zstdlib/{zstd-1.4.2/lib/common → zstd-1.5.0/lib}/zstd_errors.h +3 -1
  66. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzclose.c +1 -1
  67. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzcompatibility.h +1 -1
  68. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzguts.h +0 -0
  69. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzlib.c +9 -9
  70. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzread.c +16 -8
  71. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzwrite.c +8 -8
  72. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.c +131 -45
  73. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  74. metadata +74 -65
  75. data/ext/zstdlib/zstd-1.4.2/lib/common/compiler.h +0 -147
  76. data/ext/zstdlib/zstd-1.4.2/lib/common/debug.c +0 -44
  77. data/ext/zstdlib/zstd-1.4.2/lib/common/entropy_common.c +0 -236
  78. data/ext/zstdlib/zstd-1.4.2/lib/common/zstd_internal.h +0 -371
  79. data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_compress.c +0 -3904
  80. data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_lazy.c +0 -1111
  81. data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_lazy.h +0 -67
  82. data/ext/zstdlib/zstd-1.4.2/lib/compress/zstdmt_compress.h +0 -192
@@ -0,0 +1,125 @@
1
+ /*
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ #ifndef ZSTD_LAZY_H
12
+ #define ZSTD_LAZY_H
13
+
14
+ #if defined (__cplusplus)
15
+ extern "C" {
16
+ #endif
17
+
18
+ #include "zstd_compress_internal.h"
19
+
20
+ /**
21
+ * Dedicated Dictionary Search Structure bucket log. In the
22
+ * ZSTD_dedicatedDictSearch mode, the hashTable has
23
+ * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just
24
+ * one.
25
+ */
26
+ #define ZSTD_LAZY_DDSS_BUCKET_LOG 2
27
+
28
+ U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
29
+ void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
30
+
31
+ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip);
32
+
33
+ void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */
34
+
35
+ size_t ZSTD_compressBlock_btlazy2(
36
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
37
+ void const* src, size_t srcSize);
38
+ size_t ZSTD_compressBlock_lazy2(
39
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
40
+ void const* src, size_t srcSize);
41
+ size_t ZSTD_compressBlock_lazy(
42
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
43
+ void const* src, size_t srcSize);
44
+ size_t ZSTD_compressBlock_greedy(
45
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
46
+ void const* src, size_t srcSize);
47
+ size_t ZSTD_compressBlock_lazy2_row(
48
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
49
+ void const* src, size_t srcSize);
50
+ size_t ZSTD_compressBlock_lazy_row(
51
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
52
+ void const* src, size_t srcSize);
53
+ size_t ZSTD_compressBlock_greedy_row(
54
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
55
+ void const* src, size_t srcSize);
56
+
57
+ size_t ZSTD_compressBlock_btlazy2_dictMatchState(
58
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
59
+ void const* src, size_t srcSize);
60
+ size_t ZSTD_compressBlock_lazy2_dictMatchState(
61
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
62
+ void const* src, size_t srcSize);
63
+ size_t ZSTD_compressBlock_lazy_dictMatchState(
64
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
65
+ void const* src, size_t srcSize);
66
+ size_t ZSTD_compressBlock_greedy_dictMatchState(
67
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
68
+ void const* src, size_t srcSize);
69
+ size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
70
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
71
+ void const* src, size_t srcSize);
72
+ size_t ZSTD_compressBlock_lazy_dictMatchState_row(
73
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
74
+ void const* src, size_t srcSize);
75
+ size_t ZSTD_compressBlock_greedy_dictMatchState_row(
76
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
77
+ void const* src, size_t srcSize);
78
+
79
+ size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
80
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
81
+ void const* src, size_t srcSize);
82
+ size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
83
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
84
+ void const* src, size_t srcSize);
85
+ size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
86
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
87
+ void const* src, size_t srcSize);
88
+ size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
89
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
90
+ void const* src, size_t srcSize);
91
+ size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
92
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
93
+ void const* src, size_t srcSize);
94
+ size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
95
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
96
+ void const* src, size_t srcSize);
97
+
98
+ size_t ZSTD_compressBlock_greedy_extDict(
99
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
100
+ void const* src, size_t srcSize);
101
+ size_t ZSTD_compressBlock_lazy_extDict(
102
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
103
+ void const* src, size_t srcSize);
104
+ size_t ZSTD_compressBlock_lazy2_extDict(
105
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
106
+ void const* src, size_t srcSize);
107
+ size_t ZSTD_compressBlock_greedy_extDict_row(
108
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
109
+ void const* src, size_t srcSize);
110
+ size_t ZSTD_compressBlock_lazy_extDict_row(
111
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
112
+ void const* src, size_t srcSize);
113
+ size_t ZSTD_compressBlock_lazy2_extDict_row(
114
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
115
+ void const* src, size_t srcSize);
116
+ size_t ZSTD_compressBlock_btlazy2_extDict(
117
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
118
+ void const* src, size_t srcSize);
119
+
120
+
121
+ #if defined (__cplusplus)
122
+ }
123
+ #endif
124
+
125
+ #endif /* ZSTD_LAZY_H */
@@ -1,22 +1,136 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
6
6
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
7
  * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #include "zstd_ldm.h"
11
12
 
12
- #include "debug.h"
13
+ #include "../common/debug.h"
14
+ #include "../common/xxhash.h"
13
15
  #include "zstd_fast.h" /* ZSTD_fillHashTable() */
14
16
  #include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */
17
+ #include "zstd_ldm_geartab.h"
15
18
 
16
19
  #define LDM_BUCKET_SIZE_LOG 3
17
20
  #define LDM_MIN_MATCH_LENGTH 64
18
21
  #define LDM_HASH_RLOG 7
19
- #define LDM_HASH_CHAR_OFFSET 10
22
+
23
+ typedef struct {
24
+ U64 rolling;
25
+ U64 stopMask;
26
+ } ldmRollingHashState_t;
27
+
28
+ /** ZSTD_ldm_gear_init():
29
+ *
30
+ * Initializes the rolling hash state such that it will honor the
31
+ * settings in params. */
32
+ static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params)
33
+ {
34
+ unsigned maxBitsInMask = MIN(params->minMatchLength, 64);
35
+ unsigned hashRateLog = params->hashRateLog;
36
+
37
+ state->rolling = ~(U32)0;
38
+
39
+ /* The choice of the splitting criterion is subject to two conditions:
40
+ * 1. it has to trigger on average every 2^(hashRateLog) bytes;
41
+ * 2. ideally, it has to depend on a window of minMatchLength bytes.
42
+ *
43
+ * In the gear hash algorithm, bit n depends on the last n bytes;
44
+ * so in order to obtain a good quality splitting criterion it is
45
+ * preferable to use bits with high weight.
46
+ *
47
+ * To match condition 1 we use a mask with hashRateLog bits set
48
+ * and, because of the previous remark, we make sure these bits
49
+ * have the highest possible weight while still respecting
50
+ * condition 2.
51
+ */
52
+ if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) {
53
+ state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog);
54
+ } else {
55
+ /* In this degenerate case we simply honor the hash rate. */
56
+ state->stopMask = ((U64)1 << hashRateLog) - 1;
57
+ }
58
+ }
59
+
60
+ /** ZSTD_ldm_gear_reset()
61
+ * Feeds [data, data + minMatchLength) into the hash without registering any
62
+ * splits. This effectively resets the hash state. This is used when skipping
63
+ * over data, either at the beginning of a block, or skipping sections.
64
+ */
65
+ static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state,
66
+ BYTE const* data, size_t minMatchLength)
67
+ {
68
+ U64 hash = state->rolling;
69
+ size_t n = 0;
70
+
71
+ #define GEAR_ITER_ONCE() do { \
72
+ hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \
73
+ n += 1; \
74
+ } while (0)
75
+ while (n + 3 < minMatchLength) {
76
+ GEAR_ITER_ONCE();
77
+ GEAR_ITER_ONCE();
78
+ GEAR_ITER_ONCE();
79
+ GEAR_ITER_ONCE();
80
+ }
81
+ while (n < minMatchLength) {
82
+ GEAR_ITER_ONCE();
83
+ }
84
+ #undef GEAR_ITER_ONCE
85
+ }
86
+
87
+ /** ZSTD_ldm_gear_feed():
88
+ *
89
+ * Registers in the splits array all the split points found in the first
90
+ * size bytes following the data pointer. This function terminates when
91
+ * either all the data has been processed or LDM_BATCH_SIZE splits are
92
+ * present in the splits array.
93
+ *
94
+ * Precondition: The splits array must not be full.
95
+ * Returns: The number of bytes processed. */
96
+ static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state,
97
+ BYTE const* data, size_t size,
98
+ size_t* splits, unsigned* numSplits)
99
+ {
100
+ size_t n;
101
+ U64 hash, mask;
102
+
103
+ hash = state->rolling;
104
+ mask = state->stopMask;
105
+ n = 0;
106
+
107
+ #define GEAR_ITER_ONCE() do { \
108
+ hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \
109
+ n += 1; \
110
+ if (UNLIKELY((hash & mask) == 0)) { \
111
+ splits[*numSplits] = n; \
112
+ *numSplits += 1; \
113
+ if (*numSplits == LDM_BATCH_SIZE) \
114
+ goto done; \
115
+ } \
116
+ } while (0)
117
+
118
+ while (n + 3 < size) {
119
+ GEAR_ITER_ONCE();
120
+ GEAR_ITER_ONCE();
121
+ GEAR_ITER_ONCE();
122
+ GEAR_ITER_ONCE();
123
+ }
124
+ while (n < size) {
125
+ GEAR_ITER_ONCE();
126
+ }
127
+
128
+ #undef GEAR_ITER_ONCE
129
+
130
+ done:
131
+ state->rolling = hash;
132
+ return n;
133
+ }
20
134
 
21
135
  void ZSTD_ldm_adjustParameters(ldmParams_t* params,
22
136
  ZSTD_compressionParameters const* cParams)
@@ -26,13 +140,6 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params,
26
140
  DEBUGLOG(4, "ZSTD_ldm_adjustParameters");
27
141
  if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
28
142
  if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH;
29
- if (cParams->strategy >= ZSTD_btopt) {
30
- /* Get out of the way of the optimal parser */
31
- U32 const minMatch = MAX(cParams->targetLength, params->minMatchLength);
32
- assert(minMatch >= ZSTD_LDM_MINMATCH_MIN);
33
- assert(minMatch <= ZSTD_LDM_MINMATCH_MAX);
34
- params->minMatchLength = minMatch;
35
- }
36
143
  if (params->hashLog == 0) {
37
144
  params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
38
145
  assert(params->hashLog <= ZSTD_HASHLOG_MAX);
@@ -49,9 +156,9 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params)
49
156
  {
50
157
  size_t const ldmHSize = ((size_t)1) << params.hashLog;
51
158
  size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog);
52
- size_t const ldmBucketSize =
53
- ((size_t)1) << (params.hashLog - ldmBucketSizeLog);
54
- size_t const totalSize = ldmBucketSize + ldmHSize * sizeof(ldmEntry_t);
159
+ size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog);
160
+ size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize)
161
+ + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t));
55
162
  return params.enableLdm ? totalSize : 0;
56
163
  }
57
164
 
@@ -60,41 +167,6 @@ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize)
60
167
  return params.enableLdm ? (maxChunkSize / params.minMatchLength) : 0;
61
168
  }
62
169
 
63
- /** ZSTD_ldm_getSmallHash() :
64
- * numBits should be <= 32
65
- * If numBits==0, returns 0.
66
- * @return : the most significant numBits of value. */
67
- static U32 ZSTD_ldm_getSmallHash(U64 value, U32 numBits)
68
- {
69
- assert(numBits <= 32);
70
- return numBits == 0 ? 0 : (U32)(value >> (64 - numBits));
71
- }
72
-
73
- /** ZSTD_ldm_getChecksum() :
74
- * numBitsToDiscard should be <= 32
75
- * @return : the next most significant 32 bits after numBitsToDiscard */
76
- static U32 ZSTD_ldm_getChecksum(U64 hash, U32 numBitsToDiscard)
77
- {
78
- assert(numBitsToDiscard <= 32);
79
- return (hash >> (64 - 32 - numBitsToDiscard)) & 0xFFFFFFFF;
80
- }
81
-
82
- /** ZSTD_ldm_getTag() ;
83
- * Given the hash, returns the most significant numTagBits bits
84
- * after (32 + hbits) bits.
85
- *
86
- * If there are not enough bits remaining, return the last
87
- * numTagBits bits. */
88
- static U32 ZSTD_ldm_getTag(U64 hash, U32 hbits, U32 numTagBits)
89
- {
90
- assert(numTagBits < 32 && hbits <= 32);
91
- if (32 - hbits < numTagBits) {
92
- return hash & (((U32)1 << numTagBits) - 1);
93
- } else {
94
- return (hash >> (32 - hbits - numTagBits)) & (((U32)1 << numTagBits) - 1);
95
- }
96
- }
97
-
98
170
  /** ZSTD_ldm_getBucket() :
99
171
  * Returns a pointer to the start of the bucket associated with hash. */
100
172
  static ldmEntry_t* ZSTD_ldm_getBucket(
@@ -109,38 +181,12 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
109
181
  size_t const hash, const ldmEntry_t entry,
110
182
  ldmParams_t const ldmParams)
111
183
  {
112
- BYTE* const bucketOffsets = ldmState->bucketOffsets;
113
- *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + bucketOffsets[hash]) = entry;
114
- bucketOffsets[hash]++;
115
- bucketOffsets[hash] &= ((U32)1 << ldmParams.bucketSizeLog) - 1;
116
- }
184
+ BYTE* const pOffset = ldmState->bucketOffsets + hash;
185
+ unsigned const offset = *pOffset;
186
+
187
+ *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry;
188
+ *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1));
117
189
 
118
- /** ZSTD_ldm_makeEntryAndInsertByTag() :
119
- *
120
- * Gets the small hash, checksum, and tag from the rollingHash.
121
- *
122
- * If the tag matches (1 << ldmParams.hashRateLog)-1, then
123
- * creates an ldmEntry from the offset, and inserts it into the hash table.
124
- *
125
- * hBits is the length of the small hash, which is the most significant hBits
126
- * of rollingHash. The checksum is the next 32 most significant bits, followed
127
- * by ldmParams.hashRateLog bits that make up the tag. */
128
- static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
129
- U64 const rollingHash,
130
- U32 const hBits,
131
- U32 const offset,
132
- ldmParams_t const ldmParams)
133
- {
134
- U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashRateLog);
135
- U32 const tagMask = ((U32)1 << ldmParams.hashRateLog) - 1;
136
- if (tag == tagMask) {
137
- U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
138
- U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
139
- ldmEntry_t entry;
140
- entry.offset = offset;
141
- entry.checksum = checksum;
142
- ZSTD_ldm_insertEntry(ldmState, hash, entry, ldmParams);
143
- }
144
190
  }
145
191
 
146
192
  /** ZSTD_ldm_countBackwardsMatch() :
@@ -149,10 +195,10 @@ static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
149
195
  * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */
150
196
  static size_t ZSTD_ldm_countBackwardsMatch(
151
197
  const BYTE* pIn, const BYTE* pAnchor,
152
- const BYTE* pMatch, const BYTE* pBase)
198
+ const BYTE* pMatch, const BYTE* pMatchBase)
153
199
  {
154
200
  size_t matchLength = 0;
155
- while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
201
+ while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) {
156
202
  pIn--;
157
203
  pMatch--;
158
204
  matchLength++;
@@ -160,6 +206,27 @@ static size_t ZSTD_ldm_countBackwardsMatch(
160
206
  return matchLength;
161
207
  }
162
208
 
209
+ /** ZSTD_ldm_countBackwardsMatch_2segments() :
210
+ * Returns the number of bytes that match backwards from pMatch,
211
+ * even with the backwards match spanning 2 different segments.
212
+ *
213
+ * On reaching `pMatchBase`, start counting from mEnd */
214
+ static size_t ZSTD_ldm_countBackwardsMatch_2segments(
215
+ const BYTE* pIn, const BYTE* pAnchor,
216
+ const BYTE* pMatch, const BYTE* pMatchBase,
217
+ const BYTE* pExtDictStart, const BYTE* pExtDictEnd)
218
+ {
219
+ size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase);
220
+ if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) {
221
+ /* If backwards match is entirely in the extDict or prefix, immediately return */
222
+ return matchLength;
223
+ }
224
+ DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength);
225
+ matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart);
226
+ DEBUGLOG(7, "final backwards match length = %zu", matchLength);
227
+ return matchLength;
228
+ }
229
+
163
230
  /** ZSTD_ldm_fillFastTables() :
164
231
  *
165
232
  * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies.
@@ -197,30 +264,43 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
197
264
  return 0;
198
265
  }
199
266
 
200
- /** ZSTD_ldm_fillLdmHashTable() :
201
- *
202
- * Fills hashTable from (lastHashed + 1) to iend (non-inclusive).
203
- * lastHash is the rolling hash that corresponds to lastHashed.
204
- *
205
- * Returns the rolling hash corresponding to position iend-1. */
206
- static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state,
207
- U64 lastHash, const BYTE* lastHashed,
208
- const BYTE* iend, const BYTE* base,
209
- U32 hBits, ldmParams_t const ldmParams)
267
+ void ZSTD_ldm_fillHashTable(
268
+ ldmState_t* ldmState, const BYTE* ip,
269
+ const BYTE* iend, ldmParams_t const* params)
210
270
  {
211
- U64 rollingHash = lastHash;
212
- const BYTE* cur = lastHashed + 1;
213
-
214
- while (cur < iend) {
215
- rollingHash = ZSTD_rollingHash_rotate(rollingHash, cur[-1],
216
- cur[ldmParams.minMatchLength-1],
217
- state->hashPower);
218
- ZSTD_ldm_makeEntryAndInsertByTag(state,
219
- rollingHash, hBits,
220
- (U32)(cur - base), ldmParams);
221
- ++cur;
271
+ U32 const minMatchLength = params->minMatchLength;
272
+ U32 const hBits = params->hashLog - params->bucketSizeLog;
273
+ BYTE const* const base = ldmState->window.base;
274
+ BYTE const* const istart = ip;
275
+ ldmRollingHashState_t hashState;
276
+ size_t* const splits = ldmState->splitIndices;
277
+ unsigned numSplits;
278
+
279
+ DEBUGLOG(5, "ZSTD_ldm_fillHashTable");
280
+
281
+ ZSTD_ldm_gear_init(&hashState, params);
282
+ while (ip < iend) {
283
+ size_t hashed;
284
+ unsigned n;
285
+
286
+ numSplits = 0;
287
+ hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits);
288
+
289
+ for (n = 0; n < numSplits; n++) {
290
+ if (ip + splits[n] >= istart + minMatchLength) {
291
+ BYTE const* const split = ip + splits[n] - minMatchLength;
292
+ U64 const xxhash = XXH64(split, minMatchLength, 0);
293
+ U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1));
294
+ ldmEntry_t entry;
295
+
296
+ entry.offset = (U32)(split - base);
297
+ entry.checksum = (U32)(xxhash >> 32);
298
+ ZSTD_ldm_insertEntry(ldmState, hash, entry, *params);
299
+ }
300
+ }
301
+
302
+ ip += hashed;
222
303
  }
223
- return rollingHash;
224
304
  }
225
305
 
226
306
 
@@ -231,10 +311,10 @@ static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state,
231
311
  * (after a long match, only update tables a limited amount). */
232
312
  static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
233
313
  {
234
- U32 const current = (U32)(anchor - ms->window.base);
235
- if (current > ms->nextToUpdate + 1024) {
314
+ U32 const curr = (U32)(anchor - ms->window.base);
315
+ if (curr > ms->nextToUpdate + 1024) {
236
316
  ms->nextToUpdate =
237
- current - MIN(512, current - ms->nextToUpdate - 1024);
317
+ curr - MIN(512, curr - ms->nextToUpdate - 1024);
238
318
  }
239
319
  }
240
320
 
@@ -245,11 +325,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
245
325
  /* LDM parameters */
246
326
  int const extDict = ZSTD_window_hasExtDict(ldmState->window);
247
327
  U32 const minMatchLength = params->minMatchLength;
248
- U64 const hashPower = ldmState->hashPower;
328
+ U32 const entsPerBucket = 1U << params->bucketSizeLog;
249
329
  U32 const hBits = params->hashLog - params->bucketSizeLog;
250
- U32 const ldmBucketSize = 1U << params->bucketSizeLog;
251
- U32 const hashRateLog = params->hashRateLog;
252
- U32 const ldmTagMask = (1U << params->hashRateLog) - 1;
253
330
  /* Prefix and extDict parameters */
254
331
  U32 const dictLimit = ldmState->window.dictLimit;
255
332
  U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
@@ -261,45 +338,69 @@ static size_t ZSTD_ldm_generateSequences_internal(
261
338
  /* Input bounds */
262
339
  BYTE const* const istart = (BYTE const*)src;
263
340
  BYTE const* const iend = istart + srcSize;
264
- BYTE const* const ilimit = iend - MAX(minMatchLength, HASH_READ_SIZE);
341
+ BYTE const* const ilimit = iend - HASH_READ_SIZE;
265
342
  /* Input positions */
266
343
  BYTE const* anchor = istart;
267
344
  BYTE const* ip = istart;
268
- /* Rolling hash */
269
- BYTE const* lastHashed = NULL;
270
- U64 rollingHash = 0;
271
-
272
- while (ip <= ilimit) {
273
- size_t mLength;
274
- U32 const current = (U32)(ip - base);
275
- size_t forwardMatchLength = 0, backwardMatchLength = 0;
276
- ldmEntry_t* bestEntry = NULL;
277
- if (ip != istart) {
278
- rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0],
279
- lastHashed[minMatchLength],
280
- hashPower);
281
- } else {
282
- rollingHash = ZSTD_rollingHash_compute(ip, minMatchLength);
345
+ /* Rolling hash state */
346
+ ldmRollingHashState_t hashState;
347
+ /* Arrays for staged-processing */
348
+ size_t* const splits = ldmState->splitIndices;
349
+ ldmMatchCandidate_t* const candidates = ldmState->matchCandidates;
350
+ unsigned numSplits;
351
+
352
+ if (srcSize < minMatchLength)
353
+ return iend - anchor;
354
+
355
+ /* Initialize the rolling hash state with the first minMatchLength bytes */
356
+ ZSTD_ldm_gear_init(&hashState, params);
357
+ ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength);
358
+ ip += minMatchLength;
359
+
360
+ while (ip < ilimit) {
361
+ size_t hashed;
362
+ unsigned n;
363
+
364
+ numSplits = 0;
365
+ hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip,
366
+ splits, &numSplits);
367
+
368
+ for (n = 0; n < numSplits; n++) {
369
+ BYTE const* const split = ip + splits[n] - minMatchLength;
370
+ U64 const xxhash = XXH64(split, minMatchLength, 0);
371
+ U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1));
372
+
373
+ candidates[n].split = split;
374
+ candidates[n].hash = hash;
375
+ candidates[n].checksum = (U32)(xxhash >> 32);
376
+ candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
377
+ PREFETCH_L1(candidates[n].bucket);
283
378
  }
284
- lastHashed = ip;
285
379
 
286
- /* Do not insert and do not look for a match */
287
- if (ZSTD_ldm_getTag(rollingHash, hBits, hashRateLog) != ldmTagMask) {
288
- ip++;
289
- continue;
290
- }
380
+ for (n = 0; n < numSplits; n++) {
381
+ size_t forwardMatchLength = 0, backwardMatchLength = 0,
382
+ bestMatchLength = 0, mLength;
383
+ U32 offset;
384
+ BYTE const* const split = candidates[n].split;
385
+ U32 const checksum = candidates[n].checksum;
386
+ U32 const hash = candidates[n].hash;
387
+ ldmEntry_t* const bucket = candidates[n].bucket;
388
+ ldmEntry_t const* cur;
389
+ ldmEntry_t const* bestEntry = NULL;
390
+ ldmEntry_t newEntry;
391
+
392
+ newEntry.offset = (U32)(split - base);
393
+ newEntry.checksum = checksum;
394
+
395
+ /* If a split point would generate a sequence overlapping with
396
+ * the previous one, we merely register it in the hash table and
397
+ * move on */
398
+ if (split < anchor) {
399
+ ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
400
+ continue;
401
+ }
291
402
 
292
- /* Get the best entry and compute the match lengths */
293
- {
294
- ldmEntry_t* const bucket =
295
- ZSTD_ldm_getBucket(ldmState,
296
- ZSTD_ldm_getSmallHash(rollingHash, hBits),
297
- *params);
298
- ldmEntry_t* cur;
299
- size_t bestMatchLength = 0;
300
- U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
301
-
302
- for (cur = bucket; cur < bucket + ldmBucketSize; ++cur) {
403
+ for (cur = bucket; cur < bucket + entsPerBucket; cur++) {
303
404
  size_t curForwardMatchLength, curBackwardMatchLength,
304
405
  curTotalMatchLength;
305
406
  if (cur->checksum != checksum || cur->offset <= lowestIndex) {
@@ -313,30 +414,23 @@ static size_t ZSTD_ldm_generateSequences_internal(
313
414
  cur->offset < dictLimit ? dictEnd : iend;
314
415
  BYTE const* const lowMatchPtr =
315
416
  cur->offset < dictLimit ? dictStart : lowPrefixPtr;
316
-
317
- curForwardMatchLength = ZSTD_count_2segments(
318
- ip, pMatch, iend,
319
- matchEnd, lowPrefixPtr);
417
+ curForwardMatchLength =
418
+ ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr);
320
419
  if (curForwardMatchLength < minMatchLength) {
321
420
  continue;
322
421
  }
323
- curBackwardMatchLength =
324
- ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch,
325
- lowMatchPtr);
326
- curTotalMatchLength = curForwardMatchLength +
327
- curBackwardMatchLength;
422
+ curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments(
423
+ split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd);
328
424
  } else { /* !extDict */
329
425
  BYTE const* const pMatch = base + cur->offset;
330
- curForwardMatchLength = ZSTD_count(ip, pMatch, iend);
426
+ curForwardMatchLength = ZSTD_count(split, pMatch, iend);
331
427
  if (curForwardMatchLength < minMatchLength) {
332
428
  continue;
333
429
  }
334
430
  curBackwardMatchLength =
335
- ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch,
336
- lowPrefixPtr);
337
- curTotalMatchLength = curForwardMatchLength +
338
- curBackwardMatchLength;
431
+ ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr);
339
432
  }
433
+ curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength;
340
434
 
341
435
  if (curTotalMatchLength > bestMatchLength) {
342
436
  bestMatchLength = curTotalMatchLength;
@@ -345,57 +439,54 @@ static size_t ZSTD_ldm_generateSequences_internal(
345
439
  bestEntry = cur;
346
440
  }
347
441
  }
348
- }
349
442
 
350
- /* No match found -- continue searching */
351
- if (bestEntry == NULL) {
352
- ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash,
353
- hBits, current,
354
- *params);
355
- ip++;
356
- continue;
357
- }
358
-
359
- /* Match found */
360
- mLength = forwardMatchLength + backwardMatchLength;
361
- ip -= backwardMatchLength;
443
+ /* No match found -- insert an entry into the hash table
444
+ * and process the next candidate match */
445
+ if (bestEntry == NULL) {
446
+ ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
447
+ continue;
448
+ }
362
449
 
363
- {
364
- /* Store the sequence:
365
- * ip = current - backwardMatchLength
366
- * The match is at (bestEntry->offset - backwardMatchLength)
367
- */
368
- U32 const matchIndex = bestEntry->offset;
369
- U32 const offset = current - matchIndex;
370
- rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
371
-
372
- /* Out of sequence storage */
373
- if (rawSeqStore->size == rawSeqStore->capacity)
374
- return ERROR(dstSize_tooSmall);
375
- seq->litLength = (U32)(ip - anchor);
376
- seq->matchLength = (U32)mLength;
377
- seq->offset = offset;
378
- rawSeqStore->size++;
379
- }
450
+ /* Match found */
451
+ offset = (U32)(split - base) - bestEntry->offset;
452
+ mLength = forwardMatchLength + backwardMatchLength;
453
+ {
454
+ rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
455
+
456
+ /* Out of sequence storage */
457
+ if (rawSeqStore->size == rawSeqStore->capacity)
458
+ return ERROR(dstSize_tooSmall);
459
+ seq->litLength = (U32)(split - backwardMatchLength - anchor);
460
+ seq->matchLength = (U32)mLength;
461
+ seq->offset = offset;
462
+ rawSeqStore->size++;
463
+ }
380
464
 
381
- /* Insert the current entry into the hash table */
382
- ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, hBits,
383
- (U32)(lastHashed - base),
384
- *params);
465
+ /* Insert the current entry into the hash table --- it must be
466
+ * done after the previous block to avoid clobbering bestEntry */
467
+ ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
385
468
 
386
- assert(ip + backwardMatchLength == lastHashed);
469
+ anchor = split + forwardMatchLength;
387
470
 
388
- /* Fill the hash table from lastHashed+1 to ip+mLength*/
389
- /* Heuristic: don't need to fill the entire table at end of block */
390
- if (ip + mLength <= ilimit) {
391
- rollingHash = ZSTD_ldm_fillLdmHashTable(
392
- ldmState, rollingHash, lastHashed,
393
- ip + mLength, base, hBits, *params);
394
- lastHashed = ip + mLength - 1;
471
+ /* If we find a match that ends after the data that we've hashed
472
+ * then we have a repeating, overlapping, pattern. E.g. all zeros.
473
+ * If one repetition of the pattern matches our `stopMask` then all
474
+ * repetitions will. We don't need to insert them all into out table,
475
+ * only the first one. So skip over overlapping matches.
476
+ * This is a major speed boost (20x) for compressing a single byte
477
+ * repeated, when that byte ends up in the table.
478
+ */
479
+ if (anchor > ip + hashed) {
480
+ ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength);
481
+ /* Continue the outter loop at anchor (ip + hashed == anchor). */
482
+ ip = anchor - hashed;
483
+ break;
484
+ }
395
485
  }
396
- ip += mLength;
397
- anchor = ip;
486
+
487
+ ip += hashed;
398
488
  }
489
+
399
490
  return iend - anchor;
400
491
  }
401
492
 
@@ -444,11 +535,13 @@ size_t ZSTD_ldm_generateSequences(
444
535
 
445
536
  assert(chunkStart < iend);
446
537
  /* 1. Perform overflow correction if necessary. */
447
- if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) {
538
+ if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) {
448
539
  U32 const ldmHSize = 1U << params->hashLog;
449
540
  U32 const correction = ZSTD_window_correctOverflow(
450
541
  &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart);
451
542
  ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction);
543
+ /* invalidate dictionaries on overflow correction */
544
+ ldmState->loadedDictEnd = 0;
452
545
  }
453
546
  /* 2. We enforce the maximum offset allowed.
454
547
  *
@@ -457,8 +550,14 @@ size_t ZSTD_ldm_generateSequences(
457
550
  * TODO: * Test the chunk size.
458
551
  * * Try invalidation after the sequence generation and test the
459
552
  * the offset against maxDist directly.
553
+ *
554
+ * NOTE: Because of dictionaries + sequence splitting we MUST make sure
555
+ * that any offset used is valid at the END of the sequence, since it may
556
+ * be split into two sequences. This condition holds when using
557
+ * ZSTD_window_enforceMaxDist(), but if we move to checking offsets
558
+ * against maxDist directly, we'll have to carefully handle that case.
460
559
  */
461
- ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, NULL, NULL);
560
+ ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL);
462
561
  /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */
463
562
  newLeftoverSize = ZSTD_ldm_generateSequences_internal(
464
563
  ldmState, sequences, params, chunkStart, chunkSize);
@@ -539,14 +638,32 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
539
638
  return sequence;
540
639
  }
541
640
 
641
+ void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
642
+ U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
643
+ while (currPos && rawSeqStore->pos < rawSeqStore->size) {
644
+ rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
645
+ if (currPos >= currSeq.litLength + currSeq.matchLength) {
646
+ currPos -= currSeq.litLength + currSeq.matchLength;
647
+ rawSeqStore->pos++;
648
+ } else {
649
+ rawSeqStore->posInSequence = currPos;
650
+ break;
651
+ }
652
+ }
653
+ if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
654
+ rawSeqStore->posInSequence = 0;
655
+ }
656
+ }
657
+
542
658
  size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
543
659
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
660
+ ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
544
661
  void const* src, size_t srcSize)
545
662
  {
546
663
  const ZSTD_compressionParameters* const cParams = &ms->cParams;
547
664
  unsigned const minMatch = cParams->minMatch;
548
665
  ZSTD_blockCompressor const blockCompressor =
549
- ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms));
666
+ ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms));
550
667
  /* Input bounds */
551
668
  BYTE const* const istart = (BYTE const*)src;
552
669
  BYTE const* const iend = istart + srcSize;
@@ -554,9 +671,18 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
554
671
  BYTE const* ip = istart;
555
672
 
556
673
  DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize);
674
+ /* If using opt parser, use LDMs only as candidates rather than always accepting them */
675
+ if (cParams->strategy >= ZSTD_btopt) {
676
+ size_t lastLLSize;
677
+ ms->ldmSeqStore = rawSeqStore;
678
+ lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize);
679
+ ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize);
680
+ return lastLLSize;
681
+ }
682
+
557
683
  assert(rawSeqStore->pos <= rawSeqStore->size);
558
684
  assert(rawSeqStore->size <= rawSeqStore->capacity);
559
- /* Loop through each sequence and apply the block compressor to the lits */
685
+ /* Loop through each sequence and apply the block compressor to the literals */
560
686
  while (rawSeqStore->pos < rawSeqStore->size && ip < iend) {
561
687
  /* maybeSplitSequence updates rawSeqStore->pos */
562
688
  rawSeq const sequence = maybeSplitSequence(rawSeqStore,
@@ -566,14 +692,13 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
566
692
  if (sequence.offset == 0)
567
693
  break;
568
694
 
569
- assert(sequence.offset <= (1U << cParams->windowLog));
570
695
  assert(ip + sequence.litLength + sequence.matchLength <= iend);
571
696
 
572
697
  /* Fill tables for block compressor */
573
698
  ZSTD_ldm_limitTableUpdate(ms, ip);
574
699
  ZSTD_ldm_fillFastTables(ms, ip);
575
700
  /* Run the block compressor */
576
- DEBUGLOG(5, "calling block compressor on segment of size %u", sequence.litLength);
701
+ DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength);
577
702
  {
578
703
  size_t const newLitLength =
579
704
  blockCompressor(ms, seqStore, rep, ip, sequence.litLength);
@@ -583,7 +708,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
583
708
  rep[i] = rep[i-1];
584
709
  rep[0] = sequence.offset;
585
710
  /* Store the sequence */
586
- ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength,
711
+ ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend,
587
712
  sequence.offset + ZSTD_REP_MOVE,
588
713
  sequence.matchLength - MINMATCH);
589
714
  ip += sequence.matchLength;