zstdlib 0.1.2-x86-mingw32 → 0.2.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.yardopts +6 -0
  3. data/CHANGES.md +7 -0
  4. data/README.md +4 -3
  5. data/ext/zstdlib/extconf.rb +1 -1
  6. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/bitstream.h +0 -0
  7. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/compiler.h +1 -1
  8. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/cpu.h +0 -0
  9. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/debug.c +0 -0
  10. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/debug.h +0 -0
  11. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/entropy_common.c +0 -0
  12. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/error_private.c +0 -0
  13. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/error_private.h +0 -0
  14. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/fse.h +1 -1
  15. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/fse_decompress.c +0 -0
  16. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/huf.h +0 -0
  17. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/mem.h +0 -0
  18. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/pool.c +0 -0
  19. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/pool.h +0 -0
  20. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/threading.c +2 -2
  21. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/threading.h +0 -0
  22. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/xxhash.c +2 -2
  23. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/xxhash.h +0 -0
  24. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/zstd_common.c +0 -0
  25. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/zstd_errors.h +0 -0
  26. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/common/zstd_internal.h +55 -2
  27. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/fse_compress.c +2 -2
  28. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/hist.c +0 -0
  29. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/hist.h +0 -0
  30. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/huf_compress.c +0 -0
  31. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_compress.c +423 -296
  32. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_compress_internal.h +14 -11
  33. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_double_fast.c +0 -0
  34. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_double_fast.h +0 -0
  35. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_fast.c +203 -124
  36. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_fast.h +0 -0
  37. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_lazy.c +0 -0
  38. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_lazy.h +1 -1
  39. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_ldm.c +1 -1
  40. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_ldm.h +0 -0
  41. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_opt.c +27 -11
  42. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstd_opt.h +0 -0
  43. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstdmt_compress.c +41 -49
  44. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/compress/zstdmt_compress.h +43 -26
  45. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/decompress/huf_decompress.c +0 -0
  46. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/decompress/zstd_ddict.c +4 -4
  47. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/decompress/zstd_ddict.h +0 -0
  48. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/decompress/zstd_decompress.c +257 -164
  49. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/decompress/zstd_decompress_block.c +51 -47
  50. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/decompress/zstd_decompress_block.h +0 -0
  51. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/decompress/zstd_decompress_internal.h +7 -0
  52. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/lib/zstd.h +689 -542
  53. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/zlibWrapper/gzclose.c +0 -0
  54. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/zlibWrapper/gzcompatibility.h +0 -0
  55. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/zlibWrapper/gzguts.h +2 -0
  56. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/zlibWrapper/gzlib.c +0 -0
  57. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/zlibWrapper/gzread.c +0 -0
  58. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/zlibWrapper/gzwrite.c +0 -0
  59. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/zlibWrapper/zstd_zlibwrapper.c +0 -0
  60. data/ext/zstdlib/{zstd-1.3.8 → zstd-1.4.0}/zlibWrapper/zstd_zlibwrapper.h +0 -0
  61. data/lib/2.2/zstdlib.so +0 -0
  62. data/lib/2.3/zstdlib.so +0 -0
  63. data/lib/2.4/zstdlib.so +0 -0
  64. data/lib/2.5/zstdlib.so +0 -0
  65. data/lib/2.6/zstdlib.so +0 -0
  66. metadata +58 -57
@@ -36,9 +36,9 @@ extern "C" {
36
36
  #define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index 1 now means "unsorted".
37
37
  It could be confused for a real successor at index "1", if sorted as larger than its predecessor.
38
38
  It's not a big deal though : candidate will just be sorted again.
39
- Additionnally, candidate position 1 will be lost.
39
+ Additionally, candidate position 1 will be lost.
40
40
  But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss.
41
- The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be misdhandled after table re-use with a different strategy
41
+ The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy
42
42
  Constant required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */
43
43
 
44
44
 
@@ -54,6 +54,14 @@ typedef struct ZSTD_prefixDict_s {
54
54
  ZSTD_dictContentType_e dictContentType;
55
55
  } ZSTD_prefixDict;
56
56
 
57
+ typedef struct {
58
+ void* dictBuffer;
59
+ void const* dict;
60
+ size_t dictSize;
61
+ ZSTD_dictContentType_e dictContentType;
62
+ ZSTD_CDict* cdict;
63
+ } ZSTD_localDict;
64
+
57
65
  typedef struct {
58
66
  U32 CTable[HUF_CTABLE_SIZE_U32(255)];
59
67
  HUF_repeat repeatMode;
@@ -107,6 +115,7 @@ typedef struct {
107
115
  U32 offCodeSumBasePrice; /* to compare to log2(offreq) */
108
116
  ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */
109
117
  const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */
118
+ ZSTD_literalCompressionMode_e literalCompressionMode;
110
119
  } optState_t;
111
120
 
112
121
  typedef struct {
@@ -188,6 +197,7 @@ struct ZSTD_CCtx_params_s {
188
197
  * 1<<wLog, even for dictionary */
189
198
 
190
199
  ZSTD_dictAttachPref_e attachDictPref;
200
+ ZSTD_literalCompressionMode_e literalCompressionMode;
191
201
 
192
202
  /* Multithreading: used to pass parameters to mtctx */
193
203
  int nbWorkers;
@@ -243,7 +253,7 @@ struct ZSTD_CCtx_s {
243
253
  U32 frameEnded;
244
254
 
245
255
  /* Dictionary */
246
- ZSTD_CDict* cdictLocal;
256
+ ZSTD_localDict localDict;
247
257
  const ZSTD_CDict* cdict;
248
258
  ZSTD_prefixDict prefixDict; /* single-usage dictionary */
249
259
 
@@ -806,13 +816,6 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
806
816
 
807
817
  void ZSTD_resetSeqStore(seqStore_t* ssPtr);
808
818
 
809
- /*! ZSTD_compressStream_generic() :
810
- * Private use only. To be called from zstdmt_compress.c in single-thread mode. */
811
- size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
812
- ZSTD_outBuffer* output,
813
- ZSTD_inBuffer* input,
814
- ZSTD_EndDirective const flushMode);
815
-
816
819
  /*! ZSTD_getCParamsFromCDict() :
817
820
  * as the name implies */
818
821
  ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
@@ -839,7 +842,7 @@ size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
839
842
  /* ZSTD_writeLastEmptyBlock() :
840
843
  * output an empty Block with end-of-frame mark to complete a frame
841
844
  * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h))
842
- * or an error code if `dstCapcity` is too small (<ZSTD_blockHeaderSize)
845
+ * or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize)
843
846
  */
844
847
  size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity);
845
848
 
@@ -45,7 +45,155 @@ FORCE_INLINE_TEMPLATE
45
45
  size_t ZSTD_compressBlock_fast_generic(
46
46
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
47
47
  void const* src, size_t srcSize,
48
- U32 const mls, ZSTD_dictMode_e const dictMode)
48
+ U32 const mls)
49
+ {
50
+ const ZSTD_compressionParameters* const cParams = &ms->cParams;
51
+ U32* const hashTable = ms->hashTable;
52
+ U32 const hlog = cParams->hashLog;
53
+ /* support stepSize of 0 */
54
+ size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1;
55
+ const BYTE* const base = ms->window.base;
56
+ const BYTE* const istart = (const BYTE*)src;
57
+ /* We check ip0 (ip + 0) and ip1 (ip + 1) each loop */
58
+ const BYTE* ip0 = istart;
59
+ const BYTE* ip1;
60
+ const BYTE* anchor = istart;
61
+ const U32 prefixStartIndex = ms->window.dictLimit;
62
+ const BYTE* const prefixStart = base + prefixStartIndex;
63
+ const BYTE* const iend = istart + srcSize;
64
+ const BYTE* const ilimit = iend - HASH_READ_SIZE;
65
+ U32 offset_1=rep[0], offset_2=rep[1];
66
+ U32 offsetSaved = 0;
67
+
68
+ /* init */
69
+ ip0 += (ip0 == prefixStart);
70
+ ip1 = ip0 + 1;
71
+ {
72
+ U32 const maxRep = (U32)(ip0 - prefixStart);
73
+ if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
74
+ if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
75
+ }
76
+
77
+ /* Main Search Loop */
78
+ while (ip1 < ilimit) { /* < instead of <=, because check at ip0+2 */
79
+ size_t mLength;
80
+ BYTE const* ip2 = ip0 + 2;
81
+ size_t const h0 = ZSTD_hashPtr(ip0, hlog, mls);
82
+ U32 const val0 = MEM_read32(ip0);
83
+ size_t const h1 = ZSTD_hashPtr(ip1, hlog, mls);
84
+ U32 const val1 = MEM_read32(ip1);
85
+ U32 const current0 = (U32)(ip0-base);
86
+ U32 const current1 = (U32)(ip1-base);
87
+ U32 const matchIndex0 = hashTable[h0];
88
+ U32 const matchIndex1 = hashTable[h1];
89
+ BYTE const* repMatch = ip2-offset_1;
90
+ const BYTE* match0 = base + matchIndex0;
91
+ const BYTE* match1 = base + matchIndex1;
92
+ U32 offcode;
93
+ hashTable[h0] = current0; /* update hash table */
94
+ hashTable[h1] = current1; /* update hash table */
95
+
96
+ assert(ip0 + 1 == ip1);
97
+
98
+ if ((offset_1 > 0) & (MEM_read32(repMatch) == MEM_read32(ip2))) {
99
+ mLength = ip2[-1] == repMatch[-1] ? 1 : 0;
100
+ ip0 = ip2 - mLength;
101
+ match0 = repMatch - mLength;
102
+ offcode = 0;
103
+ goto _match;
104
+ }
105
+ if ((matchIndex0 > prefixStartIndex) && MEM_read32(match0) == val0) {
106
+ /* found a regular match */
107
+ goto _offset;
108
+ }
109
+ if ((matchIndex1 > prefixStartIndex) && MEM_read32(match1) == val1) {
110
+ /* found a regular match after one literal */
111
+ ip0 = ip1;
112
+ match0 = match1;
113
+ goto _offset;
114
+ }
115
+ {
116
+ size_t const step = ((ip0-anchor) >> (kSearchStrength - 1)) + stepSize;
117
+ assert(step >= 2);
118
+ ip0 += step;
119
+ ip1 += step;
120
+ continue;
121
+ }
122
+ _offset: /* Requires: ip0, match0 */
123
+ /* Compute the offset code */
124
+ offset_2 = offset_1;
125
+ offset_1 = (U32)(ip0-match0);
126
+ offcode = offset_1 + ZSTD_REP_MOVE;
127
+ mLength = 0;
128
+ /* Count the backwards match length */
129
+ while (((ip0>anchor) & (match0>prefixStart))
130
+ && (ip0[-1] == match0[-1])) { ip0--; match0--; mLength++; } /* catch up */
131
+
132
+ _match: /* Requires: ip0, match0, offcode */
133
+ /* Count the forward length */
134
+ mLength += ZSTD_count(ip0+mLength+4, match0+mLength+4, iend) + 4;
135
+ ZSTD_storeSeq(seqStore, ip0-anchor, anchor, offcode, mLength-MINMATCH);
136
+ /* match found */
137
+ ip0 += mLength;
138
+ anchor = ip0;
139
+ ip1 = ip0 + 1;
140
+
141
+ if (ip0 <= ilimit) {
142
+ /* Fill Table */
143
+ assert(base+current0+2 > istart); /* check base overflow */
144
+ hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */
145
+ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
146
+
147
+ while ( (ip0 <= ilimit)
148
+ && ( (offset_2>0)
149
+ & (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) )) {
150
+ /* store sequence */
151
+ size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4;
152
+ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
153
+ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
154
+ ip0 += rLength;
155
+ ip1 = ip0 + 1;
156
+ ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
157
+ anchor = ip0;
158
+ continue; /* faster when present (confirmed on gcc-8) ... (?) */
159
+ }
160
+ }
161
+ }
162
+
163
+ /* save reps for next block */
164
+ rep[0] = offset_1 ? offset_1 : offsetSaved;
165
+ rep[1] = offset_2 ? offset_2 : offsetSaved;
166
+
167
+ /* Return the last literals size */
168
+ return iend - anchor;
169
+ }
170
+
171
+
172
+ size_t ZSTD_compressBlock_fast(
173
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
174
+ void const* src, size_t srcSize)
175
+ {
176
+ ZSTD_compressionParameters const* cParams = &ms->cParams;
177
+ U32 const mls = cParams->minMatch;
178
+ assert(ms->dictMatchState == NULL);
179
+ switch(mls)
180
+ {
181
+ default: /* includes case 3 */
182
+ case 4 :
183
+ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4);
184
+ case 5 :
185
+ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5);
186
+ case 6 :
187
+ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6);
188
+ case 7 :
189
+ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7);
190
+ }
191
+ }
192
+
193
+ FORCE_INLINE_TEMPLATE
194
+ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
195
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
196
+ void const* src, size_t srcSize, U32 const mls)
49
197
  {
50
198
  const ZSTD_compressionParameters* const cParams = &ms->cParams;
51
199
  U32* const hashTable = ms->hashTable;
@@ -64,46 +212,26 @@ size_t ZSTD_compressBlock_fast_generic(
64
212
  U32 offsetSaved = 0;
65
213
 
66
214
  const ZSTD_matchState_t* const dms = ms->dictMatchState;
67
- const ZSTD_compressionParameters* const dictCParams =
68
- dictMode == ZSTD_dictMatchState ?
69
- &dms->cParams : NULL;
70
- const U32* const dictHashTable = dictMode == ZSTD_dictMatchState ?
71
- dms->hashTable : NULL;
72
- const U32 dictStartIndex = dictMode == ZSTD_dictMatchState ?
73
- dms->window.dictLimit : 0;
74
- const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ?
75
- dms->window.base : NULL;
76
- const BYTE* const dictStart = dictMode == ZSTD_dictMatchState ?
77
- dictBase + dictStartIndex : NULL;
78
- const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ?
79
- dms->window.nextSrc : NULL;
80
- const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ?
81
- prefixStartIndex - (U32)(dictEnd - dictBase) :
82
- 0;
215
+ const ZSTD_compressionParameters* const dictCParams = &dms->cParams ;
216
+ const U32* const dictHashTable = dms->hashTable;
217
+ const U32 dictStartIndex = dms->window.dictLimit;
218
+ const BYTE* const dictBase = dms->window.base;
219
+ const BYTE* const dictStart = dictBase + dictStartIndex;
220
+ const BYTE* const dictEnd = dms->window.nextSrc;
221
+ const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase);
83
222
  const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart);
84
- const U32 dictHLog = dictMode == ZSTD_dictMatchState ?
85
- dictCParams->hashLog : hlog;
86
-
87
- assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState);
223
+ const U32 dictHLog = dictCParams->hashLog;
88
224
 
89
225
  /* otherwise, we would get index underflow when translating a dict index
90
226
  * into a local index */
91
- assert(dictMode != ZSTD_dictMatchState
92
- || prefixStartIndex >= (U32)(dictEnd - dictBase));
227
+ assert(prefixStartIndex >= (U32)(dictEnd - dictBase));
93
228
 
94
229
  /* init */
95
230
  ip += (dictAndPrefixLength == 0);
96
- if (dictMode == ZSTD_noDict) {
97
- U32 const maxRep = (U32)(ip - prefixStart);
98
- if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
99
- if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
100
- }
101
- if (dictMode == ZSTD_dictMatchState) {
102
- /* dictMatchState repCode checks don't currently handle repCode == 0
103
- * disabling. */
104
- assert(offset_1 <= dictAndPrefixLength);
105
- assert(offset_2 <= dictAndPrefixLength);
106
- }
231
+ /* dictMatchState repCode checks don't currently handle repCode == 0
232
+ * disabling. */
233
+ assert(offset_1 <= dictAndPrefixLength);
234
+ assert(offset_2 <= dictAndPrefixLength);
107
235
 
108
236
  /* Main Search Loop */
109
237
  while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
@@ -113,50 +241,37 @@ size_t ZSTD_compressBlock_fast_generic(
113
241
  U32 const matchIndex = hashTable[h];
114
242
  const BYTE* match = base + matchIndex;
115
243
  const U32 repIndex = current + 1 - offset_1;
116
- const BYTE* repMatch = (dictMode == ZSTD_dictMatchState
117
- && repIndex < prefixStartIndex) ?
244
+ const BYTE* repMatch = (repIndex < prefixStartIndex) ?
118
245
  dictBase + (repIndex - dictIndexDelta) :
119
246
  base + repIndex;
120
247
  hashTable[h] = current; /* update hash table */
121
248
 
122
- if ( (dictMode == ZSTD_dictMatchState)
123
- && ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
249
+ if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
124
250
  && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
125
251
  const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
126
252
  mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
127
253
  ip++;
128
254
  ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
129
- } else if ( dictMode == ZSTD_noDict
130
- && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) {
131
- mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
132
- ip++;
133
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
134
255
  } else if ( (matchIndex <= prefixStartIndex) ) {
135
- if (dictMode == ZSTD_dictMatchState) {
136
- size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls);
137
- U32 const dictMatchIndex = dictHashTable[dictHash];
138
- const BYTE* dictMatch = dictBase + dictMatchIndex;
139
- if (dictMatchIndex <= dictStartIndex ||
140
- MEM_read32(dictMatch) != MEM_read32(ip)) {
141
- assert(stepSize >= 1);
142
- ip += ((ip-anchor) >> kSearchStrength) + stepSize;
143
- continue;
144
- } else {
145
- /* found a dict match */
146
- U32 const offset = (U32)(current-dictMatchIndex-dictIndexDelta);
147
- mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4;
148
- while (((ip>anchor) & (dictMatch>dictStart))
149
- && (ip[-1] == dictMatch[-1])) {
150
- ip--; dictMatch--; mLength++;
151
- } /* catch up */
152
- offset_2 = offset_1;
153
- offset_1 = offset;
154
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
155
- }
156
- } else {
256
+ size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls);
257
+ U32 const dictMatchIndex = dictHashTable[dictHash];
258
+ const BYTE* dictMatch = dictBase + dictMatchIndex;
259
+ if (dictMatchIndex <= dictStartIndex ||
260
+ MEM_read32(dictMatch) != MEM_read32(ip)) {
157
261
  assert(stepSize >= 1);
158
262
  ip += ((ip-anchor) >> kSearchStrength) + stepSize;
159
263
  continue;
264
+ } else {
265
+ /* found a dict match */
266
+ U32 const offset = (U32)(current-dictMatchIndex-dictIndexDelta);
267
+ mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4;
268
+ while (((ip>anchor) & (dictMatch>dictStart))
269
+ && (ip[-1] == dictMatch[-1])) {
270
+ ip--; dictMatch--; mLength++;
271
+ } /* catch up */
272
+ offset_2 = offset_1;
273
+ offset_1 = offset;
274
+ ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
160
275
  }
161
276
  } else if (MEM_read32(match) != MEM_read32(ip)) {
162
277
  /* it's not a match, and we're not going to check the dictionary */
@@ -185,41 +300,27 @@ size_t ZSTD_compressBlock_fast_generic(
185
300
  hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
186
301
 
187
302
  /* check immediate repcode */
188
- if (dictMode == ZSTD_dictMatchState) {
189
- while (ip <= ilimit) {
190
- U32 const current2 = (U32)(ip-base);
191
- U32 const repIndex2 = current2 - offset_2;
192
- const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
193
- dictBase - dictIndexDelta + repIndex2 :
194
- base + repIndex2;
195
- if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
196
- && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
197
- const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
198
- size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
199
- U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
200
- ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
201
- hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
202
- ip += repLength2;
203
- anchor = ip;
204
- continue;
205
- }
206
- break;
303
+ while (ip <= ilimit) {
304
+ U32 const current2 = (U32)(ip-base);
305
+ U32 const repIndex2 = current2 - offset_2;
306
+ const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
307
+ dictBase - dictIndexDelta + repIndex2 :
308
+ base + repIndex2;
309
+ if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
310
+ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
311
+ const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
312
+ size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
313
+ U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
314
+ ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH);
315
+ hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
316
+ ip += repLength2;
317
+ anchor = ip;
318
+ continue;
207
319
  }
320
+ break;
208
321
  }
209
-
210
- if (dictMode == ZSTD_noDict) {
211
- while ( (ip <= ilimit)
212
- && ( (offset_2>0)
213
- & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
214
- /* store sequence */
215
- size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
216
- U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
217
- hashTable[ZSTD_hashPtr(ip, hlog, mls)] = (U32)(ip-base);
218
- ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH);
219
- ip += rLength;
220
- anchor = ip;
221
- continue; /* faster when present ... (?) */
222
- } } } }
322
+ }
323
+ }
223
324
 
224
325
  /* save reps for next block */
225
326
  rep[0] = offset_1 ? offset_1 : offsetSaved;
@@ -229,28 +330,6 @@ size_t ZSTD_compressBlock_fast_generic(
229
330
  return iend - anchor;
230
331
  }
231
332
 
232
-
233
- size_t ZSTD_compressBlock_fast(
234
- ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
235
- void const* src, size_t srcSize)
236
- {
237
- ZSTD_compressionParameters const* cParams = &ms->cParams;
238
- U32 const mls = cParams->minMatch;
239
- assert(ms->dictMatchState == NULL);
240
- switch(mls)
241
- {
242
- default: /* includes case 3 */
243
- case 4 :
244
- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict);
245
- case 5 :
246
- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict);
247
- case 6 :
248
- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict);
249
- case 7 :
250
- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict);
251
- }
252
- }
253
-
254
333
  size_t ZSTD_compressBlock_fast_dictMatchState(
255
334
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
256
335
  void const* src, size_t srcSize)
@@ -262,13 +341,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
262
341
  {
263
342
  default: /* includes case 3 */
264
343
  case 4 :
265
- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState);
344
+ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4);
266
345
  case 5 :
267
- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState);
346
+ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5);
268
347
  case 6 :
269
- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState);
348
+ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6);
270
349
  case 7 :
271
- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState);
350
+ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7);
272
351
  }
273
352
  }
274
353
 
@@ -19,7 +19,7 @@ extern "C" {
19
19
 
20
20
  U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
21
21
 
22
- void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). pre-emptively increase value of ZSTD_DUBT_UNSORTED_MARK */
22
+ void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */
23
23
 
24
24
  size_t ZSTD_compressBlock_btlazy2(
25
25
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -429,7 +429,7 @@ size_t ZSTD_ldm_generateSequences(
429
429
  */
430
430
  assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize);
431
431
  /* The input could be very large (in zstdmt), so it must be broken up into
432
- * chunks to enforce the maximmum distance and handle overflow correction.
432
+ * chunks to enforce the maximum distance and handle overflow correction.
433
433
  */
434
434
  assert(sequences->pos <= sequences->size);
435
435
  assert(sequences->size <= sequences->capacity);
@@ -64,9 +64,15 @@ MEM_STATIC double ZSTD_fCost(U32 price)
64
64
  }
65
65
  #endif
66
66
 
67
+ static int ZSTD_compressedLiterals(optState_t const* const optPtr)
68
+ {
69
+ return optPtr->literalCompressionMode != ZSTD_lcm_uncompressed;
70
+ }
71
+
67
72
  static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
68
73
  {
69
- optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel);
74
+ if (ZSTD_compressedLiterals(optPtr))
75
+ optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel);
70
76
  optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel);
71
77
  optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel);
72
78
  optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel);
@@ -99,6 +105,7 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
99
105
  const BYTE* const src, size_t const srcSize,
100
106
  int const optLevel)
101
107
  {
108
+ int const compressedLiterals = ZSTD_compressedLiterals(optPtr);
102
109
  DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
103
110
  optPtr->priceType = zop_dynamic;
104
111
 
@@ -113,9 +120,10 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
113
120
  /* huffman table presumed generated by dictionary */
114
121
  optPtr->priceType = zop_dynamic;
115
122
 
116
- assert(optPtr->litFreq != NULL);
117
- optPtr->litSum = 0;
118
- { unsigned lit;
123
+ if (compressedLiterals) {
124
+ unsigned lit;
125
+ assert(optPtr->litFreq != NULL);
126
+ optPtr->litSum = 0;
119
127
  for (lit=0; lit<=MaxLit; lit++) {
120
128
  U32 const scaleLog = 11; /* scale to 2K */
121
129
  U32 const bitCost = HUF_getNbBits(optPtr->symbolCosts->huf.CTable, lit);
@@ -163,10 +171,11 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
163
171
  } else { /* not a dictionary */
164
172
 
165
173
  assert(optPtr->litFreq != NULL);
166
- { unsigned lit = MaxLit;
174
+ if (compressedLiterals) {
175
+ unsigned lit = MaxLit;
167
176
  HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */
177
+ optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1);
168
178
  }
169
- optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1);
170
179
 
171
180
  { unsigned ll;
172
181
  for (ll=0; ll<=MaxLL; ll++)
@@ -190,7 +199,8 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
190
199
 
191
200
  } else { /* new block : re-use previous statistics, scaled down */
192
201
 
193
- optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1);
202
+ if (compressedLiterals)
203
+ optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1);
194
204
  optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0);
195
205
  optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0);
196
206
  optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0);
@@ -207,6 +217,10 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
207
217
  int optLevel)
208
218
  {
209
219
  if (litLength == 0) return 0;
220
+
221
+ if (!ZSTD_compressedLiterals(optPtr))
222
+ return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */
223
+
210
224
  if (optPtr->priceType == zop_predef)
211
225
  return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */
212
226
 
@@ -310,7 +324,8 @@ static void ZSTD_updateStats(optState_t* const optPtr,
310
324
  U32 offsetCode, U32 matchLength)
311
325
  {
312
326
  /* literals */
313
- { U32 u;
327
+ if (ZSTD_compressedLiterals(optPtr)) {
328
+ U32 u;
314
329
  for (u=0; u < litLength; u++)
315
330
  optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
316
331
  optPtr->litSum += litLength*ZSTD_LITFREQ_ADD;
@@ -870,7 +885,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
870
885
  /* large match -> immediate encoding */
871
886
  { U32 const maxML = matches[nbMatches-1].len;
872
887
  U32 const maxOffset = matches[nbMatches-1].off;
873
- DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new serie",
888
+ DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series",
874
889
  nbMatches, maxML, maxOffset, (U32)(ip-prefixStart));
875
890
 
876
891
  if (maxML > sufficient_len) {
@@ -1108,7 +1123,8 @@ static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus)
1108
1123
  /* used in 2-pass strategy */
1109
1124
  MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
1110
1125
  {
1111
- optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0);
1126
+ if (ZSTD_compressedLiterals(optPtr))
1127
+ optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0);
1112
1128
  optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0);
1113
1129
  optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0);
1114
1130
  optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0);
@@ -1117,7 +1133,7 @@ MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
1117
1133
  /* ZSTD_initStats_ultra():
1118
1134
  * make a first compression pass, just to seed stats with more accurate starting values.
1119
1135
  * only works on first block, with no dictionary and no ldm.
1120
- * this function cannot error, hence its constract must be respected.
1136
+ * this function cannot error, hence its contract must be respected.
1121
1137
  */
1122
1138
  static void
1123
1139
  ZSTD_initStats_ultra(ZSTD_matchState_t* ms,