zstdlib 0.7.0 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +5 -0
  3. data/ext/zstdlib/extconf.rb +1 -1
  4. data/ext/zstdlib/ruby/zlib-3.0/zstdlib.c +4994 -0
  5. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/bitstream.h +25 -16
  6. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/compiler.h +118 -4
  7. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/cpu.h +1 -3
  8. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/debug.c +1 -1
  9. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/debug.h +12 -19
  10. data/ext/zstdlib/zstd-1.5.0/lib/common/entropy_common.c +362 -0
  11. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/error_private.c +2 -1
  12. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/error_private.h +3 -3
  13. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/fse.h +40 -12
  14. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/fse_decompress.c +139 -22
  15. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/huf.h +29 -7
  16. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/mem.h +69 -98
  17. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/pool.c +23 -17
  18. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/pool.h +2 -2
  19. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/threading.c +6 -5
  20. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/threading.h +0 -0
  21. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/xxhash.c +20 -60
  22. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/xxhash.h +2 -2
  23. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/zstd_common.c +10 -10
  24. data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_deps.h +111 -0
  25. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/zstd_internal.h +105 -62
  26. data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_trace.h +154 -0
  27. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/fse_compress.c +31 -24
  28. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/hist.c +27 -29
  29. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/hist.h +2 -2
  30. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/huf_compress.c +265 -126
  31. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress.c +2843 -728
  32. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_internal.h +305 -63
  33. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_literals.c +8 -8
  34. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_literals.h +1 -1
  35. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.c +29 -7
  36. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.h +1 -1
  37. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_superblock.c +22 -295
  38. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_superblock.h +1 -1
  39. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_cwksp.h +204 -67
  40. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_double_fast.c +25 -25
  41. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_double_fast.h +1 -1
  42. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_fast.c +23 -23
  43. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_fast.h +1 -1
  44. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.c +2184 -0
  45. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.h +125 -0
  46. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_ldm.c +314 -211
  47. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_ldm.h +9 -2
  48. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_ldm_geartab.h +103 -0
  49. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_opt.c +191 -46
  50. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_opt.h +1 -1
  51. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstdmt_compress.c +93 -415
  52. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstdmt_compress.h +110 -0
  53. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/huf_decompress.c +342 -239
  54. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_ddict.c +9 -9
  55. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_ddict.h +2 -2
  56. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress.c +369 -87
  57. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.c +191 -75
  58. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.h +6 -3
  59. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress_internal.h +27 -11
  60. data/ext/zstdlib/zstd-1.5.0/lib/zdict.h +452 -0
  61. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/zstd.h +568 -126
  62. data/ext/zstdlib/{zstd-1.4.5/lib/common → zstd-1.5.0/lib}/zstd_errors.h +2 -1
  63. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzclose.c +0 -0
  64. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzcompatibility.h +1 -1
  65. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzguts.h +0 -0
  66. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzlib.c +0 -0
  67. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzread.c +0 -0
  68. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzwrite.c +0 -0
  69. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.c +126 -44
  70. data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  71. metadata +70 -65
  72. data/ext/zstdlib/zstd-1.4.5/lib/common/entropy_common.c +0 -216
  73. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.c +0 -1138
  74. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.h +0 -67
  75. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstdmt_compress.h +0 -192
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -28,7 +28,6 @@
28
28
  extern "C" {
29
29
  #endif
30
30
 
31
-
32
31
  /*-*************************************
33
32
  * Constants
34
33
  ***************************************/
@@ -64,7 +63,7 @@ typedef struct {
64
63
  } ZSTD_localDict;
65
64
 
66
65
  typedef struct {
67
- U32 CTable[HUF_CTABLE_SIZE_U32(255)];
66
+ HUF_CElt CTable[HUF_CTABLE_SIZE_U32(255)];
68
67
  HUF_repeat repeatMode;
69
68
  } ZSTD_hufCTables_t;
70
69
 
@@ -82,11 +81,75 @@ typedef struct {
82
81
  ZSTD_fseCTables_t fse;
83
82
  } ZSTD_entropyCTables_t;
84
83
 
84
+ /***********************************************
85
+ * Entropy buffer statistics structs and funcs *
86
+ ***********************************************/
87
+ /** ZSTD_hufCTablesMetadata_t :
88
+ * Stores Literals Block Type for a super-block in hType, and
89
+ * huffman tree description in hufDesBuffer.
90
+ * hufDesSize refers to the size of huffman tree description in bytes.
91
+ * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */
85
92
  typedef struct {
86
- U32 off;
87
- U32 len;
93
+ symbolEncodingType_e hType;
94
+ BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
95
+ size_t hufDesSize;
96
+ } ZSTD_hufCTablesMetadata_t;
97
+
98
+ /** ZSTD_fseCTablesMetadata_t :
99
+ * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
100
+ * fse tables in fseTablesBuffer.
101
+ * fseTablesSize refers to the size of fse tables in bytes.
102
+ * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */
103
+ typedef struct {
104
+ symbolEncodingType_e llType;
105
+ symbolEncodingType_e ofType;
106
+ symbolEncodingType_e mlType;
107
+ BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE];
108
+ size_t fseTablesSize;
109
+ size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
110
+ } ZSTD_fseCTablesMetadata_t;
111
+
112
+ typedef struct {
113
+ ZSTD_hufCTablesMetadata_t hufMetadata;
114
+ ZSTD_fseCTablesMetadata_t fseMetadata;
115
+ } ZSTD_entropyCTablesMetadata_t;
116
+
117
+ /** ZSTD_buildBlockEntropyStats() :
118
+ * Builds entropy for the block.
119
+ * @return : 0 on success or error code */
120
+ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
121
+ const ZSTD_entropyCTables_t* prevEntropy,
122
+ ZSTD_entropyCTables_t* nextEntropy,
123
+ const ZSTD_CCtx_params* cctxParams,
124
+ ZSTD_entropyCTablesMetadata_t* entropyMetadata,
125
+ void* workspace, size_t wkspSize);
126
+
127
+ /*********************************
128
+ * Compression internals structs *
129
+ *********************************/
130
+
131
+ typedef struct {
132
+ U32 off; /* Offset code (offset + ZSTD_REP_MOVE) for the match */
133
+ U32 len; /* Raw length of match */
88
134
  } ZSTD_match_t;
89
135
 
136
+ typedef struct {
137
+ U32 offset; /* Offset of sequence */
138
+ U32 litLength; /* Length of literals prior to match */
139
+ U32 matchLength; /* Raw length of match */
140
+ } rawSeq;
141
+
142
+ typedef struct {
143
+ rawSeq* seq; /* The start of the sequences */
144
+ size_t pos; /* The index in seq where reading stopped. pos <= size. */
145
+ size_t posInSequence; /* The position within the sequence at seq[pos] where reading
146
+ stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */
147
+ size_t size; /* The number of sequences. <= capacity. */
148
+ size_t capacity; /* The capacity starting from `seq` pointer */
149
+ } rawSeqStore_t;
150
+
151
+ UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
152
+
90
153
  typedef struct {
91
154
  int price;
92
155
  U32 off;
@@ -125,14 +188,21 @@ typedef struct {
125
188
  } ZSTD_compressedBlockState_t;
126
189
 
127
190
  typedef struct {
128
- BYTE const* nextSrc; /* next block here to continue on current prefix */
129
- BYTE const* base; /* All regular indexes relative to this position */
130
- BYTE const* dictBase; /* extDict indexes relative to this position */
131
- U32 dictLimit; /* below that point, need extDict */
132
- U32 lowLimit; /* below that point, no more valid data */
191
+ BYTE const* nextSrc; /* next block here to continue on current prefix */
192
+ BYTE const* base; /* All regular indexes relative to this position */
193
+ BYTE const* dictBase; /* extDict indexes relative to this position */
194
+ U32 dictLimit; /* below that point, need extDict */
195
+ U32 lowLimit; /* below that point, no more valid data */
196
+ U32 nbOverflowCorrections; /* Number of times overflow correction has run since
197
+ * ZSTD_window_init(). Useful for debugging coredumps
198
+ * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY.
199
+ */
133
200
  } ZSTD_window_t;
134
201
 
135
202
  typedef struct ZSTD_matchState_t ZSTD_matchState_t;
203
+
204
+ #define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */
205
+
136
206
  struct ZSTD_matchState_t {
137
207
  ZSTD_window_t window; /* State for window round buffer management */
138
208
  U32 loadedDictEnd; /* index of end of dictionary, within context's referential.
@@ -144,12 +214,24 @@ struct ZSTD_matchState_t {
144
214
  */
145
215
  U32 nextToUpdate; /* index from which to continue table update */
146
216
  U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */
217
+
218
+ U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/
219
+ U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */
220
+ U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */
221
+
147
222
  U32* hashTable;
148
223
  U32* hashTable3;
149
224
  U32* chainTable;
225
+
226
+ U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */
227
+
228
+ int dedicatedDictSearch; /* Indicates whether this matchState is using the
229
+ * dedicated dictionary search structure.
230
+ */
150
231
  optState_t opt; /* optimal parser state */
151
232
  const ZSTD_matchState_t* dictMatchState;
152
233
  ZSTD_compressionParameters cParams;
234
+ const rawSeqStore_t* ldmSeqStore;
153
235
  };
154
236
 
155
237
  typedef struct {
@@ -163,13 +245,22 @@ typedef struct {
163
245
  U32 checksum;
164
246
  } ldmEntry_t;
165
247
 
248
+ typedef struct {
249
+ BYTE const* split;
250
+ U32 hash;
251
+ U32 checksum;
252
+ ldmEntry_t* bucket;
253
+ } ldmMatchCandidate_t;
254
+
255
+ #define LDM_BATCH_SIZE 64
256
+
166
257
  typedef struct {
167
258
  ZSTD_window_t window; /* State for the window round buffer management */
168
259
  ldmEntry_t* hashTable;
169
260
  U32 loadedDictEnd;
170
261
  BYTE* bucketOffsets; /* Next position in bucket to insert entry */
171
- U64 hashPower; /* Used to compute the rolling hash.
172
- * Depends on ldmParams.minMatchLength */
262
+ size_t splitIndices[LDM_BATCH_SIZE];
263
+ ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE];
173
264
  } ldmState_t;
174
265
 
175
266
  typedef struct {
@@ -181,19 +272,6 @@ typedef struct {
181
272
  U32 windowLog; /* Window log for the LDM */
182
273
  } ldmParams_t;
183
274
 
184
- typedef struct {
185
- U32 offset;
186
- U32 litLength;
187
- U32 matchLength;
188
- } rawSeq;
189
-
190
- typedef struct {
191
- rawSeq* seq; /* The start of the sequences */
192
- size_t pos; /* The position where reading stopped. <= size. */
193
- size_t size; /* The number of sequences. <= capacity. */
194
- size_t capacity; /* The capacity starting from `seq` pointer */
195
- } rawSeqStore_t;
196
-
197
275
  typedef struct {
198
276
  int collectSequences;
199
277
  ZSTD_Sequence* seqStart;
@@ -228,17 +306,52 @@ struct ZSTD_CCtx_params_s {
228
306
  /* Long distance matching parameters */
229
307
  ldmParams_t ldmParams;
230
308
 
309
+ /* Dedicated dict search algorithm trigger */
310
+ int enableDedicatedDictSearch;
311
+
312
+ /* Input/output buffer modes */
313
+ ZSTD_bufferMode_e inBufferMode;
314
+ ZSTD_bufferMode_e outBufferMode;
315
+
316
+ /* Sequence compression API */
317
+ ZSTD_sequenceFormat_e blockDelimiters;
318
+ int validateSequences;
319
+
320
+ /* Block splitting */
321
+ int splitBlocks;
322
+
323
+ /* Param for deciding whether to use row-based matchfinder */
324
+ ZSTD_useRowMatchFinderMode_e useRowMatchFinder;
325
+
326
+ /* Always load a dictionary in ext-dict mode (not prefix mode)? */
327
+ int deterministicRefPrefix;
328
+
231
329
  /* Internal use, for createCCtxParams() and freeCCtxParams() only */
232
330
  ZSTD_customMem customMem;
233
331
  }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
234
332
 
333
+ #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
334
+ #define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE)
335
+
336
+ /**
337
+ * Indicates whether this compression proceeds directly from user-provided
338
+ * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or
339
+ * whether the context needs to buffer the input/output (ZSTDb_buffered).
340
+ */
341
+ typedef enum {
342
+ ZSTDb_not_buffered,
343
+ ZSTDb_buffered
344
+ } ZSTD_buffered_policy_e;
345
+
235
346
  struct ZSTD_CCtx_s {
236
347
  ZSTD_compressionStage_e stage;
237
348
  int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
238
349
  int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
239
350
  ZSTD_CCtx_params requestedParams;
240
351
  ZSTD_CCtx_params appliedParams;
352
+ ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */
241
353
  U32 dictID;
354
+ size_t dictContentSize;
242
355
 
243
356
  ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */
244
357
  size_t blockSize;
@@ -247,6 +360,7 @@ struct ZSTD_CCtx_s {
247
360
  unsigned long long producedCSize;
248
361
  XXH64_state_t xxhState;
249
362
  ZSTD_customMem customMem;
363
+ ZSTD_threadPool* pool;
250
364
  size_t staticSize;
251
365
  SeqCollector seqCollector;
252
366
  int isFirstBlock;
@@ -258,7 +372,10 @@ struct ZSTD_CCtx_s {
258
372
  size_t maxNbLdmSequences;
259
373
  rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */
260
374
  ZSTD_blockState_t blockState;
261
- U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */
375
+ U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */
376
+
377
+ /* Wether we are streaming or not */
378
+ ZSTD_buffered_policy_e bufferedPolicy;
262
379
 
263
380
  /* streaming */
264
381
  char* inBuff;
@@ -273,6 +390,10 @@ struct ZSTD_CCtx_s {
273
390
  ZSTD_cStreamStage streamStage;
274
391
  U32 frameEnded;
275
392
 
393
+ /* Stable in/out buffer verification */
394
+ ZSTD_inBuffer expectedInBuffer;
395
+ size_t expectedOutBufferSize;
396
+
276
397
  /* Dictionary */
277
398
  ZSTD_localDict localDict;
278
399
  const ZSTD_CDict* cdict;
@@ -282,17 +403,46 @@ struct ZSTD_CCtx_s {
282
403
  #ifdef ZSTD_MULTITHREAD
283
404
  ZSTDMT_CCtx* mtctx;
284
405
  #endif
406
+
407
+ /* Tracing */
408
+ #if ZSTD_TRACE
409
+ ZSTD_TraceCtx traceCtx;
410
+ #endif
285
411
  };
286
412
 
287
413
  typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
288
414
 
289
- typedef enum { ZSTD_noDict = 0, ZSTD_extDict = 1, ZSTD_dictMatchState = 2 } ZSTD_dictMode_e;
290
-
415
+ typedef enum {
416
+ ZSTD_noDict = 0,
417
+ ZSTD_extDict = 1,
418
+ ZSTD_dictMatchState = 2,
419
+ ZSTD_dedicatedDictSearch = 3
420
+ } ZSTD_dictMode_e;
421
+
422
+ typedef enum {
423
+ ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict.
424
+ * In this mode we use both the srcSize and the dictSize
425
+ * when selecting and adjusting parameters.
426
+ */
427
+ ZSTD_cpm_attachDict = 1, /* Compression with ZSTD_dictMatchState or ZSTD_dedicatedDictSearch.
428
+ * In this mode we only take the srcSize into account when selecting
429
+ * and adjusting parameters.
430
+ */
431
+ ZSTD_cpm_createCDict = 2, /* Creating a CDict.
432
+ * In this mode we take both the source size and the dictionary size
433
+ * into account when selecting and adjusting the parameters.
434
+ */
435
+ ZSTD_cpm_unknown = 3, /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
436
+ * We don't know what these parameters are for. We default to the legacy
437
+ * behavior of taking both the source size and the dict size into account
438
+ * when selecting and adjusting parameters.
439
+ */
440
+ } ZSTD_cParamMode_e;
291
441
 
292
442
  typedef size_t (*ZSTD_blockCompressor) (
293
443
  ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
294
444
  void const* src, size_t srcSize);
295
- ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode);
445
+ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_useRowMatchFinderMode_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
296
446
 
297
447
 
298
448
  MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
@@ -345,7 +495,7 @@ MEM_STATIC repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 con
345
495
  newReps.rep[1] = rep[0];
346
496
  newReps.rep[0] = currentOffset;
347
497
  } else { /* repCode == 0 */
348
- memcpy(&newReps, rep, sizeof(newReps));
498
+ ZSTD_memcpy(&newReps, rep, sizeof(newReps));
349
499
  }
350
500
  }
351
501
  return newReps;
@@ -372,7 +522,7 @@ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const voi
372
522
  RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity,
373
523
  dstSize_tooSmall, "dst buf too small for uncompressed block");
374
524
  MEM_writeLE24(dst, cBlockHeader24);
375
- memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
525
+ ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
376
526
  return ZSTD_blockHeaderSize + srcSize;
377
527
  }
378
528
 
@@ -469,8 +619,8 @@ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* litera
469
619
 
470
620
  /* literal Length */
471
621
  if (litLength>0xFFFF) {
472
- assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
473
- seqStorePtr->longLengthID = 1;
622
+ assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
623
+ seqStorePtr->longLengthType = ZSTD_llt_literalLength;
474
624
  seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
475
625
  }
476
626
  seqStorePtr->sequences[0].litLength = (U16)litLength;
@@ -480,8 +630,8 @@ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* litera
480
630
 
481
631
  /* match Length */
482
632
  if (mlBase>0xFFFF) {
483
- assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
484
- seqStorePtr->longLengthID = 2;
633
+ assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
634
+ seqStorePtr->longLengthType = ZSTD_llt_matchLength;
485
635
  seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
486
636
  }
487
637
  seqStorePtr->sequences[0].matchLength = (U16)mlBase;
@@ -498,8 +648,12 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
498
648
  if (MEM_isLittleEndian()) {
499
649
  if (MEM_64bits()) {
500
650
  # if defined(_MSC_VER) && defined(_WIN64)
501
- unsigned long r = 0;
502
- return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0;
651
+ # if STATIC_BMI2
652
+ return _tzcnt_u64(val) >> 3;
653
+ # else
654
+ unsigned long r = 0;
655
+ return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0;
656
+ # endif
503
657
  # elif defined(__GNUC__) && (__GNUC__ >= 4)
504
658
  return (__builtin_ctzll((U64)val) >> 3);
505
659
  # else
@@ -530,8 +684,12 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
530
684
  } else { /* Big Endian CPU */
531
685
  if (MEM_64bits()) {
532
686
  # if defined(_MSC_VER) && defined(_WIN64)
533
- unsigned long r = 0;
534
- return _BitScanReverse64( &r, val ) ? (unsigned)(r >> 3) : 0;
687
+ # if STATIC_BMI2
688
+ return _lzcnt_u64(val) >> 3;
689
+ # else
690
+ unsigned long r = 0;
691
+ return _BitScanReverse64(&r, (U64)val) ? (unsigned)(r >> 3) : 0;
692
+ # endif
535
693
  # elif defined(__GNUC__) && (__GNUC__ >= 4)
536
694
  return (__builtin_clzll(val) >> 3);
537
695
  # else
@@ -626,7 +784,8 @@ static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
626
784
  static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
627
785
  static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
628
786
 
629
- MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
787
+ MEM_STATIC FORCE_INLINE_ATTR
788
+ size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
630
789
  {
631
790
  switch(mls)
632
791
  {
@@ -723,6 +882,13 @@ MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window)
723
882
  window->dictLimit = end;
724
883
  }
725
884
 
885
+ MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window)
886
+ {
887
+ return window.dictLimit == 1 &&
888
+ window.lowLimit == 1 &&
889
+ (window.nextSrc - window.base) == 1;
890
+ }
891
+
726
892
  /**
727
893
  * ZSTD_window_hasExtDict():
728
894
  * Returns non-zero if the window has a non-empty extDict.
@@ -742,20 +908,74 @@ MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
742
908
  return ZSTD_window_hasExtDict(ms->window) ?
743
909
  ZSTD_extDict :
744
910
  ms->dictMatchState != NULL ?
745
- ZSTD_dictMatchState :
911
+ (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) :
746
912
  ZSTD_noDict;
747
913
  }
748
914
 
915
+ /* Defining this macro to non-zero tells zstd to run the overflow correction
916
+ * code much more frequently. This is very inefficient, and should only be
917
+ * used for tests and fuzzers.
918
+ */
919
+ #ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY
920
+ # ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
921
+ # define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1
922
+ # else
923
+ # define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0
924
+ # endif
925
+ #endif
926
+
927
+ /**
928
+ * ZSTD_window_canOverflowCorrect():
929
+ * Returns non-zero if the indices are large enough for overflow correction
930
+ * to work correctly without impacting compression ratio.
931
+ */
932
+ MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window,
933
+ U32 cycleLog,
934
+ U32 maxDist,
935
+ U32 loadedDictEnd,
936
+ void const* src)
937
+ {
938
+ U32 const cycleSize = 1u << cycleLog;
939
+ U32 const curr = (U32)((BYTE const*)src - window.base);
940
+ U32 const minIndexToOverflowCorrect = cycleSize + MAX(maxDist, cycleSize);
941
+
942
+ /* Adjust the min index to backoff the overflow correction frequency,
943
+ * so we don't waste too much CPU in overflow correction. If this
944
+ * computation overflows we don't really care, we just need to make
945
+ * sure it is at least minIndexToOverflowCorrect.
946
+ */
947
+ U32 const adjustment = window.nbOverflowCorrections + 1;
948
+ U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment,
949
+ minIndexToOverflowCorrect);
950
+ U32 const indexLargeEnough = curr > adjustedIndex;
951
+
952
+ /* Only overflow correct early if the dictionary is invalidated already,
953
+ * so we don't hurt compression ratio.
954
+ */
955
+ U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd;
956
+
957
+ return indexLargeEnough && dictionaryInvalidated;
958
+ }
959
+
749
960
  /**
750
961
  * ZSTD_window_needOverflowCorrection():
751
962
  * Returns non-zero if the indices are getting too large and need overflow
752
963
  * protection.
753
964
  */
754
965
  MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
966
+ U32 cycleLog,
967
+ U32 maxDist,
968
+ U32 loadedDictEnd,
969
+ void const* src,
755
970
  void const* srcEnd)
756
971
  {
757
- U32 const current = (U32)((BYTE const*)srcEnd - window.base);
758
- return current > ZSTD_CURRENT_MAX;
972
+ U32 const curr = (U32)((BYTE const*)srcEnd - window.base);
973
+ if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
974
+ if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) {
975
+ return 1;
976
+ }
977
+ }
978
+ return curr > ZSTD_CURRENT_MAX;
759
979
  }
760
980
 
761
981
  /**
@@ -766,7 +986,6 @@ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
766
986
  *
767
987
  * The least significant cycleLog bits of the indices must remain the same,
768
988
  * which may be 0. Every index up to maxDist in the past must be valid.
769
- * NOTE: (maxDist & cycleMask) must be zero.
770
989
  */
771
990
  MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
772
991
  U32 maxDist, void const* src)
@@ -790,17 +1009,25 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
790
1009
  * 3. (cctx->lowLimit + 1<<windowLog) < 1<<32:
791
1010
  * windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
792
1011
  */
793
- U32 const cycleMask = (1U << cycleLog) - 1;
794
- U32 const current = (U32)((BYTE const*)src - window->base);
795
- U32 const currentCycle0 = current & cycleMask;
1012
+ U32 const cycleSize = 1u << cycleLog;
1013
+ U32 const cycleMask = cycleSize - 1;
1014
+ U32 const curr = (U32)((BYTE const*)src - window->base);
1015
+ U32 const currentCycle0 = curr & cycleMask;
796
1016
  /* Exclude zero so that newCurrent - maxDist >= 1. */
797
- U32 const currentCycle1 = currentCycle0 == 0 ? (1U << cycleLog) : currentCycle0;
798
- U32 const newCurrent = currentCycle1 + maxDist;
799
- U32 const correction = current - newCurrent;
800
- assert((maxDist & cycleMask) == 0);
801
- assert(current > newCurrent);
802
- /* Loose bound, should be around 1<<29 (see above) */
803
- assert(correction > 1<<28);
1017
+ U32 const currentCycle1 = currentCycle0 == 0 ? cycleSize : currentCycle0;
1018
+ U32 const newCurrent = currentCycle1 + MAX(maxDist, cycleSize);
1019
+ U32 const correction = curr - newCurrent;
1020
+ /* maxDist must be a power of two so that:
1021
+ * (newCurrent & cycleMask) == (curr & cycleMask)
1022
+ * This is required to not corrupt the chains / binary tree.
1023
+ */
1024
+ assert((maxDist & (maxDist - 1)) == 0);
1025
+ assert((curr & cycleMask) == (newCurrent & cycleMask));
1026
+ assert(curr > newCurrent);
1027
+ if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
1028
+ /* Loose bound, should be around 1<<29 (see above) */
1029
+ assert(correction > 1<<28);
1030
+ }
804
1031
 
805
1032
  window->base += correction;
806
1033
  window->dictBase += correction;
@@ -816,6 +1043,8 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
816
1043
  assert(window->lowLimit <= newCurrent);
817
1044
  assert(window->dictLimit <= newCurrent);
818
1045
 
1046
+ ++window->nbOverflowCorrections;
1047
+
819
1048
  DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
820
1049
  window->lowLimit);
821
1050
  return correction;
@@ -919,12 +1148,13 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
919
1148
  }
920
1149
 
921
1150
  MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
922
- memset(window, 0, sizeof(*window));
1151
+ ZSTD_memset(window, 0, sizeof(*window));
923
1152
  window->base = (BYTE const*)"";
924
1153
  window->dictBase = (BYTE const*)"";
925
1154
  window->dictLimit = 1; /* start from 1, so that 1st position is valid */
926
1155
  window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
927
1156
  window->nextSrc = window->base + 1; /* see issue #1241 */
1157
+ window->nbOverflowCorrections = 0;
928
1158
  }
929
1159
 
930
1160
  /**
@@ -935,7 +1165,8 @@ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
935
1165
  * Returns non-zero if the segment is contiguous.
936
1166
  */
937
1167
  MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
938
- void const* src, size_t srcSize)
1168
+ void const* src, size_t srcSize,
1169
+ int forceNonContiguous)
939
1170
  {
940
1171
  BYTE const* const ip = (BYTE const*)src;
941
1172
  U32 contiguous = 1;
@@ -945,7 +1176,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
945
1176
  assert(window->base != NULL);
946
1177
  assert(window->dictBase != NULL);
947
1178
  /* Check if blocks follow each other */
948
- if (src != window->nextSrc) {
1179
+ if (src != window->nextSrc || forceNonContiguous) {
949
1180
  /* not contiguous */
950
1181
  size_t const distanceFromBase = (size_t)(window->nextSrc - window->base);
951
1182
  DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit);
@@ -973,12 +1204,16 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
973
1204
  /**
974
1205
  * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix.
975
1206
  */
976
- MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog)
1207
+ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
977
1208
  {
978
1209
  U32 const maxDistance = 1U << windowLog;
979
1210
  U32 const lowestValid = ms->window.lowLimit;
980
- U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid;
1211
+ U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
981
1212
  U32 const isDictionary = (ms->loadedDictEnd != 0);
1213
+ /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary
1214
+ * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't
1215
+ * valid for the entire block. So this check is sufficient to find the lowest valid match index.
1216
+ */
982
1217
  U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
983
1218
  return matchLowest;
984
1219
  }
@@ -986,12 +1221,15 @@ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current
986
1221
  /**
987
1222
  * Returns the lowest allowed match index in the prefix.
988
1223
  */
989
- MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog)
1224
+ MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
990
1225
  {
991
1226
  U32 const maxDistance = 1U << windowLog;
992
1227
  U32 const lowestValid = ms->window.dictLimit;
993
- U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid;
1228
+ U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
994
1229
  U32 const isDictionary = (ms->loadedDictEnd != 0);
1230
+ /* When computing the lowest prefix index we need to take the dictionary into account to handle
1231
+ * the edge case where the dictionary and the source are contiguous in memory.
1232
+ */
995
1233
  U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
996
1234
  return matchLowest;
997
1235
  }
@@ -1045,7 +1283,6 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
1045
1283
  * assumptions : magic number supposed already checked
1046
1284
  * and dictSize >= 8 */
1047
1285
  size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
1048
- short* offcodeNCount, unsigned* offcodeMaxValue,
1049
1286
  const void* const dict, size_t dictSize);
1050
1287
 
1051
1288
  void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
@@ -1061,7 +1298,7 @@ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
1061
1298
  * Note: srcSizeHint == 0 means 0!
1062
1299
  */
1063
1300
  ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
1064
- const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize);
1301
+ const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
1065
1302
 
1066
1303
  /*! ZSTD_initCStream_internal() :
1067
1304
  * Private use only. Init streaming operation.
@@ -1122,4 +1359,9 @@ size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSe
1122
1359
  * condition for correct operation : hashLog > 1 */
1123
1360
  U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
1124
1361
 
1362
+ /** ZSTD_CCtx_trace() :
1363
+ * Trace the end of a compression call.
1364
+ */
1365
+ void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
1366
+
1125
1367
  #endif /* ZSTD_COMPRESS_H */