zstdlib 0.6.0-x86-mingw32 → 0.9.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (141) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +20 -0
  3. data/README.md +7 -1
  4. data/Rakefile +38 -8
  5. data/ext/{zstdlib → zstdlib_c}/extconf.rb +10 -5
  6. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.2/zstdlib.c +2 -2
  7. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.3/zstdlib.c +2 -2
  8. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.4/zstdlib.c +2 -2
  9. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.5/zstdlib.c +2 -2
  10. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.6/zstdlib.c +2 -2
  11. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.7/zstdlib.c +2 -2
  12. data/ext/zstdlib_c/ruby/zlib-3.0/zstdlib.c +4994 -0
  13. data/ext/zstdlib_c/ruby/zlib-3.1/zstdlib.c +5076 -0
  14. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/adler32.c +0 -0
  15. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/compress.c +0 -0
  16. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/crc32.c +0 -0
  17. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/crc32.h +0 -0
  18. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/deflate.c +0 -0
  19. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/deflate.h +0 -0
  20. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzclose.c +0 -0
  21. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzguts.h +0 -0
  22. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzlib.c +0 -0
  23. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzread.c +0 -0
  24. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzwrite.c +0 -0
  25. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/infback.c +0 -0
  26. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffast.c +0 -0
  27. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffast.h +0 -0
  28. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffixed.h +0 -0
  29. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inflate.c +0 -0
  30. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inflate.h +0 -0
  31. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inftrees.c +0 -0
  32. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inftrees.h +0 -0
  33. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/trees.c +0 -0
  34. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/trees.h +0 -0
  35. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/uncompr.c +0 -0
  36. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zconf.h +0 -0
  37. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zlib.h +0 -0
  38. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zutil.c +0 -0
  39. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zutil.h +0 -0
  40. data/ext/{zstdlib → zstdlib_c}/zlib.mk +0 -0
  41. data/ext/{zstdlib → zstdlib_c}/zlibwrapper/zlibwrapper.c +1 -5
  42. data/ext/{zstdlib → zstdlib_c}/zlibwrapper.mk +0 -0
  43. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/bitstream.h +75 -57
  44. data/ext/zstdlib_c/zstd-1.5.2/lib/common/compiler.h +335 -0
  45. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/cpu.h +1 -3
  46. data/ext/zstdlib_c/zstd-1.5.2/lib/common/debug.c +24 -0
  47. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/debug.h +22 -49
  48. data/ext/zstdlib_c/zstd-1.5.2/lib/common/entropy_common.c +368 -0
  49. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/error_private.c +3 -1
  50. data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.h +159 -0
  51. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/fse.h +51 -42
  52. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/fse_decompress.c +149 -57
  53. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/huf.h +60 -54
  54. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/mem.h +87 -98
  55. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/pool.c +34 -23
  56. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/pool.h +5 -5
  57. data/ext/zstdlib_c/zstd-1.5.2/lib/common/portability_macros.h +137 -0
  58. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/threading.c +10 -8
  59. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/threading.h +4 -3
  60. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.c +24 -0
  61. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.h +5686 -0
  62. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_common.c +10 -10
  63. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_deps.h +111 -0
  64. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_internal.h +493 -0
  65. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_trace.h +163 -0
  66. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/clevels.h +134 -0
  67. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/fse_compress.c +105 -85
  68. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.c +41 -63
  69. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.h +13 -33
  70. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/huf_compress.c +1370 -0
  71. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress.c +6327 -0
  72. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_internal.h +537 -82
  73. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.c +21 -16
  74. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.h +4 -2
  75. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.c +61 -34
  76. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.h +10 -3
  77. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_superblock.c +573 -0
  78. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_superblock.h +32 -0
  79. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_cwksp.h +236 -95
  80. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.c +321 -143
  81. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.h +2 -2
  82. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.c +328 -137
  83. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.h +2 -2
  84. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.c +2104 -0
  85. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.h +125 -0
  86. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.c +336 -209
  87. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.h +15 -3
  88. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm_geartab.h +106 -0
  89. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.c +439 -239
  90. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.h +1 -1
  91. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstdmt_compress.c +205 -462
  92. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.h +113 -0
  93. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress.c +1889 -0
  94. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress_amd64.S +585 -0
  95. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.c +20 -16
  96. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.h +3 -3
  97. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress.c +691 -230
  98. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/zstd_decompress_block.c +2072 -0
  99. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.h +16 -7
  100. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_internal.h +71 -10
  101. data/ext/zstdlib_c/zstd-1.5.2/lib/zdict.h +452 -0
  102. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/zstd.h +760 -234
  103. data/ext/{zstdlib/zstd-1.4.4/lib/common → zstdlib_c/zstd-1.5.2/lib}/zstd_errors.h +3 -1
  104. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzclose.c +0 -0
  105. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzcompatibility.h +1 -1
  106. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzguts.h +0 -0
  107. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzlib.c +0 -0
  108. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzread.c +0 -0
  109. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzwrite.c +0 -0
  110. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.c +133 -44
  111. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  112. data/ext/zstdlib_c/zstd.mk +15 -0
  113. data/lib/2.4/zstdlib_c.so +0 -0
  114. data/lib/2.5/zstdlib_c.so +0 -0
  115. data/lib/2.6/zstdlib_c.so +0 -0
  116. data/lib/2.7/zstdlib_c.so +0 -0
  117. data/lib/3.0/zstdlib_c.so +0 -0
  118. data/lib/3.1/zstdlib_c.so +0 -0
  119. data/lib/zstdlib.rb +2 -2
  120. metadata +125 -114
  121. data/ext/zstdlib/zstd-1.4.4/lib/common/compiler.h +0 -159
  122. data/ext/zstdlib/zstd-1.4.4/lib/common/debug.c +0 -44
  123. data/ext/zstdlib/zstd-1.4.4/lib/common/entropy_common.c +0 -236
  124. data/ext/zstdlib/zstd-1.4.4/lib/common/error_private.h +0 -76
  125. data/ext/zstdlib/zstd-1.4.4/lib/common/xxhash.c +0 -882
  126. data/ext/zstdlib/zstd-1.4.4/lib/common/xxhash.h +0 -305
  127. data/ext/zstdlib/zstd-1.4.4/lib/common/zstd_internal.h +0 -350
  128. data/ext/zstdlib/zstd-1.4.4/lib/compress/huf_compress.c +0 -798
  129. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_compress.c +0 -4103
  130. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_lazy.c +0 -1115
  131. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_lazy.h +0 -67
  132. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstdmt_compress.h +0 -192
  133. data/ext/zstdlib/zstd-1.4.4/lib/decompress/huf_decompress.c +0 -1234
  134. data/ext/zstdlib/zstd-1.4.4/lib/decompress/zstd_decompress_block.c +0 -1323
  135. data/ext/zstdlib/zstd.mk +0 -14
  136. data/lib/2.2/zstdlib.so +0 -0
  137. data/lib/2.3/zstdlib.so +0 -0
  138. data/lib/2.4/zstdlib.so +0 -0
  139. data/lib/2.5/zstdlib.so +0 -0
  140. data/lib/2.6/zstdlib.so +0 -0
  141. data/lib/2.7/zstdlib.so +0 -0
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -18,7 +18,7 @@
18
18
  /*-*************************************
19
19
  * Dependencies
20
20
  ***************************************/
21
- #include "zstd_internal.h"
21
+ #include "../common/zstd_internal.h"
22
22
  #include "zstd_cwksp.h"
23
23
  #ifdef ZSTD_MULTITHREAD
24
24
  # include "zstdmt_compress.h"
@@ -28,7 +28,6 @@
28
28
  extern "C" {
29
29
  #endif
30
30
 
31
-
32
31
  /*-*************************************
33
32
  * Constants
34
33
  ***************************************/
@@ -64,7 +63,7 @@ typedef struct {
64
63
  } ZSTD_localDict;
65
64
 
66
65
  typedef struct {
67
- U32 CTable[HUF_CTABLE_SIZE_U32(255)];
66
+ HUF_CElt CTable[HUF_CTABLE_SIZE_ST(255)];
68
67
  HUF_repeat repeatMode;
69
68
  } ZSTD_hufCTables_t;
70
69
 
@@ -82,11 +81,75 @@ typedef struct {
82
81
  ZSTD_fseCTables_t fse;
83
82
  } ZSTD_entropyCTables_t;
84
83
 
84
+ /***********************************************
85
+ * Entropy buffer statistics structs and funcs *
86
+ ***********************************************/
87
+ /** ZSTD_hufCTablesMetadata_t :
88
+ * Stores Literals Block Type for a super-block in hType, and
89
+ * huffman tree description in hufDesBuffer.
90
+ * hufDesSize refers to the size of huffman tree description in bytes.
91
+ * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */
85
92
  typedef struct {
86
- U32 off;
87
- U32 len;
93
+ symbolEncodingType_e hType;
94
+ BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
95
+ size_t hufDesSize;
96
+ } ZSTD_hufCTablesMetadata_t;
97
+
98
+ /** ZSTD_fseCTablesMetadata_t :
99
+ * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
100
+ * fse tables in fseTablesBuffer.
101
+ * fseTablesSize refers to the size of fse tables in bytes.
102
+ * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */
103
+ typedef struct {
104
+ symbolEncodingType_e llType;
105
+ symbolEncodingType_e ofType;
106
+ symbolEncodingType_e mlType;
107
+ BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE];
108
+ size_t fseTablesSize;
109
+ size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
110
+ } ZSTD_fseCTablesMetadata_t;
111
+
112
+ typedef struct {
113
+ ZSTD_hufCTablesMetadata_t hufMetadata;
114
+ ZSTD_fseCTablesMetadata_t fseMetadata;
115
+ } ZSTD_entropyCTablesMetadata_t;
116
+
117
+ /** ZSTD_buildBlockEntropyStats() :
118
+ * Builds entropy for the block.
119
+ * @return : 0 on success or error code */
120
+ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
121
+ const ZSTD_entropyCTables_t* prevEntropy,
122
+ ZSTD_entropyCTables_t* nextEntropy,
123
+ const ZSTD_CCtx_params* cctxParams,
124
+ ZSTD_entropyCTablesMetadata_t* entropyMetadata,
125
+ void* workspace, size_t wkspSize);
126
+
127
+ /*********************************
128
+ * Compression internals structs *
129
+ *********************************/
130
+
131
+ typedef struct {
132
+ U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */
133
+ U32 len; /* Raw length of match */
88
134
  } ZSTD_match_t;
89
135
 
136
+ typedef struct {
137
+ U32 offset; /* Offset of sequence */
138
+ U32 litLength; /* Length of literals prior to match */
139
+ U32 matchLength; /* Raw length of match */
140
+ } rawSeq;
141
+
142
+ typedef struct {
143
+ rawSeq* seq; /* The start of the sequences */
144
+ size_t pos; /* The index in seq where reading stopped. pos <= size. */
145
+ size_t posInSequence; /* The position within the sequence at seq[pos] where reading
146
+ stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */
147
+ size_t size; /* The number of sequences. <= capacity. */
148
+ size_t capacity; /* The capacity starting from `seq` pointer */
149
+ } rawSeqStore_t;
150
+
151
+ UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
152
+
90
153
  typedef struct {
91
154
  int price;
92
155
  U32 off;
@@ -116,7 +179,7 @@ typedef struct {
116
179
  U32 offCodeSumBasePrice; /* to compare to log2(offreq) */
117
180
  ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */
118
181
  const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */
119
- ZSTD_literalCompressionMode_e literalCompressionMode;
182
+ ZSTD_paramSwitch_e literalCompressionMode;
120
183
  } optState_t;
121
184
 
122
185
  typedef struct {
@@ -125,14 +188,23 @@ typedef struct {
125
188
  } ZSTD_compressedBlockState_t;
126
189
 
127
190
  typedef struct {
128
- BYTE const* nextSrc; /* next block here to continue on current prefix */
129
- BYTE const* base; /* All regular indexes relative to this position */
130
- BYTE const* dictBase; /* extDict indexes relative to this position */
131
- U32 dictLimit; /* below that point, need extDict */
132
- U32 lowLimit; /* below that point, no more valid data */
191
+ BYTE const* nextSrc; /* next block here to continue on current prefix */
192
+ BYTE const* base; /* All regular indexes relative to this position */
193
+ BYTE const* dictBase; /* extDict indexes relative to this position */
194
+ U32 dictLimit; /* below that point, need extDict */
195
+ U32 lowLimit; /* below that point, no more valid data */
196
+ U32 nbOverflowCorrections; /* Number of times overflow correction has run since
197
+ * ZSTD_window_init(). Useful for debugging coredumps
198
+ * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY.
199
+ */
133
200
  } ZSTD_window_t;
134
201
 
202
+ #define ZSTD_WINDOW_START_INDEX 2
203
+
135
204
  typedef struct ZSTD_matchState_t ZSTD_matchState_t;
205
+
206
+ #define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */
207
+
136
208
  struct ZSTD_matchState_t {
137
209
  ZSTD_window_t window; /* State for window round buffer management */
138
210
  U32 loadedDictEnd; /* index of end of dictionary, within context's referential.
@@ -144,12 +216,24 @@ struct ZSTD_matchState_t {
144
216
  */
145
217
  U32 nextToUpdate; /* index from which to continue table update */
146
218
  U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */
219
+
220
+ U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/
221
+ U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */
222
+ U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */
223
+
147
224
  U32* hashTable;
148
225
  U32* hashTable3;
149
226
  U32* chainTable;
227
+
228
+ U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */
229
+
230
+ int dedicatedDictSearch; /* Indicates whether this matchState is using the
231
+ * dedicated dictionary search structure.
232
+ */
150
233
  optState_t opt; /* optimal parser state */
151
234
  const ZSTD_matchState_t* dictMatchState;
152
235
  ZSTD_compressionParameters cParams;
236
+ const rawSeqStore_t* ldmSeqStore;
153
237
  };
154
238
 
155
239
  typedef struct {
@@ -163,16 +247,26 @@ typedef struct {
163
247
  U32 checksum;
164
248
  } ldmEntry_t;
165
249
 
250
+ typedef struct {
251
+ BYTE const* split;
252
+ U32 hash;
253
+ U32 checksum;
254
+ ldmEntry_t* bucket;
255
+ } ldmMatchCandidate_t;
256
+
257
+ #define LDM_BATCH_SIZE 64
258
+
166
259
  typedef struct {
167
260
  ZSTD_window_t window; /* State for the window round buffer management */
168
261
  ldmEntry_t* hashTable;
262
+ U32 loadedDictEnd;
169
263
  BYTE* bucketOffsets; /* Next position in bucket to insert entry */
170
- U64 hashPower; /* Used to compute the rolling hash.
171
- * Depends on ldmParams.minMatchLength */
264
+ size_t splitIndices[LDM_BATCH_SIZE];
265
+ ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE];
172
266
  } ldmState_t;
173
267
 
174
268
  typedef struct {
175
- U32 enableLdm; /* 1 if enable long distance matching */
269
+ ZSTD_paramSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */
176
270
  U32 hashLog; /* Log size of hashTable */
177
271
  U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
178
272
  U32 minMatchLength; /* Minimum match length */
@@ -180,19 +274,6 @@ typedef struct {
180
274
  U32 windowLog; /* Window log for the LDM */
181
275
  } ldmParams_t;
182
276
 
183
- typedef struct {
184
- U32 offset;
185
- U32 litLength;
186
- U32 matchLength;
187
- } rawSeq;
188
-
189
- typedef struct {
190
- rawSeq* seq; /* The start of the sequences */
191
- size_t pos; /* The position where reading stopped. <= size. */
192
- size_t size; /* The number of sequences. <= capacity. */
193
- size_t capacity; /* The capacity starting from `seq` pointer */
194
- } rawSeqStore_t;
195
-
196
277
  typedef struct {
197
278
  int collectSequences;
198
279
  ZSTD_Sequence* seqStart;
@@ -216,7 +297,7 @@ struct ZSTD_CCtx_params_s {
216
297
  * There is no guarantee that hint is close to actual source size */
217
298
 
218
299
  ZSTD_dictAttachPref_e attachDictPref;
219
- ZSTD_literalCompressionMode_e literalCompressionMode;
300
+ ZSTD_paramSwitch_e literalCompressionMode;
220
301
 
221
302
  /* Multithreading: used to pass parameters to mtctx */
222
303
  int nbWorkers;
@@ -227,17 +308,68 @@ struct ZSTD_CCtx_params_s {
227
308
  /* Long distance matching parameters */
228
309
  ldmParams_t ldmParams;
229
310
 
311
+ /* Dedicated dict search algorithm trigger */
312
+ int enableDedicatedDictSearch;
313
+
314
+ /* Input/output buffer modes */
315
+ ZSTD_bufferMode_e inBufferMode;
316
+ ZSTD_bufferMode_e outBufferMode;
317
+
318
+ /* Sequence compression API */
319
+ ZSTD_sequenceFormat_e blockDelimiters;
320
+ int validateSequences;
321
+
322
+ /* Block splitting */
323
+ ZSTD_paramSwitch_e useBlockSplitter;
324
+
325
+ /* Param for deciding whether to use row-based matchfinder */
326
+ ZSTD_paramSwitch_e useRowMatchFinder;
327
+
328
+ /* Always load a dictionary in ext-dict mode (not prefix mode)? */
329
+ int deterministicRefPrefix;
330
+
230
331
  /* Internal use, for createCCtxParams() and freeCCtxParams() only */
231
332
  ZSTD_customMem customMem;
232
333
  }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
233
334
 
335
+ #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
336
+ #define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE)
337
+
338
+ /**
339
+ * Indicates whether this compression proceeds directly from user-provided
340
+ * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or
341
+ * whether the context needs to buffer the input/output (ZSTDb_buffered).
342
+ */
343
+ typedef enum {
344
+ ZSTDb_not_buffered,
345
+ ZSTDb_buffered
346
+ } ZSTD_buffered_policy_e;
347
+
348
+ /**
349
+ * Struct that contains all elements of block splitter that should be allocated
350
+ * in a wksp.
351
+ */
352
+ #define ZSTD_MAX_NB_BLOCK_SPLITS 196
353
+ typedef struct {
354
+ seqStore_t fullSeqStoreChunk;
355
+ seqStore_t firstHalfSeqStore;
356
+ seqStore_t secondHalfSeqStore;
357
+ seqStore_t currSeqStore;
358
+ seqStore_t nextSeqStore;
359
+
360
+ U32 partitions[ZSTD_MAX_NB_BLOCK_SPLITS];
361
+ ZSTD_entropyCTablesMetadata_t entropyMetadata;
362
+ } ZSTD_blockSplitCtx;
363
+
234
364
  struct ZSTD_CCtx_s {
235
365
  ZSTD_compressionStage_e stage;
236
366
  int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
237
367
  int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
238
368
  ZSTD_CCtx_params requestedParams;
239
369
  ZSTD_CCtx_params appliedParams;
370
+ ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */
240
371
  U32 dictID;
372
+ size_t dictContentSize;
241
373
 
242
374
  ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */
243
375
  size_t blockSize;
@@ -246,9 +378,11 @@ struct ZSTD_CCtx_s {
246
378
  unsigned long long producedCSize;
247
379
  XXH64_state_t xxhState;
248
380
  ZSTD_customMem customMem;
381
+ ZSTD_threadPool* pool;
249
382
  size_t staticSize;
250
383
  SeqCollector seqCollector;
251
384
  int isFirstBlock;
385
+ int initialized;
252
386
 
253
387
  seqStore_t seqStore; /* sequences storage ptrs */
254
388
  ldmState_t ldmState; /* long distance matching state */
@@ -256,7 +390,10 @@ struct ZSTD_CCtx_s {
256
390
  size_t maxNbLdmSequences;
257
391
  rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */
258
392
  ZSTD_blockState_t blockState;
259
- U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */
393
+ U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */
394
+
395
+ /* Whether we are streaming or not */
396
+ ZSTD_buffered_policy_e bufferedPolicy;
260
397
 
261
398
  /* streaming */
262
399
  char* inBuff;
@@ -271,6 +408,10 @@ struct ZSTD_CCtx_s {
271
408
  ZSTD_cStreamStage streamStage;
272
409
  U32 frameEnded;
273
410
 
411
+ /* Stable in/out buffer verification */
412
+ ZSTD_inBuffer expectedInBuffer;
413
+ size_t expectedOutBufferSize;
414
+
274
415
  /* Dictionary */
275
416
  ZSTD_localDict localDict;
276
417
  const ZSTD_CDict* cdict;
@@ -280,17 +421,49 @@ struct ZSTD_CCtx_s {
280
421
  #ifdef ZSTD_MULTITHREAD
281
422
  ZSTDMT_CCtx* mtctx;
282
423
  #endif
424
+
425
+ /* Tracing */
426
+ #if ZSTD_TRACE
427
+ ZSTD_TraceCtx traceCtx;
428
+ #endif
429
+
430
+ /* Workspace for block splitter */
431
+ ZSTD_blockSplitCtx blockSplitCtx;
283
432
  };
284
433
 
285
434
  typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
286
435
 
287
- typedef enum { ZSTD_noDict = 0, ZSTD_extDict = 1, ZSTD_dictMatchState = 2 } ZSTD_dictMode_e;
288
-
436
+ typedef enum {
437
+ ZSTD_noDict = 0,
438
+ ZSTD_extDict = 1,
439
+ ZSTD_dictMatchState = 2,
440
+ ZSTD_dedicatedDictSearch = 3
441
+ } ZSTD_dictMode_e;
442
+
443
+ typedef enum {
444
+ ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict.
445
+ * In this mode we use both the srcSize and the dictSize
446
+ * when selecting and adjusting parameters.
447
+ */
448
+ ZSTD_cpm_attachDict = 1, /* Compression with ZSTD_dictMatchState or ZSTD_dedicatedDictSearch.
449
+ * In this mode we only take the srcSize into account when selecting
450
+ * and adjusting parameters.
451
+ */
452
+ ZSTD_cpm_createCDict = 2, /* Creating a CDict.
453
+ * In this mode we take both the source size and the dictionary size
454
+ * into account when selecting and adjusting the parameters.
455
+ */
456
+ ZSTD_cpm_unknown = 3, /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
457
+ * We don't know what these parameters are for. We default to the legacy
458
+ * behavior of taking both the source size and the dict size into account
459
+ * when selecting and adjusting parameters.
460
+ */
461
+ } ZSTD_cParamMode_e;
289
462
 
290
463
  typedef size_t (*ZSTD_blockCompressor) (
291
464
  ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
292
465
  void const* src, size_t srcSize);
293
- ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode);
466
+ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
294
467
 
295
468
 
296
469
  MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
@@ -336,6 +509,30 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
336
509
  return 1;
337
510
  }
338
511
 
512
+ /* ZSTD_noCompressBlock() :
513
+ * Writes uncompressed block to dst buffer from given src.
514
+ * Returns the size of the block */
515
+ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
516
+ {
517
+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3);
518
+ RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity,
519
+ dstSize_tooSmall, "dst buf too small for uncompressed block");
520
+ MEM_writeLE24(dst, cBlockHeader24);
521
+ ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
522
+ return ZSTD_blockHeaderSize + srcSize;
523
+ }
524
+
525
+ MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock)
526
+ {
527
+ BYTE* const op = (BYTE*)dst;
528
+ U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3);
529
+ RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, "");
530
+ MEM_writeLE24(op, cBlockHeader);
531
+ op[3] = src;
532
+ return 4;
533
+ }
534
+
535
+
339
536
  /* ZSTD_minGain() :
340
537
  * minimum compression required
341
538
  * to generate a compress block or a compressed literals section.
@@ -348,12 +545,29 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
348
545
  return (srcSize >> minlog) + 2;
349
546
  }
350
547
 
548
+ MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams)
549
+ {
550
+ switch (cctxParams->literalCompressionMode) {
551
+ case ZSTD_ps_enable:
552
+ return 0;
553
+ case ZSTD_ps_disable:
554
+ return 1;
555
+ default:
556
+ assert(0 /* impossible: pre-validated */);
557
+ ZSTD_FALLTHROUGH;
558
+ case ZSTD_ps_auto:
559
+ return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
560
+ }
561
+ }
562
+
351
563
  /*! ZSTD_safecopyLiterals() :
352
564
  * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w.
353
565
  * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single
354
566
  * large copies.
355
567
  */
356
- static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) {
568
+ static void
569
+ ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w)
570
+ {
357
571
  assert(iend > ilimit_w);
358
572
  if (ip <= ilimit_w) {
359
573
  ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap);
@@ -363,14 +577,30 @@ static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const ie
363
577
  while (ip < iend) *op++ = *ip++;
364
578
  }
365
579
 
580
+ #define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
581
+ #define STORE_REPCODE_1 STORE_REPCODE(1)
582
+ #define STORE_REPCODE_2 STORE_REPCODE(2)
583
+ #define STORE_REPCODE_3 STORE_REPCODE(3)
584
+ #define STORE_REPCODE(r) (assert((r)>=1), assert((r)<=3), (r)-1)
585
+ #define STORE_OFFSET(o) (assert((o)>0), o + ZSTD_REP_MOVE)
586
+ #define STORED_IS_OFFSET(o) ((o) > ZSTD_REP_MOVE)
587
+ #define STORED_IS_REPCODE(o) ((o) <= ZSTD_REP_MOVE)
588
+ #define STORED_OFFSET(o) (assert(STORED_IS_OFFSET(o)), (o)-ZSTD_REP_MOVE)
589
+ #define STORED_REPCODE(o) (assert(STORED_IS_REPCODE(o)), (o)+1) /* returns ID 1,2,3 */
590
+ #define STORED_TO_OFFBASE(o) ((o)+1)
591
+ #define OFFBASE_TO_STORED(o) ((o)-1)
592
+
366
593
  /*! ZSTD_storeSeq() :
367
- * Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t.
368
- * `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes).
369
- * `mlBase` : matchLength - MINMATCH
594
+ * Store a sequence (litlen, litPtr, offCode and matchLength) into seqStore_t.
595
+ * @offBase_minus1 : Users should use employ macros STORE_REPCODE_X and STORE_OFFSET().
596
+ * @matchLength : must be >= MINMATCH
370
597
  * Allowed to overread literals up to litLimit.
371
598
  */
372
- HINT_INLINE UNUSED_ATTR
373
- void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase)
599
+ HINT_INLINE UNUSED_ATTR void
600
+ ZSTD_storeSeq(seqStore_t* seqStorePtr,
601
+ size_t litLength, const BYTE* literals, const BYTE* litLimit,
602
+ U32 offBase_minus1,
603
+ size_t matchLength)
374
604
  {
375
605
  BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH;
376
606
  BYTE const* const litEnd = literals + litLength;
@@ -379,7 +609,7 @@ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* litera
379
609
  if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
380
610
  { U32 const pos = (U32)((const BYTE*)literals - g_start);
381
611
  DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u",
382
- pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offCode);
612
+ pos, (U32)litLength, (U32)matchLength, (U32)offBase_minus1);
383
613
  }
384
614
  #endif
385
615
  assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
@@ -403,26 +633,66 @@ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* litera
403
633
 
404
634
  /* literal Length */
405
635
  if (litLength>0xFFFF) {
406
- assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
407
- seqStorePtr->longLengthID = 1;
636
+ assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
637
+ seqStorePtr->longLengthType = ZSTD_llt_literalLength;
408
638
  seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
409
639
  }
410
640
  seqStorePtr->sequences[0].litLength = (U16)litLength;
411
641
 
412
642
  /* match offset */
413
- seqStorePtr->sequences[0].offset = offCode + 1;
643
+ seqStorePtr->sequences[0].offBase = STORED_TO_OFFBASE(offBase_minus1);
414
644
 
415
645
  /* match Length */
416
- if (mlBase>0xFFFF) {
417
- assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
418
- seqStorePtr->longLengthID = 2;
419
- seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
646
+ assert(matchLength >= MINMATCH);
647
+ { size_t const mlBase = matchLength - MINMATCH;
648
+ if (mlBase>0xFFFF) {
649
+ assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
650
+ seqStorePtr->longLengthType = ZSTD_llt_matchLength;
651
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
652
+ }
653
+ seqStorePtr->sequences[0].mlBase = (U16)mlBase;
420
654
  }
421
- seqStorePtr->sequences[0].matchLength = (U16)mlBase;
422
655
 
423
656
  seqStorePtr->sequences++;
424
657
  }
425
658
 
659
+ /* ZSTD_updateRep() :
660
+ * updates in-place @rep (array of repeat offsets)
661
+ * @offBase_minus1 : sum-type, with same numeric representation as ZSTD_storeSeq()
662
+ */
663
+ MEM_STATIC void
664
+ ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0)
665
+ {
666
+ if (STORED_IS_OFFSET(offBase_minus1)) { /* full offset */
667
+ rep[2] = rep[1];
668
+ rep[1] = rep[0];
669
+ rep[0] = STORED_OFFSET(offBase_minus1);
670
+ } else { /* repcode */
671
+ U32 const repCode = STORED_REPCODE(offBase_minus1) - 1 + ll0;
672
+ if (repCode > 0) { /* note : if repCode==0, no change */
673
+ U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
674
+ rep[2] = (repCode >= 2) ? rep[1] : rep[2];
675
+ rep[1] = rep[0];
676
+ rep[0] = currentOffset;
677
+ } else { /* repCode == 0 */
678
+ /* nothing to do */
679
+ }
680
+ }
681
+ }
682
+
683
+ typedef struct repcodes_s {
684
+ U32 rep[3];
685
+ } repcodes_t;
686
+
687
+ MEM_STATIC repcodes_t
688
+ ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0)
689
+ {
690
+ repcodes_t newReps;
691
+ ZSTD_memcpy(&newReps, rep, sizeof(newReps));
692
+ ZSTD_updateRep(newReps.rep, offBase_minus1, ll0);
693
+ return newReps;
694
+ }
695
+
426
696
 
427
697
  /*-*************************************
428
698
  * Match length counter
@@ -432,9 +702,18 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
432
702
  if (MEM_isLittleEndian()) {
433
703
  if (MEM_64bits()) {
434
704
  # if defined(_MSC_VER) && defined(_WIN64)
435
- unsigned long r = 0;
436
- _BitScanForward64( &r, (U64)val );
437
- return (unsigned)(r>>3);
705
+ # if STATIC_BMI2
706
+ return _tzcnt_u64(val) >> 3;
707
+ # else
708
+ if (val != 0) {
709
+ unsigned long r;
710
+ _BitScanForward64(&r, (U64)val);
711
+ return (unsigned)(r >> 3);
712
+ } else {
713
+ /* Should not reach this code path */
714
+ __assume(0);
715
+ }
716
+ # endif
438
717
  # elif defined(__GNUC__) && (__GNUC__ >= 4)
439
718
  return (__builtin_ctzll((U64)val) >> 3);
440
719
  # else
@@ -450,9 +729,14 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
450
729
  # endif
451
730
  } else { /* 32 bits */
452
731
  # if defined(_MSC_VER)
453
- unsigned long r=0;
454
- _BitScanForward( &r, (U32)val );
455
- return (unsigned)(r>>3);
732
+ if (val != 0) {
733
+ unsigned long r;
734
+ _BitScanForward(&r, (U32)val);
735
+ return (unsigned)(r >> 3);
736
+ } else {
737
+ /* Should not reach this code path */
738
+ __assume(0);
739
+ }
456
740
  # elif defined(__GNUC__) && (__GNUC__ >= 3)
457
741
  return (__builtin_ctz((U32)val) >> 3);
458
742
  # else
@@ -466,9 +750,18 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
466
750
  } else { /* Big Endian CPU */
467
751
  if (MEM_64bits()) {
468
752
  # if defined(_MSC_VER) && defined(_WIN64)
469
- unsigned long r = 0;
470
- _BitScanReverse64( &r, val );
471
- return (unsigned)(r>>3);
753
+ # if STATIC_BMI2
754
+ return _lzcnt_u64(val) >> 3;
755
+ # else
756
+ if (val != 0) {
757
+ unsigned long r;
758
+ _BitScanReverse64(&r, (U64)val);
759
+ return (unsigned)(r >> 3);
760
+ } else {
761
+ /* Should not reach this code path */
762
+ __assume(0);
763
+ }
764
+ # endif
472
765
  # elif defined(__GNUC__) && (__GNUC__ >= 4)
473
766
  return (__builtin_clzll(val) >> 3);
474
767
  # else
@@ -481,9 +774,14 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
481
774
  # endif
482
775
  } else { /* 32 bits */
483
776
  # if defined(_MSC_VER)
484
- unsigned long r = 0;
485
- _BitScanReverse( &r, (unsigned long)val );
486
- return (unsigned)(r>>3);
777
+ if (val != 0) {
778
+ unsigned long r;
779
+ _BitScanReverse(&r, (unsigned long)val);
780
+ return (unsigned)(r >> 3);
781
+ } else {
782
+ /* Should not reach this code path */
783
+ __assume(0);
784
+ }
487
785
  # elif defined(__GNUC__) && (__GNUC__ >= 3)
488
786
  return (__builtin_clz((U32)val) >> 3);
489
787
  # else
@@ -564,7 +862,8 @@ static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
564
862
  static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
565
863
  static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
566
864
 
567
- MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
865
+ MEM_STATIC FORCE_INLINE_ATTR
866
+ size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
568
867
  {
569
868
  switch(mls)
570
869
  {
@@ -661,6 +960,13 @@ MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window)
661
960
  window->dictLimit = end;
662
961
  }
663
962
 
963
+ MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window)
964
+ {
965
+ return window.dictLimit == ZSTD_WINDOW_START_INDEX &&
966
+ window.lowLimit == ZSTD_WINDOW_START_INDEX &&
967
+ (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX;
968
+ }
969
+
664
970
  /**
665
971
  * ZSTD_window_hasExtDict():
666
972
  * Returns non-zero if the window has a non-empty extDict.
@@ -680,20 +986,76 @@ MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
680
986
  return ZSTD_window_hasExtDict(ms->window) ?
681
987
  ZSTD_extDict :
682
988
  ms->dictMatchState != NULL ?
683
- ZSTD_dictMatchState :
989
+ (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) :
684
990
  ZSTD_noDict;
685
991
  }
686
992
 
993
+ /* Defining this macro to non-zero tells zstd to run the overflow correction
994
+ * code much more frequently. This is very inefficient, and should only be
995
+ * used for tests and fuzzers.
996
+ */
997
+ #ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY
998
+ # ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
999
+ # define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1
1000
+ # else
1001
+ # define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0
1002
+ # endif
1003
+ #endif
1004
+
1005
+ /**
1006
+ * ZSTD_window_canOverflowCorrect():
1007
+ * Returns non-zero if the indices are large enough for overflow correction
1008
+ * to work correctly without impacting compression ratio.
1009
+ */
1010
+ MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window,
1011
+ U32 cycleLog,
1012
+ U32 maxDist,
1013
+ U32 loadedDictEnd,
1014
+ void const* src)
1015
+ {
1016
+ U32 const cycleSize = 1u << cycleLog;
1017
+ U32 const curr = (U32)((BYTE const*)src - window.base);
1018
+ U32 const minIndexToOverflowCorrect = cycleSize
1019
+ + MAX(maxDist, cycleSize)
1020
+ + ZSTD_WINDOW_START_INDEX;
1021
+
1022
+ /* Adjust the min index to backoff the overflow correction frequency,
1023
+ * so we don't waste too much CPU in overflow correction. If this
1024
+ * computation overflows we don't really care, we just need to make
1025
+ * sure it is at least minIndexToOverflowCorrect.
1026
+ */
1027
+ U32 const adjustment = window.nbOverflowCorrections + 1;
1028
+ U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment,
1029
+ minIndexToOverflowCorrect);
1030
+ U32 const indexLargeEnough = curr > adjustedIndex;
1031
+
1032
+ /* Only overflow correct early if the dictionary is invalidated already,
1033
+ * so we don't hurt compression ratio.
1034
+ */
1035
+ U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd;
1036
+
1037
+ return indexLargeEnough && dictionaryInvalidated;
1038
+ }
1039
+
687
1040
  /**
688
1041
  * ZSTD_window_needOverflowCorrection():
689
1042
  * Returns non-zero if the indices are getting too large and need overflow
690
1043
  * protection.
691
1044
  */
692
1045
  MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
1046
+ U32 cycleLog,
1047
+ U32 maxDist,
1048
+ U32 loadedDictEnd,
1049
+ void const* src,
693
1050
  void const* srcEnd)
694
1051
  {
695
- U32 const current = (U32)((BYTE const*)srcEnd - window.base);
696
- return current > ZSTD_CURRENT_MAX;
1052
+ U32 const curr = (U32)((BYTE const*)srcEnd - window.base);
1053
+ if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
1054
+ if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) {
1055
+ return 1;
1056
+ }
1057
+ }
1058
+ return curr > ZSTD_CURRENT_MAX;
697
1059
  }
698
1060
 
699
1061
  /**
@@ -704,7 +1066,6 @@ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
704
1066
  *
705
1067
  * The least significant cycleLog bits of the indices must remain the same,
706
1068
  * which may be 0. Every index up to maxDist in the past must be valid.
707
- * NOTE: (maxDist & cycleMask) must be zero.
708
1069
  */
709
1070
  MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
710
1071
  U32 maxDist, void const* src)
@@ -728,19 +1089,51 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
728
1089
  * 3. (cctx->lowLimit + 1<<windowLog) < 1<<32:
729
1090
  * windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
730
1091
  */
731
- U32 const cycleMask = (1U << cycleLog) - 1;
732
- U32 const current = (U32)((BYTE const*)src - window->base);
733
- U32 const newCurrent = (current & cycleMask) + maxDist;
734
- U32 const correction = current - newCurrent;
735
- assert((maxDist & cycleMask) == 0);
736
- assert(current > newCurrent);
737
- /* Loose bound, should be around 1<<29 (see above) */
738
- assert(correction > 1<<28);
1092
+ U32 const cycleSize = 1u << cycleLog;
1093
+ U32 const cycleMask = cycleSize - 1;
1094
+ U32 const curr = (U32)((BYTE const*)src - window->base);
1095
+ U32 const currentCycle = curr & cycleMask;
1096
+ /* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */
1097
+ U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX
1098
+ ? MAX(cycleSize, ZSTD_WINDOW_START_INDEX)
1099
+ : 0;
1100
+ U32 const newCurrent = currentCycle
1101
+ + currentCycleCorrection
1102
+ + MAX(maxDist, cycleSize);
1103
+ U32 const correction = curr - newCurrent;
1104
+ /* maxDist must be a power of two so that:
1105
+ * (newCurrent & cycleMask) == (curr & cycleMask)
1106
+ * This is required to not corrupt the chains / binary tree.
1107
+ */
1108
+ assert((maxDist & (maxDist - 1)) == 0);
1109
+ assert((curr & cycleMask) == (newCurrent & cycleMask));
1110
+ assert(curr > newCurrent);
1111
+ if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
1112
+ /* Loose bound, should be around 1<<29 (see above) */
1113
+ assert(correction > 1<<28);
1114
+ }
739
1115
 
740
1116
  window->base += correction;
741
1117
  window->dictBase += correction;
742
- window->lowLimit -= correction;
743
- window->dictLimit -= correction;
1118
+ if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) {
1119
+ window->lowLimit = ZSTD_WINDOW_START_INDEX;
1120
+ } else {
1121
+ window->lowLimit -= correction;
1122
+ }
1123
+ if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) {
1124
+ window->dictLimit = ZSTD_WINDOW_START_INDEX;
1125
+ } else {
1126
+ window->dictLimit -= correction;
1127
+ }
1128
+
1129
+ /* Ensure we can still reference the full window. */
1130
+ assert(newCurrent >= maxDist);
1131
+ assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX);
1132
+ /* Ensure that lowLimit and dictLimit didn't underflow. */
1133
+ assert(window->lowLimit <= newCurrent);
1134
+ assert(window->dictLimit <= newCurrent);
1135
+
1136
+ ++window->nbOverflowCorrections;
744
1137
 
745
1138
  DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
746
1139
  window->lowLimit);
@@ -844,6 +1237,17 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
844
1237
  } } }
845
1238
  }
846
1239
 
1240
+ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
1241
+ ZSTD_memset(window, 0, sizeof(*window));
1242
+ window->base = (BYTE const*)" ";
1243
+ window->dictBase = (BYTE const*)" ";
1244
+ ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */
1245
+ window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */
1246
+ window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */
1247
+ window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */
1248
+ window->nbOverflowCorrections = 0;
1249
+ }
1250
+
847
1251
  /**
848
1252
  * ZSTD_window_update():
849
1253
  * Updates the window by appending [src, src + srcSize) to the window.
@@ -852,13 +1256,18 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
852
1256
  * Returns non-zero if the segment is contiguous.
853
1257
  */
854
1258
  MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
855
- void const* src, size_t srcSize)
1259
+ void const* src, size_t srcSize,
1260
+ int forceNonContiguous)
856
1261
  {
857
1262
  BYTE const* const ip = (BYTE const*)src;
858
1263
  U32 contiguous = 1;
859
1264
  DEBUGLOG(5, "ZSTD_window_update");
1265
+ if (srcSize == 0)
1266
+ return contiguous;
1267
+ assert(window->base != NULL);
1268
+ assert(window->dictBase != NULL);
860
1269
  /* Check if blocks follow each other */
861
- if (src != window->nextSrc) {
1270
+ if (src != window->nextSrc || forceNonContiguous) {
862
1271
  /* not contiguous */
863
1272
  size_t const distanceFromBase = (size_t)(window->nextSrc - window->base);
864
1273
  DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit);
@@ -867,7 +1276,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
867
1276
  window->dictLimit = (U32)distanceFromBase;
868
1277
  window->dictBase = window->base;
869
1278
  window->base = ip - distanceFromBase;
870
- // ms->nextToUpdate = window->dictLimit;
1279
+ /* ms->nextToUpdate = window->dictLimit; */
871
1280
  if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */
872
1281
  contiguous = 0;
873
1282
  }
@@ -883,12 +1292,35 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
883
1292
  return contiguous;
884
1293
  }
885
1294
 
886
- MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog)
1295
+ /**
1296
+ * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix.
1297
+ */
1298
+ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
1299
+ {
1300
+ U32 const maxDistance = 1U << windowLog;
1301
+ U32 const lowestValid = ms->window.lowLimit;
1302
+ U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
1303
+ U32 const isDictionary = (ms->loadedDictEnd != 0);
1304
+ /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary
1305
+ * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't
1306
+ * valid for the entire block. So this check is sufficient to find the lowest valid match index.
1307
+ */
1308
+ U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
1309
+ return matchLowest;
1310
+ }
1311
+
1312
+ /**
1313
+ * Returns the lowest allowed match index in the prefix.
1314
+ */
1315
+ MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
887
1316
  {
888
1317
  U32 const maxDistance = 1U << windowLog;
889
- U32 const lowestValid = ms->window.lowLimit;
890
- U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid;
1318
+ U32 const lowestValid = ms->window.dictLimit;
1319
+ U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
891
1320
  U32 const isDictionary = (ms->loadedDictEnd != 0);
1321
+ /* When computing the lowest prefix index we need to take the dictionary into account to handle
1322
+ * the edge case where the dictionary and the source are contiguous in memory.
1323
+ */
892
1324
  U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
893
1325
  return matchLowest;
894
1326
  }
@@ -931,6 +1363,20 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
931
1363
  }
932
1364
  #endif
933
1365
 
1366
+ /* ===============================================================
1367
+ * Shared internal declarations
1368
+ * These prototypes may be called from sources not in lib/compress
1369
+ * =============================================================== */
1370
+
1371
+ /* ZSTD_loadCEntropy() :
1372
+ * dict : must point at beginning of a valid zstd dictionary.
1373
+ * return : size of dictionary header (size of magic number + dict ID + entropy tables)
1374
+ * assumptions : magic number supposed already checked
1375
+ * and dictSize >= 8 */
1376
+ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
1377
+ const void* const dict, size_t dictSize);
1378
+
1379
+ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
934
1380
 
935
1381
  /* ==============================================================
936
1382
  * Private declarations
@@ -940,9 +1386,10 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
940
1386
  /* ZSTD_getCParamsFromCCtxParams() :
941
1387
  * cParams are built depending on compressionLevel, src size hints,
942
1388
  * LDM and manually set compression parameters.
1389
+ * Note: srcSizeHint == 0 means 0!
943
1390
  */
944
1391
  ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
945
- const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize);
1392
+ const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
946
1393
 
947
1394
  /*! ZSTD_initCStream_internal() :
948
1395
  * Private use only. Init streaming operation.
@@ -999,5 +1446,13 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity);
999
1446
  */
1000
1447
  size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq);
1001
1448
 
1449
+ /** ZSTD_cycleLog() :
1450
+ * condition for correct operation : hashLog > 1 */
1451
+ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
1452
+
1453
+ /** ZSTD_CCtx_trace() :
1454
+ * Trace the end of a compression call.
1455
+ */
1456
+ void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
1002
1457
 
1003
1458
  #endif /* ZSTD_COMPRESS_H */