extzstd 0.3.3 → 0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (69) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -3
  3. data/contrib/zstd/CHANGELOG +39 -2
  4. data/contrib/zstd/CONTRIBUTING.md +3 -3
  5. data/contrib/zstd/Makefile +34 -20
  6. data/contrib/zstd/README.md +6 -2
  7. data/contrib/zstd/SECURITY.md +15 -0
  8. data/contrib/zstd/lib/Makefile +40 -28
  9. data/contrib/zstd/lib/README.md +14 -1
  10. data/contrib/zstd/lib/common/allocations.h +1 -1
  11. data/contrib/zstd/lib/common/bitstream.h +49 -29
  12. data/contrib/zstd/lib/common/compiler.h +114 -22
  13. data/contrib/zstd/lib/common/cpu.h +36 -0
  14. data/contrib/zstd/lib/common/debug.c +6 -0
  15. data/contrib/zstd/lib/common/debug.h +20 -11
  16. data/contrib/zstd/lib/common/error_private.h +45 -36
  17. data/contrib/zstd/lib/common/fse.h +3 -2
  18. data/contrib/zstd/lib/common/fse_decompress.c +19 -17
  19. data/contrib/zstd/lib/common/huf.h +14 -1
  20. data/contrib/zstd/lib/common/mem.h +0 -9
  21. data/contrib/zstd/lib/common/pool.c +1 -1
  22. data/contrib/zstd/lib/common/pool.h +1 -1
  23. data/contrib/zstd/lib/common/portability_macros.h +2 -0
  24. data/contrib/zstd/lib/common/threading.c +8 -2
  25. data/contrib/zstd/lib/common/xxhash.c +5 -11
  26. data/contrib/zstd/lib/common/xxhash.h +2341 -1007
  27. data/contrib/zstd/lib/common/zstd_internal.h +5 -5
  28. data/contrib/zstd/lib/compress/fse_compress.c +8 -7
  29. data/contrib/zstd/lib/compress/huf_compress.c +54 -25
  30. data/contrib/zstd/lib/compress/zstd_compress.c +282 -161
  31. data/contrib/zstd/lib/compress/zstd_compress_internal.h +29 -27
  32. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +224 -113
  33. data/contrib/zstd/lib/compress/zstd_cwksp.h +19 -13
  34. data/contrib/zstd/lib/compress/zstd_double_fast.c +17 -5
  35. data/contrib/zstd/lib/compress/zstd_double_fast.h +11 -0
  36. data/contrib/zstd/lib/compress/zstd_fast.c +14 -6
  37. data/contrib/zstd/lib/compress/zstd_lazy.c +129 -87
  38. data/contrib/zstd/lib/compress/zstd_lazy.h +103 -28
  39. data/contrib/zstd/lib/compress/zstd_ldm.c +8 -2
  40. data/contrib/zstd/lib/compress/zstd_opt.c +216 -112
  41. data/contrib/zstd/lib/compress/zstd_opt.h +31 -7
  42. data/contrib/zstd/lib/compress/zstdmt_compress.c +94 -79
  43. data/contrib/zstd/lib/decompress/huf_decompress.c +188 -126
  44. data/contrib/zstd/lib/decompress/huf_decompress_amd64.S +38 -19
  45. data/contrib/zstd/lib/decompress/zstd_decompress.c +84 -32
  46. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +231 -208
  47. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +1 -1
  48. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +2 -0
  49. data/contrib/zstd/lib/dictBuilder/cover.c +16 -12
  50. data/contrib/zstd/lib/dictBuilder/cover.h +2 -8
  51. data/contrib/zstd/lib/dictBuilder/fastcover.c +2 -2
  52. data/contrib/zstd/lib/dictBuilder/zdict.c +12 -6
  53. data/contrib/zstd/lib/legacy/zstd_legacy.h +30 -0
  54. data/contrib/zstd/lib/legacy/zstd_v01.c +2 -0
  55. data/contrib/zstd/lib/legacy/zstd_v02.c +4 -16
  56. data/contrib/zstd/lib/legacy/zstd_v03.c +4 -16
  57. data/contrib/zstd/lib/legacy/zstd_v04.c +4 -11
  58. data/contrib/zstd/lib/legacy/zstd_v05.c +1 -0
  59. data/contrib/zstd/lib/legacy/zstd_v06.c +2 -9
  60. data/contrib/zstd/lib/legacy/zstd_v07.c +2 -10
  61. data/contrib/zstd/lib/libzstd.mk +34 -11
  62. data/contrib/zstd/lib/zstd.h +129 -60
  63. data/ext/extconf.rb +19 -1
  64. data/ext/extzstd.c +38 -14
  65. data/ext/extzstd.h +33 -6
  66. data/ext/extzstd_stream.c +74 -31
  67. metadata +4 -5
  68. data/contrib/zstd/appveyor.yml +0 -205
  69. data/ext/depend +0 -2
@@ -27,98 +27,173 @@ extern "C" {
27
27
 
28
28
  #define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */
29
29
 
30
+ #if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \
31
+ || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
32
+ || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
33
+ || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR)
30
34
  U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
31
35
  void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
32
36
 
33
37
  void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip);
34
38
 
35
39
  void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */
40
+ #endif
36
41
 
37
- size_t ZSTD_compressBlock_btlazy2(
42
+ #ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
43
+ size_t ZSTD_compressBlock_greedy(
38
44
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
39
45
  void const* src, size_t srcSize);
40
- size_t ZSTD_compressBlock_lazy2(
46
+ size_t ZSTD_compressBlock_greedy_row(
41
47
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
42
48
  void const* src, size_t srcSize);
43
- size_t ZSTD_compressBlock_lazy(
49
+ size_t ZSTD_compressBlock_greedy_dictMatchState(
44
50
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
45
51
  void const* src, size_t srcSize);
46
- size_t ZSTD_compressBlock_greedy(
52
+ size_t ZSTD_compressBlock_greedy_dictMatchState_row(
47
53
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
48
54
  void const* src, size_t srcSize);
49
- size_t ZSTD_compressBlock_lazy2_row(
55
+ size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
50
56
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
51
57
  void const* src, size_t srcSize);
52
- size_t ZSTD_compressBlock_lazy_row(
58
+ size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
53
59
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
54
60
  void const* src, size_t srcSize);
55
- size_t ZSTD_compressBlock_greedy_row(
61
+ size_t ZSTD_compressBlock_greedy_extDict(
56
62
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
57
63
  void const* src, size_t srcSize);
58
-
59
- size_t ZSTD_compressBlock_btlazy2_dictMatchState(
64
+ size_t ZSTD_compressBlock_greedy_extDict_row(
60
65
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
61
66
  void const* src, size_t srcSize);
62
- size_t ZSTD_compressBlock_lazy2_dictMatchState(
67
+
68
+ #define ZSTD_COMPRESSBLOCK_GREEDY ZSTD_compressBlock_greedy
69
+ #define ZSTD_COMPRESSBLOCK_GREEDY_ROW ZSTD_compressBlock_greedy_row
70
+ #define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE ZSTD_compressBlock_greedy_dictMatchState
71
+ #define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW ZSTD_compressBlock_greedy_dictMatchState_row
72
+ #define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH ZSTD_compressBlock_greedy_dedicatedDictSearch
73
+ #define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_greedy_dedicatedDictSearch_row
74
+ #define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT ZSTD_compressBlock_greedy_extDict
75
+ #define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW ZSTD_compressBlock_greedy_extDict_row
76
+ #else
77
+ #define ZSTD_COMPRESSBLOCK_GREEDY NULL
78
+ #define ZSTD_COMPRESSBLOCK_GREEDY_ROW NULL
79
+ #define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE NULL
80
+ #define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW NULL
81
+ #define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH NULL
82
+ #define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW NULL
83
+ #define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT NULL
84
+ #define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW NULL
85
+ #endif
86
+
87
+ #ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
88
+ size_t ZSTD_compressBlock_lazy(
63
89
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
64
90
  void const* src, size_t srcSize);
65
- size_t ZSTD_compressBlock_lazy_dictMatchState(
91
+ size_t ZSTD_compressBlock_lazy_row(
66
92
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
67
93
  void const* src, size_t srcSize);
68
- size_t ZSTD_compressBlock_greedy_dictMatchState(
94
+ size_t ZSTD_compressBlock_lazy_dictMatchState(
69
95
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
70
96
  void const* src, size_t srcSize);
71
- size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
97
+ size_t ZSTD_compressBlock_lazy_dictMatchState_row(
72
98
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
73
99
  void const* src, size_t srcSize);
74
- size_t ZSTD_compressBlock_lazy_dictMatchState_row(
100
+ size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
75
101
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
76
102
  void const* src, size_t srcSize);
77
- size_t ZSTD_compressBlock_greedy_dictMatchState_row(
103
+ size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
78
104
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
79
105
  void const* src, size_t srcSize);
80
-
81
- size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
106
+ size_t ZSTD_compressBlock_lazy_extDict(
82
107
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
83
108
  void const* src, size_t srcSize);
84
- size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
109
+ size_t ZSTD_compressBlock_lazy_extDict_row(
85
110
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
86
111
  void const* src, size_t srcSize);
87
- size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
112
+
113
+ #define ZSTD_COMPRESSBLOCK_LAZY ZSTD_compressBlock_lazy
114
+ #define ZSTD_COMPRESSBLOCK_LAZY_ROW ZSTD_compressBlock_lazy_row
115
+ #define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE ZSTD_compressBlock_lazy_dictMatchState
116
+ #define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW ZSTD_compressBlock_lazy_dictMatchState_row
117
+ #define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH ZSTD_compressBlock_lazy_dedicatedDictSearch
118
+ #define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_lazy_dedicatedDictSearch_row
119
+ #define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT ZSTD_compressBlock_lazy_extDict
120
+ #define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW ZSTD_compressBlock_lazy_extDict_row
121
+ #else
122
+ #define ZSTD_COMPRESSBLOCK_LAZY NULL
123
+ #define ZSTD_COMPRESSBLOCK_LAZY_ROW NULL
124
+ #define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE NULL
125
+ #define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW NULL
126
+ #define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH NULL
127
+ #define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW NULL
128
+ #define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT NULL
129
+ #define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW NULL
130
+ #endif
131
+
132
+ #ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
133
+ size_t ZSTD_compressBlock_lazy2(
88
134
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
89
135
  void const* src, size_t srcSize);
90
- size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
136
+ size_t ZSTD_compressBlock_lazy2_row(
91
137
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
92
138
  void const* src, size_t srcSize);
93
- size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
139
+ size_t ZSTD_compressBlock_lazy2_dictMatchState(
94
140
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
95
141
  void const* src, size_t srcSize);
96
- size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
142
+ size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
97
143
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
98
144
  void const* src, size_t srcSize);
99
-
100
- size_t ZSTD_compressBlock_greedy_extDict(
145
+ size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
101
146
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
102
147
  void const* src, size_t srcSize);
103
- size_t ZSTD_compressBlock_lazy_extDict(
148
+ size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
104
149
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
105
150
  void const* src, size_t srcSize);
106
151
  size_t ZSTD_compressBlock_lazy2_extDict(
107
152
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
108
153
  void const* src, size_t srcSize);
109
- size_t ZSTD_compressBlock_greedy_extDict_row(
154
+ size_t ZSTD_compressBlock_lazy2_extDict_row(
110
155
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
111
156
  void const* src, size_t srcSize);
112
- size_t ZSTD_compressBlock_lazy_extDict_row(
157
+
158
+ #define ZSTD_COMPRESSBLOCK_LAZY2 ZSTD_compressBlock_lazy2
159
+ #define ZSTD_COMPRESSBLOCK_LAZY2_ROW ZSTD_compressBlock_lazy2_row
160
+ #define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE ZSTD_compressBlock_lazy2_dictMatchState
161
+ #define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW ZSTD_compressBlock_lazy2_dictMatchState_row
162
+ #define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH ZSTD_compressBlock_lazy2_dedicatedDictSearch
163
+ #define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_lazy2_dedicatedDictSearch_row
164
+ #define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT ZSTD_compressBlock_lazy2_extDict
165
+ #define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW ZSTD_compressBlock_lazy2_extDict_row
166
+ #else
167
+ #define ZSTD_COMPRESSBLOCK_LAZY2 NULL
168
+ #define ZSTD_COMPRESSBLOCK_LAZY2_ROW NULL
169
+ #define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE NULL
170
+ #define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW NULL
171
+ #define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH NULL
172
+ #define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW NULL
173
+ #define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT NULL
174
+ #define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW NULL
175
+ #endif
176
+
177
+ #ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
178
+ size_t ZSTD_compressBlock_btlazy2(
113
179
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
114
180
  void const* src, size_t srcSize);
115
- size_t ZSTD_compressBlock_lazy2_extDict_row(
181
+ size_t ZSTD_compressBlock_btlazy2_dictMatchState(
116
182
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
117
183
  void const* src, size_t srcSize);
118
184
  size_t ZSTD_compressBlock_btlazy2_extDict(
119
185
  ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
120
186
  void const* src, size_t srcSize);
121
187
 
188
+ #define ZSTD_COMPRESSBLOCK_BTLAZY2 ZSTD_compressBlock_btlazy2
189
+ #define ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE ZSTD_compressBlock_btlazy2_dictMatchState
190
+ #define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT ZSTD_compressBlock_btlazy2_extDict
191
+ #else
192
+ #define ZSTD_COMPRESSBLOCK_BTLAZY2 NULL
193
+ #define ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE NULL
194
+ #define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT NULL
195
+ #endif
196
+
122
197
 
123
198
  #if defined (__cplusplus)
124
199
  }
@@ -246,7 +246,11 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
246
246
  break;
247
247
 
248
248
  case ZSTD_dfast:
249
+ #ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
249
250
  ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx);
251
+ #else
252
+ assert(0); /* shouldn't be called: cparams should've been adjusted. */
253
+ #endif
250
254
  break;
251
255
 
252
256
  case ZSTD_greedy:
@@ -318,7 +322,9 @@ static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
318
322
  }
319
323
  }
320
324
 
321
- static size_t ZSTD_ldm_generateSequences_internal(
325
+ static
326
+ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
327
+ size_t ZSTD_ldm_generateSequences_internal(
322
328
  ldmState_t* ldmState, rawSeqStore_t* rawSeqStore,
323
329
  ldmParams_t const* params, void const* src, size_t srcSize)
324
330
  {
@@ -689,7 +695,6 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
689
695
  /* maybeSplitSequence updates rawSeqStore->pos */
690
696
  rawSeq const sequence = maybeSplitSequence(rawSeqStore,
691
697
  (U32)(iend - ip), minMatch);
692
- int i;
693
698
  /* End signal */
694
699
  if (sequence.offset == 0)
695
700
  break;
@@ -702,6 +707,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
702
707
  /* Run the block compressor */
703
708
  DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength);
704
709
  {
710
+ int i;
705
711
  size_t const newLitLength =
706
712
  blockCompressor(ms, seqStore, rep, ip, sequence.litLength);
707
713
  ip += sequence.litLength;