brotli 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/brotli/brotli.cc +114 -24
- data/ext/brotli/brotli.h +0 -1
- data/ext/brotli/extconf.rb +30 -23
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/LICENSE +1 -1
- data/vendor/brotli/dec/Makefile +1 -1
- data/vendor/brotli/dec/bit_reader.c +3 -3
- data/vendor/brotli/dec/bit_reader.h +25 -27
- data/vendor/brotli/dec/context.h +4 -4
- data/vendor/brotli/dec/decode.c +410 -486
- data/vendor/brotli/dec/decode.h +101 -105
- data/vendor/brotli/dec/dictionary.c +1 -1
- data/vendor/brotli/dec/dictionary.h +7 -8
- data/vendor/brotli/dec/huffman.c +103 -105
- data/vendor/brotli/dec/huffman.h +18 -18
- data/vendor/brotli/dec/port.h +52 -40
- data/vendor/brotli/dec/prefix.h +2 -0
- data/vendor/brotli/dec/state.c +13 -19
- data/vendor/brotli/dec/state.h +25 -39
- data/vendor/brotli/dec/transform.h +38 -44
- data/vendor/brotli/dec/types.h +2 -2
- data/vendor/brotli/enc/Makefile +1 -1
- data/vendor/brotli/enc/backward_references.cc +455 -359
- data/vendor/brotli/enc/backward_references.h +79 -3
- data/vendor/brotli/enc/bit_cost.h +54 -32
- data/vendor/brotli/enc/block_splitter.cc +285 -193
- data/vendor/brotli/enc/block_splitter.h +4 -12
- data/vendor/brotli/enc/brotli_bit_stream.cc +623 -324
- data/vendor/brotli/enc/brotli_bit_stream.h +76 -37
- data/vendor/brotli/enc/cluster.h +161 -120
- data/vendor/brotli/enc/command.h +60 -37
- data/vendor/brotli/enc/compress_fragment.cc +701 -0
- data/vendor/brotli/enc/compress_fragment.h +47 -0
- data/vendor/brotli/enc/compress_fragment_two_pass.cc +524 -0
- data/vendor/brotli/enc/compress_fragment_two_pass.h +40 -0
- data/vendor/brotli/enc/compressor.h +15 -0
- data/vendor/brotli/enc/context.h +1 -1
- data/vendor/brotli/enc/dictionary.h +2 -2
- data/vendor/brotli/enc/encode.cc +819 -286
- data/vendor/brotli/enc/encode.h +38 -15
- data/vendor/brotli/enc/encode_parallel.cc +40 -42
- data/vendor/brotli/enc/entropy_encode.cc +144 -147
- data/vendor/brotli/enc/entropy_encode.h +32 -8
- data/vendor/brotli/enc/entropy_encode_static.h +572 -0
- data/vendor/brotli/enc/fast_log.h +7 -40
- data/vendor/brotli/enc/find_match_length.h +9 -9
- data/vendor/brotli/enc/hash.h +462 -154
- data/vendor/brotli/enc/histogram.cc +6 -6
- data/vendor/brotli/enc/histogram.h +13 -13
- data/vendor/brotli/enc/literal_cost.cc +45 -45
- data/vendor/brotli/enc/metablock.cc +92 -89
- data/vendor/brotli/enc/metablock.h +12 -12
- data/vendor/brotli/enc/port.h +7 -16
- data/vendor/brotli/enc/prefix.h +23 -22
- data/vendor/brotli/enc/ringbuffer.h +75 -29
- data/vendor/brotli/enc/static_dict.cc +56 -48
- data/vendor/brotli/enc/static_dict.h +5 -5
- data/vendor/brotli/enc/streams.cc +1 -1
- data/vendor/brotli/enc/streams.h +5 -5
- data/vendor/brotli/enc/transform.h +40 -35
- data/vendor/brotli/enc/types.h +2 -0
- data/vendor/brotli/enc/utf8_util.cc +3 -2
- data/vendor/brotli/enc/write_bits.h +6 -6
- metadata +9 -5
- data/vendor/brotli/dec/streams.c +0 -102
- data/vendor/brotli/dec/streams.h +0 -95
@@ -23,7 +23,7 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
23
23
|
uint8_t prev_byte2,
|
24
24
|
const Command* cmds,
|
25
25
|
size_t num_commands,
|
26
|
-
|
26
|
+
ContextType literal_context_mode,
|
27
27
|
MetaBlockSplit* mb) {
|
28
28
|
SplitBlock(cmds, num_commands,
|
29
29
|
ringbuffer, pos, mask,
|
@@ -31,12 +31,12 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
31
31
|
&mb->command_split,
|
32
32
|
&mb->distance_split);
|
33
33
|
|
34
|
-
std::vector<
|
35
|
-
|
34
|
+
std::vector<ContextType> literal_context_modes(mb->literal_split.num_types,
|
35
|
+
literal_context_mode);
|
36
36
|
|
37
|
-
|
37
|
+
size_t num_literal_contexts =
|
38
38
|
mb->literal_split.num_types << kLiteralContextBits;
|
39
|
-
|
39
|
+
size_t num_distance_contexts =
|
40
40
|
mb->distance_split.num_types << kDistanceContextBits;
|
41
41
|
std::vector<HistogramLiteral> literal_histograms(num_literal_contexts);
|
42
42
|
mb->command_histograms.resize(mb->command_split.num_types);
|
@@ -58,17 +58,15 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
58
58
|
// Histogram ids need to fit in one byte.
|
59
59
|
static const size_t kMaxNumberOfHistograms = 256;
|
60
60
|
|
61
|
-
mb->literal_histograms = literal_histograms;
|
62
61
|
ClusterHistograms(literal_histograms,
|
63
|
-
|
62
|
+
1u << kLiteralContextBits,
|
64
63
|
mb->literal_split.num_types,
|
65
64
|
kMaxNumberOfHistograms,
|
66
65
|
&mb->literal_histograms,
|
67
66
|
&mb->literal_context_map);
|
68
67
|
|
69
|
-
mb->distance_histograms = distance_histograms;
|
70
68
|
ClusterHistograms(distance_histograms,
|
71
|
-
|
69
|
+
1u << kDistanceContextBits,
|
72
70
|
mb->distance_split.num_types,
|
73
71
|
kMaxNumberOfHistograms,
|
74
72
|
&mb->distance_histograms,
|
@@ -79,10 +77,10 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
79
77
|
template<typename HistogramType>
|
80
78
|
class BlockSplitter {
|
81
79
|
public:
|
82
|
-
BlockSplitter(
|
83
|
-
|
80
|
+
BlockSplitter(size_t alphabet_size,
|
81
|
+
size_t min_block_size,
|
84
82
|
double split_threshold,
|
85
|
-
|
83
|
+
size_t num_symbols,
|
86
84
|
BlockSplit* split,
|
87
85
|
std::vector<HistogramType>* histograms)
|
88
86
|
: alphabet_size_(alphabet_size),
|
@@ -95,10 +93,10 @@ class BlockSplitter {
|
|
95
93
|
block_size_(0),
|
96
94
|
curr_histogram_ix_(0),
|
97
95
|
merge_last_count_(0) {
|
98
|
-
|
96
|
+
size_t max_num_blocks = num_symbols / min_block_size + 1;
|
99
97
|
// We have to allocate one more histogram than the maximum number of block
|
100
98
|
// types for the current histogram when the meta-block is too big.
|
101
|
-
|
99
|
+
size_t max_num_types = std::min<size_t>(max_num_blocks, kMaxBlockTypes + 1);
|
102
100
|
split_->lengths.resize(max_num_blocks);
|
103
101
|
split_->types.resize(max_num_blocks);
|
104
102
|
histograms_->resize(max_num_types);
|
@@ -107,7 +105,7 @@ class BlockSplitter {
|
|
107
105
|
|
108
106
|
// Adds the next symbol to the current histogram. When the current histogram
|
109
107
|
// reaches the target size, decides on merging the block.
|
110
|
-
void AddSymbol(
|
108
|
+
void AddSymbol(size_t symbol) {
|
111
109
|
(*histograms_)[curr_histogram_ix_].Add(symbol);
|
112
110
|
++block_size_;
|
113
111
|
if (block_size_ == target_block_size_) {
|
@@ -125,7 +123,7 @@ class BlockSplitter {
|
|
125
123
|
}
|
126
124
|
if (num_blocks_ == 0) {
|
127
125
|
// Create first block.
|
128
|
-
split_->lengths[0] = block_size_;
|
126
|
+
split_->lengths[0] = static_cast<uint32_t>(block_size_);
|
129
127
|
split_->types[0] = 0;
|
130
128
|
last_entropy_[0] =
|
131
129
|
BitsEntropy(&(*histograms_)[0].data_[0], alphabet_size_);
|
@@ -140,8 +138,8 @@ class BlockSplitter {
|
|
140
138
|
HistogramType combined_histo[2];
|
141
139
|
double combined_entropy[2];
|
142
140
|
double diff[2];
|
143
|
-
for (
|
144
|
-
|
141
|
+
for (size_t j = 0; j < 2; ++j) {
|
142
|
+
size_t last_histogram_ix = last_histogram_ix_[j];
|
145
143
|
combined_histo[j] = (*histograms_)[curr_histogram_ix_];
|
146
144
|
combined_histo[j].AddHistogram((*histograms_)[last_histogram_ix]);
|
147
145
|
combined_entropy[j] = BitsEntropy(
|
@@ -153,10 +151,10 @@ class BlockSplitter {
|
|
153
151
|
diff[0] > split_threshold_ &&
|
154
152
|
diff[1] > split_threshold_) {
|
155
153
|
// Create new block.
|
156
|
-
split_->lengths[num_blocks_] = block_size_;
|
157
|
-
split_->types[num_blocks_] = split_->num_types;
|
154
|
+
split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
|
155
|
+
split_->types[num_blocks_] = static_cast<uint8_t>(split_->num_types);
|
158
156
|
last_histogram_ix_[1] = last_histogram_ix_[0];
|
159
|
-
last_histogram_ix_[0] = split_->num_types;
|
157
|
+
last_histogram_ix_[0] = static_cast<uint8_t>(split_->num_types);
|
160
158
|
last_entropy_[1] = last_entropy_[0];
|
161
159
|
last_entropy_[0] = entropy;
|
162
160
|
++num_blocks_;
|
@@ -167,7 +165,7 @@ class BlockSplitter {
|
|
167
165
|
target_block_size_ = min_block_size_;
|
168
166
|
} else if (diff[1] < diff[0] - 20.0) {
|
169
167
|
// Combine this block with second last block.
|
170
|
-
split_->lengths[num_blocks_] = block_size_;
|
168
|
+
split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
|
171
169
|
split_->types[num_blocks_] = split_->types[num_blocks_ - 2];
|
172
170
|
std::swap(last_histogram_ix_[0], last_histogram_ix_[1]);
|
173
171
|
(*histograms_)[last_histogram_ix_[0]] = combined_histo[1];
|
@@ -180,7 +178,7 @@ class BlockSplitter {
|
|
180
178
|
target_block_size_ = min_block_size_;
|
181
179
|
} else {
|
182
180
|
// Combine this block with last block.
|
183
|
-
split_->lengths[num_blocks_ - 1] += block_size_;
|
181
|
+
split_->lengths[num_blocks_ - 1] += static_cast<uint32_t>(block_size_);
|
184
182
|
(*histograms_)[last_histogram_ix_[0]] = combined_histo[0];
|
185
183
|
last_entropy_[0] = combined_entropy[0];
|
186
184
|
if (split_->num_types == 1) {
|
@@ -201,35 +199,35 @@ class BlockSplitter {
|
|
201
199
|
}
|
202
200
|
|
203
201
|
private:
|
204
|
-
static const
|
202
|
+
static const uint16_t kMaxBlockTypes = 256;
|
205
203
|
|
206
204
|
// Alphabet size of particular block category.
|
207
|
-
const
|
205
|
+
const size_t alphabet_size_;
|
208
206
|
// We collect at least this many symbols for each block.
|
209
|
-
const
|
207
|
+
const size_t min_block_size_;
|
210
208
|
// We merge histograms A and B if
|
211
209
|
// entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,
|
212
210
|
// where A is the current histogram and B is the histogram of the last or the
|
213
211
|
// second last block type.
|
214
212
|
const double split_threshold_;
|
215
213
|
|
216
|
-
|
214
|
+
size_t num_blocks_;
|
217
215
|
BlockSplit* split_; // not owned
|
218
216
|
std::vector<HistogramType>* histograms_; // not owned
|
219
217
|
|
220
218
|
// The number of symbols that we want to collect before deciding on whether
|
221
219
|
// or not to merge the block with a previous one or emit a new block.
|
222
|
-
|
220
|
+
size_t target_block_size_;
|
223
221
|
// The number of symbols in the current histogram.
|
224
|
-
|
222
|
+
size_t block_size_;
|
225
223
|
// Offset of the current histogram.
|
226
|
-
|
224
|
+
size_t curr_histogram_ix_;
|
227
225
|
// Offset of the histograms of the previous two block types.
|
228
|
-
|
226
|
+
size_t last_histogram_ix_[2];
|
229
227
|
// Entropy of the previous two block types.
|
230
228
|
double last_entropy_[2];
|
231
229
|
// The number of times we merged the current block with the last one.
|
232
|
-
|
230
|
+
size_t merge_last_count_;
|
233
231
|
};
|
234
232
|
|
235
233
|
void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
|
@@ -238,7 +236,7 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
|
|
238
236
|
const Command *commands,
|
239
237
|
size_t n_commands,
|
240
238
|
MetaBlockSplit* mb) {
|
241
|
-
|
239
|
+
size_t num_literals = 0;
|
242
240
|
for (size_t i = 0; i < n_commands; ++i) {
|
243
241
|
num_literals += commands[i].insert_len_;
|
244
242
|
}
|
@@ -247,21 +245,21 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
|
|
247
245
|
256, 512, 400.0, num_literals,
|
248
246
|
&mb->literal_split, &mb->literal_histograms);
|
249
247
|
BlockSplitter<HistogramCommand> cmd_blocks(
|
250
|
-
kNumCommandPrefixes, 1024, 500.0,
|
248
|
+
kNumCommandPrefixes, 1024, 500.0, n_commands,
|
251
249
|
&mb->command_split, &mb->command_histograms);
|
252
250
|
BlockSplitter<HistogramDistance> dist_blocks(
|
253
|
-
64, 512, 100.0,
|
251
|
+
64, 512, 100.0, n_commands,
|
254
252
|
&mb->distance_split, &mb->distance_histograms);
|
255
253
|
|
256
254
|
for (size_t i = 0; i < n_commands; ++i) {
|
257
255
|
const Command cmd = commands[i];
|
258
256
|
cmd_blocks.AddSymbol(cmd.cmd_prefix_);
|
259
|
-
for (
|
257
|
+
for (size_t j = cmd.insert_len_; j != 0; --j) {
|
260
258
|
lit_blocks.AddSymbol(ringbuffer[pos & mask]);
|
261
259
|
++pos;
|
262
260
|
}
|
263
|
-
pos += cmd.
|
264
|
-
if (cmd.
|
261
|
+
pos += cmd.copy_len();
|
262
|
+
if (cmd.copy_len() && cmd.cmd_prefix_ >= 128) {
|
265
263
|
dist_blocks.AddSymbol(cmd.dist_prefix_);
|
266
264
|
}
|
267
265
|
}
|
@@ -276,11 +274,11 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
|
|
276
274
|
template<typename HistogramType>
|
277
275
|
class ContextBlockSplitter {
|
278
276
|
public:
|
279
|
-
ContextBlockSplitter(
|
280
|
-
|
281
|
-
|
277
|
+
ContextBlockSplitter(size_t alphabet_size,
|
278
|
+
size_t num_contexts,
|
279
|
+
size_t min_block_size,
|
282
280
|
double split_threshold,
|
283
|
-
|
281
|
+
size_t num_symbols,
|
284
282
|
BlockSplit* split,
|
285
283
|
std::vector<HistogramType>* histograms)
|
286
284
|
: alphabet_size_(alphabet_size),
|
@@ -296,10 +294,10 @@ class ContextBlockSplitter {
|
|
296
294
|
curr_histogram_ix_(0),
|
297
295
|
last_entropy_(2 * num_contexts),
|
298
296
|
merge_last_count_(0) {
|
299
|
-
|
297
|
+
size_t max_num_blocks = num_symbols / min_block_size + 1;
|
300
298
|
// We have to allocate one more histogram than the maximum number of block
|
301
299
|
// types for the current histogram when the meta-block is too big.
|
302
|
-
|
300
|
+
size_t max_num_types = std::min(max_num_blocks, max_block_types_ + 1);
|
303
301
|
split_->lengths.resize(max_num_blocks);
|
304
302
|
split_->types.resize(max_num_blocks);
|
305
303
|
histograms_->resize(max_num_types * num_contexts);
|
@@ -308,7 +306,7 @@ class ContextBlockSplitter {
|
|
308
306
|
|
309
307
|
// Adds the next symbol to the current block type and context. When the
|
310
308
|
// current block reaches the target size, decides on merging the block.
|
311
|
-
void AddSymbol(
|
309
|
+
void AddSymbol(size_t symbol, size_t context) {
|
312
310
|
(*histograms_)[curr_histogram_ix_ + context].Add(symbol);
|
313
311
|
++block_size_;
|
314
312
|
if (block_size_ == target_block_size_) {
|
@@ -326,9 +324,9 @@ class ContextBlockSplitter {
|
|
326
324
|
}
|
327
325
|
if (num_blocks_ == 0) {
|
328
326
|
// Create first block.
|
329
|
-
split_->lengths[0] = block_size_;
|
327
|
+
split_->lengths[0] = static_cast<uint32_t>(block_size_);
|
330
328
|
split_->types[0] = 0;
|
331
|
-
for (
|
329
|
+
for (size_t i = 0; i < num_contexts_; ++i) {
|
332
330
|
last_entropy_[i] =
|
333
331
|
BitsEntropy(&(*histograms_)[i].data_[0], alphabet_size_);
|
334
332
|
last_entropy_[num_contexts_ + i] = last_entropy_[i];
|
@@ -346,13 +344,13 @@ class ContextBlockSplitter {
|
|
346
344
|
std::vector<HistogramType> combined_histo(2 * num_contexts_);
|
347
345
|
std::vector<double> combined_entropy(2 * num_contexts_);
|
348
346
|
double diff[2] = { 0.0 };
|
349
|
-
for (
|
350
|
-
|
347
|
+
for (size_t i = 0; i < num_contexts_; ++i) {
|
348
|
+
size_t curr_histo_ix = curr_histogram_ix_ + i;
|
351
349
|
entropy[i] = BitsEntropy(&(*histograms_)[curr_histo_ix].data_[0],
|
352
350
|
alphabet_size_);
|
353
|
-
for (
|
354
|
-
|
355
|
-
|
351
|
+
for (size_t j = 0; j < 2; ++j) {
|
352
|
+
size_t jx = j * num_contexts_ + i;
|
353
|
+
size_t last_histogram_ix = last_histogram_ix_[j] + i;
|
356
354
|
combined_histo[jx] = (*histograms_)[curr_histo_ix];
|
357
355
|
combined_histo[jx].AddHistogram((*histograms_)[last_histogram_ix]);
|
358
356
|
combined_entropy[jx] = BitsEntropy(
|
@@ -365,11 +363,11 @@ class ContextBlockSplitter {
|
|
365
363
|
diff[0] > split_threshold_ &&
|
366
364
|
diff[1] > split_threshold_) {
|
367
365
|
// Create new block.
|
368
|
-
split_->lengths[num_blocks_] = block_size_;
|
369
|
-
split_->types[num_blocks_] = split_->num_types;
|
366
|
+
split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
|
367
|
+
split_->types[num_blocks_] = static_cast<uint8_t>(split_->num_types);
|
370
368
|
last_histogram_ix_[1] = last_histogram_ix_[0];
|
371
369
|
last_histogram_ix_[0] = split_->num_types * num_contexts_;
|
372
|
-
for (
|
370
|
+
for (size_t i = 0; i < num_contexts_; ++i) {
|
373
371
|
last_entropy_[num_contexts_ + i] = last_entropy_[i];
|
374
372
|
last_entropy_[i] = entropy[i];
|
375
373
|
}
|
@@ -381,10 +379,10 @@ class ContextBlockSplitter {
|
|
381
379
|
target_block_size_ = min_block_size_;
|
382
380
|
} else if (diff[1] < diff[0] - 20.0) {
|
383
381
|
// Combine this block with second last block.
|
384
|
-
split_->lengths[num_blocks_] = block_size_;
|
382
|
+
split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
|
385
383
|
split_->types[num_blocks_] = split_->types[num_blocks_ - 2];
|
386
384
|
std::swap(last_histogram_ix_[0], last_histogram_ix_[1]);
|
387
|
-
for (
|
385
|
+
for (size_t i = 0; i < num_contexts_; ++i) {
|
388
386
|
(*histograms_)[last_histogram_ix_[0] + i] =
|
389
387
|
combined_histo[num_contexts_ + i];
|
390
388
|
last_entropy_[num_contexts_ + i] = last_entropy_[i];
|
@@ -397,8 +395,8 @@ class ContextBlockSplitter {
|
|
397
395
|
target_block_size_ = min_block_size_;
|
398
396
|
} else {
|
399
397
|
// Combine this block with last block.
|
400
|
-
split_->lengths[num_blocks_ - 1] += block_size_;
|
401
|
-
for (
|
398
|
+
split_->lengths[num_blocks_ - 1] += static_cast<uint32_t>(block_size_);
|
399
|
+
for (size_t i = 0; i < num_contexts_; ++i) {
|
402
400
|
(*histograms_)[last_histogram_ix_[0] + i] = combined_histo[i];
|
403
401
|
last_entropy_[i] = combined_entropy[i];
|
404
402
|
if (split_->num_types == 1) {
|
@@ -423,34 +421,34 @@ class ContextBlockSplitter {
|
|
423
421
|
static const int kMaxBlockTypes = 256;
|
424
422
|
|
425
423
|
// Alphabet size of particular block category.
|
426
|
-
const
|
427
|
-
const
|
428
|
-
const
|
424
|
+
const size_t alphabet_size_;
|
425
|
+
const size_t num_contexts_;
|
426
|
+
const size_t max_block_types_;
|
429
427
|
// We collect at least this many symbols for each block.
|
430
|
-
const
|
428
|
+
const size_t min_block_size_;
|
431
429
|
// We merge histograms A and B if
|
432
430
|
// entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,
|
433
431
|
// where A is the current histogram and B is the histogram of the last or the
|
434
432
|
// second last block type.
|
435
433
|
const double split_threshold_;
|
436
434
|
|
437
|
-
|
435
|
+
size_t num_blocks_;
|
438
436
|
BlockSplit* split_; // not owned
|
439
437
|
std::vector<HistogramType>* histograms_; // not owned
|
440
438
|
|
441
439
|
// The number of symbols that we want to collect before deciding on whether
|
442
440
|
// or not to merge the block with a previous one or emit a new block.
|
443
|
-
|
441
|
+
size_t target_block_size_;
|
444
442
|
// The number of symbols in the current histogram.
|
445
|
-
|
443
|
+
size_t block_size_;
|
446
444
|
// Offset of the current histogram.
|
447
|
-
|
445
|
+
size_t curr_histogram_ix_;
|
448
446
|
// Offset of the histograms of the previous two block types.
|
449
|
-
|
447
|
+
size_t last_histogram_ix_[2];
|
450
448
|
// Entropy of the previous two block types.
|
451
449
|
std::vector<double> last_entropy_;
|
452
450
|
// The number of times we merged the current block with the last one.
|
453
|
-
|
451
|
+
size_t merge_last_count_;
|
454
452
|
};
|
455
453
|
|
456
454
|
void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
@@ -458,13 +456,13 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
458
456
|
size_t mask,
|
459
457
|
uint8_t prev_byte,
|
460
458
|
uint8_t prev_byte2,
|
461
|
-
|
462
|
-
|
463
|
-
const
|
459
|
+
ContextType literal_context_mode,
|
460
|
+
size_t num_contexts,
|
461
|
+
const uint32_t* static_context_map,
|
464
462
|
const Command *commands,
|
465
463
|
size_t n_commands,
|
466
464
|
MetaBlockSplit* mb) {
|
467
|
-
|
465
|
+
size_t num_literals = 0;
|
468
466
|
for (size_t i = 0; i < n_commands; ++i) {
|
469
467
|
num_literals += commands[i].insert_len_;
|
470
468
|
}
|
@@ -473,25 +471,25 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
473
471
|
256, num_contexts, 512, 400.0, num_literals,
|
474
472
|
&mb->literal_split, &mb->literal_histograms);
|
475
473
|
BlockSplitter<HistogramCommand> cmd_blocks(
|
476
|
-
kNumCommandPrefixes, 1024, 500.0,
|
474
|
+
kNumCommandPrefixes, 1024, 500.0, n_commands,
|
477
475
|
&mb->command_split, &mb->command_histograms);
|
478
476
|
BlockSplitter<HistogramDistance> dist_blocks(
|
479
|
-
64, 512, 100.0,
|
477
|
+
64, 512, 100.0, n_commands,
|
480
478
|
&mb->distance_split, &mb->distance_histograms);
|
481
479
|
|
482
480
|
for (size_t i = 0; i < n_commands; ++i) {
|
483
481
|
const Command cmd = commands[i];
|
484
482
|
cmd_blocks.AddSymbol(cmd.cmd_prefix_);
|
485
|
-
for (
|
486
|
-
|
483
|
+
for (size_t j = cmd.insert_len_; j != 0; --j) {
|
484
|
+
size_t context = Context(prev_byte, prev_byte2, literal_context_mode);
|
487
485
|
uint8_t literal = ringbuffer[pos & mask];
|
488
486
|
lit_blocks.AddSymbol(literal, static_context_map[context]);
|
489
487
|
prev_byte2 = prev_byte;
|
490
488
|
prev_byte = literal;
|
491
489
|
++pos;
|
492
490
|
}
|
493
|
-
pos += cmd.
|
494
|
-
if (cmd.
|
491
|
+
pos += cmd.copy_len();
|
492
|
+
if (cmd.copy_len()) {
|
495
493
|
prev_byte2 = ringbuffer[(pos - 2) & mask];
|
496
494
|
prev_byte = ringbuffer[(pos - 1) & mask];
|
497
495
|
if (cmd.cmd_prefix_ >= 128) {
|
@@ -506,31 +504,36 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
506
504
|
|
507
505
|
mb->literal_context_map.resize(
|
508
506
|
mb->literal_split.num_types << kLiteralContextBits);
|
509
|
-
for (
|
510
|
-
for (
|
507
|
+
for (size_t i = 0; i < mb->literal_split.num_types; ++i) {
|
508
|
+
for (size_t j = 0; j < (1u << kLiteralContextBits); ++j) {
|
511
509
|
mb->literal_context_map[(i << kLiteralContextBits) + j] =
|
512
|
-
i * num_contexts + static_context_map[j];
|
510
|
+
static_cast<uint32_t>(i * num_contexts) + static_context_map[j];
|
513
511
|
}
|
514
512
|
}
|
515
513
|
}
|
516
514
|
|
517
|
-
void OptimizeHistograms(
|
518
|
-
|
515
|
+
void OptimizeHistograms(size_t num_direct_distance_codes,
|
516
|
+
size_t distance_postfix_bits,
|
519
517
|
MetaBlockSplit* mb) {
|
518
|
+
uint8_t* good_for_rle = new uint8_t[kNumCommandPrefixes];
|
520
519
|
for (size_t i = 0; i < mb->literal_histograms.size(); ++i) {
|
521
|
-
OptimizeHuffmanCountsForRle(256, &mb->literal_histograms[i].data_[0]
|
520
|
+
OptimizeHuffmanCountsForRle(256, &mb->literal_histograms[i].data_[0],
|
521
|
+
good_for_rle);
|
522
522
|
}
|
523
523
|
for (size_t i = 0; i < mb->command_histograms.size(); ++i) {
|
524
524
|
OptimizeHuffmanCountsForRle(kNumCommandPrefixes,
|
525
|
-
&mb->command_histograms[i].data_[0]
|
525
|
+
&mb->command_histograms[i].data_[0],
|
526
|
+
good_for_rle);
|
526
527
|
}
|
527
|
-
|
528
|
+
size_t num_distance_codes =
|
528
529
|
kNumDistanceShortCodes + num_direct_distance_codes +
|
529
|
-
(
|
530
|
+
(48u << distance_postfix_bits);
|
530
531
|
for (size_t i = 0; i < mb->distance_histograms.size(); ++i) {
|
531
532
|
OptimizeHuffmanCountsForRle(num_distance_codes,
|
532
|
-
&mb->distance_histograms[i].data_[0]
|
533
|
+
&mb->distance_histograms[i].data_[0],
|
534
|
+
good_for_rle);
|
533
535
|
}
|
536
|
+
delete[] good_for_rle;
|
534
537
|
}
|
535
538
|
|
536
539
|
} // namespace brotli
|
@@ -18,19 +18,19 @@
|
|
18
18
|
namespace brotli {
|
19
19
|
|
20
20
|
struct BlockSplit {
|
21
|
-
BlockSplit() : num_types(0) {}
|
21
|
+
BlockSplit(void) : num_types(0) {}
|
22
22
|
|
23
|
-
|
24
|
-
std::vector<
|
25
|
-
std::vector<
|
23
|
+
size_t num_types;
|
24
|
+
std::vector<uint8_t> types;
|
25
|
+
std::vector<uint32_t> lengths;
|
26
26
|
};
|
27
27
|
|
28
28
|
struct MetaBlockSplit {
|
29
29
|
BlockSplit literal_split;
|
30
30
|
BlockSplit command_split;
|
31
31
|
BlockSplit distance_split;
|
32
|
-
std::vector<
|
33
|
-
std::vector<
|
32
|
+
std::vector<uint32_t> literal_context_map;
|
33
|
+
std::vector<uint32_t> distance_context_map;
|
34
34
|
std::vector<HistogramLiteral> literal_histograms;
|
35
35
|
std::vector<HistogramCommand> command_histograms;
|
36
36
|
std::vector<HistogramDistance> distance_histograms;
|
@@ -44,7 +44,7 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
44
44
|
uint8_t prev_byte2,
|
45
45
|
const Command* cmds,
|
46
46
|
size_t num_commands,
|
47
|
-
|
47
|
+
ContextType literal_context_mode,
|
48
48
|
MetaBlockSplit* mb);
|
49
49
|
|
50
50
|
// Uses a fast greedy block splitter that tries to merge current block with the
|
@@ -64,15 +64,15 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
64
64
|
size_t mask,
|
65
65
|
uint8_t prev_byte,
|
66
66
|
uint8_t prev_byte2,
|
67
|
-
|
68
|
-
|
69
|
-
const
|
67
|
+
ContextType literal_context_mode,
|
68
|
+
size_t num_contexts,
|
69
|
+
const uint32_t* static_context_map,
|
70
70
|
const Command *commands,
|
71
71
|
size_t n_commands,
|
72
72
|
MetaBlockSplit* mb);
|
73
73
|
|
74
|
-
void OptimizeHistograms(
|
75
|
-
|
74
|
+
void OptimizeHistograms(size_t num_direct_distance_codes,
|
75
|
+
size_t distance_postfix_bits,
|
76
76
|
MetaBlockSplit* mb);
|
77
77
|
|
78
78
|
} // namespace brotli
|
data/vendor/brotli/enc/port.h
CHANGED
@@ -22,10 +22,9 @@
|
|
22
22
|
/* Let's try and follow the Linux convention */
|
23
23
|
#define __BYTE_ORDER BYTE_ORDER
|
24
24
|
#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
25
|
-
#define __BIG_ENDIAN BIG_ENDIAN
|
26
25
|
#endif
|
27
26
|
|
28
|
-
// define the
|
27
|
+
// define the macro IS_LITTLE_ENDIAN
|
29
28
|
// using the above endian definitions from endian.h if
|
30
29
|
// endian.h was included
|
31
30
|
#ifdef __BYTE_ORDER
|
@@ -33,19 +32,17 @@
|
|
33
32
|
#define IS_LITTLE_ENDIAN
|
34
33
|
#endif
|
35
34
|
|
36
|
-
#if __BYTE_ORDER == __BIG_ENDIAN
|
37
|
-
#define IS_BIG_ENDIAN
|
38
|
-
#endif
|
39
|
-
|
40
35
|
#else
|
41
36
|
|
42
37
|
#if defined(__LITTLE_ENDIAN__)
|
43
38
|
#define IS_LITTLE_ENDIAN
|
44
|
-
#elif defined(__BIG_ENDIAN__)
|
45
|
-
#define IS_BIG_ENDIAN
|
46
39
|
#endif
|
47
40
|
#endif // __BYTE_ORDER
|
48
41
|
|
42
|
+
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
43
|
+
#define IS_LITTLE_ENDIAN
|
44
|
+
#endif
|
45
|
+
|
49
46
|
// Enable little-endian optimization for x64 architecture on Windows.
|
50
47
|
#if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
|
51
48
|
#define IS_LITTLE_ENDIAN
|
@@ -69,8 +66,8 @@
|
|
69
66
|
// On some platforms, like ARM, the copy functions can be more efficient
|
70
67
|
// then a load and a store.
|
71
68
|
|
72
|
-
#if defined(ARCH_PIII) ||
|
73
|
-
defined(ARCH_K8) || defined(_ARCH_PPC)
|
69
|
+
#if defined(ARCH_PIII) || \
|
70
|
+
defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC)
|
74
71
|
|
75
72
|
// x86 and x86-64 can perform unaligned loads/stores directly;
|
76
73
|
// modern PowerPC hardware can also do unaligned integer loads and stores;
|
@@ -142,10 +139,4 @@ inline void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
|
|
142
139
|
|
143
140
|
#endif
|
144
141
|
|
145
|
-
#ifdef BROTLI_ENCODE_DEBUG
|
146
|
-
#define BROTLI_DCHECK(x) assert(x)
|
147
|
-
#else
|
148
|
-
#define BROTLI_DCHECK(x)
|
149
|
-
#endif
|
150
|
-
|
151
142
|
#endif // BROTLI_ENC_PORT_H_
|
data/vendor/brotli/enc/prefix.h
CHANGED
@@ -15,18 +15,18 @@
|
|
15
15
|
|
16
16
|
namespace brotli {
|
17
17
|
|
18
|
-
static const
|
19
|
-
static const
|
20
|
-
static const
|
21
|
-
static const
|
22
|
-
static const
|
23
|
-
static const
|
18
|
+
static const uint32_t kNumInsertLenPrefixes = 24;
|
19
|
+
static const uint32_t kNumCopyLenPrefixes = 24;
|
20
|
+
static const uint32_t kNumCommandPrefixes = 704;
|
21
|
+
static const uint32_t kNumBlockLenPrefixes = 26;
|
22
|
+
static const uint32_t kNumDistanceShortCodes = 16;
|
23
|
+
static const uint32_t kNumDistancePrefixes = 520;
|
24
24
|
|
25
25
|
// Represents the range of values belonging to a prefix code:
|
26
26
|
// [offset, offset + 2^nbits)
|
27
27
|
struct PrefixCodeRange {
|
28
|
-
|
29
|
-
|
28
|
+
uint32_t offset;
|
29
|
+
uint32_t nbits;
|
30
30
|
};
|
31
31
|
|
32
32
|
static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
|
@@ -39,8 +39,8 @@ static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
|
|
39
39
|
{8433, 13}, {16625, 24}
|
40
40
|
};
|
41
41
|
|
42
|
-
inline void GetBlockLengthPrefixCode(
|
43
|
-
|
42
|
+
inline void GetBlockLengthPrefixCode(uint32_t len, uint32_t* code,
|
43
|
+
uint32_t* n_extra, uint32_t* extra) {
|
44
44
|
*code = 0;
|
45
45
|
while (*code < 25 && len >= kBlockLengthPrefixCode[*code + 1].offset) {
|
46
46
|
++(*code);
|
@@ -49,9 +49,9 @@ inline void GetBlockLengthPrefixCode(int len,
|
|
49
49
|
*extra = len - kBlockLengthPrefixCode[*code].offset;
|
50
50
|
}
|
51
51
|
|
52
|
-
inline void PrefixEncodeCopyDistance(
|
53
|
-
|
54
|
-
|
52
|
+
inline void PrefixEncodeCopyDistance(size_t distance_code,
|
53
|
+
size_t num_direct_codes,
|
54
|
+
size_t postfix_bits,
|
55
55
|
uint16_t* code,
|
56
56
|
uint32_t* extra_bits) {
|
57
57
|
if (distance_code < kNumDistanceShortCodes + num_direct_codes) {
|
@@ -59,18 +59,19 @@ inline void PrefixEncodeCopyDistance(int distance_code,
|
|
59
59
|
*extra_bits = 0;
|
60
60
|
return;
|
61
61
|
}
|
62
|
-
distance_code -= kNumDistanceShortCodes + num_direct_codes;
|
63
|
-
distance_code += (
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
62
|
+
distance_code -= kNumDistanceShortCodes + num_direct_codes; /* >= 0 */
|
63
|
+
distance_code += (1u << (postfix_bits + 2u)); /* > 0 */
|
64
|
+
size_t bucket = Log2FloorNonZero(distance_code) - 1;
|
65
|
+
size_t postfix_mask = (1 << postfix_bits) - 1;
|
66
|
+
size_t postfix = distance_code & postfix_mask;
|
67
|
+
size_t prefix = (distance_code >> bucket) & 1;
|
68
|
+
size_t offset = (2 + prefix) << bucket;
|
69
|
+
size_t nbits = bucket - postfix_bits;
|
70
70
|
*code = static_cast<uint16_t>(
|
71
71
|
(kNumDistanceShortCodes + num_direct_codes +
|
72
72
|
((2 * (nbits - 1) + prefix) << postfix_bits) + postfix));
|
73
|
-
*extra_bits = (
|
73
|
+
*extra_bits = static_cast<uint32_t>(
|
74
|
+
(nbits << 24) | ((distance_code - offset) >> postfix_bits));
|
74
75
|
}
|
75
76
|
|
76
77
|
} // namespace brotli
|