brotli 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/brotli/brotli.cc +114 -24
- data/ext/brotli/brotli.h +0 -1
- data/ext/brotli/extconf.rb +30 -23
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/LICENSE +1 -1
- data/vendor/brotli/dec/Makefile +1 -1
- data/vendor/brotli/dec/bit_reader.c +3 -3
- data/vendor/brotli/dec/bit_reader.h +25 -27
- data/vendor/brotli/dec/context.h +4 -4
- data/vendor/brotli/dec/decode.c +410 -486
- data/vendor/brotli/dec/decode.h +101 -105
- data/vendor/brotli/dec/dictionary.c +1 -1
- data/vendor/brotli/dec/dictionary.h +7 -8
- data/vendor/brotli/dec/huffman.c +103 -105
- data/vendor/brotli/dec/huffman.h +18 -18
- data/vendor/brotli/dec/port.h +52 -40
- data/vendor/brotli/dec/prefix.h +2 -0
- data/vendor/brotli/dec/state.c +13 -19
- data/vendor/brotli/dec/state.h +25 -39
- data/vendor/brotli/dec/transform.h +38 -44
- data/vendor/brotli/dec/types.h +2 -2
- data/vendor/brotli/enc/Makefile +1 -1
- data/vendor/brotli/enc/backward_references.cc +455 -359
- data/vendor/brotli/enc/backward_references.h +79 -3
- data/vendor/brotli/enc/bit_cost.h +54 -32
- data/vendor/brotli/enc/block_splitter.cc +285 -193
- data/vendor/brotli/enc/block_splitter.h +4 -12
- data/vendor/brotli/enc/brotli_bit_stream.cc +623 -324
- data/vendor/brotli/enc/brotli_bit_stream.h +76 -37
- data/vendor/brotli/enc/cluster.h +161 -120
- data/vendor/brotli/enc/command.h +60 -37
- data/vendor/brotli/enc/compress_fragment.cc +701 -0
- data/vendor/brotli/enc/compress_fragment.h +47 -0
- data/vendor/brotli/enc/compress_fragment_two_pass.cc +524 -0
- data/vendor/brotli/enc/compress_fragment_two_pass.h +40 -0
- data/vendor/brotli/enc/compressor.h +15 -0
- data/vendor/brotli/enc/context.h +1 -1
- data/vendor/brotli/enc/dictionary.h +2 -2
- data/vendor/brotli/enc/encode.cc +819 -286
- data/vendor/brotli/enc/encode.h +38 -15
- data/vendor/brotli/enc/encode_parallel.cc +40 -42
- data/vendor/brotli/enc/entropy_encode.cc +144 -147
- data/vendor/brotli/enc/entropy_encode.h +32 -8
- data/vendor/brotli/enc/entropy_encode_static.h +572 -0
- data/vendor/brotli/enc/fast_log.h +7 -40
- data/vendor/brotli/enc/find_match_length.h +9 -9
- data/vendor/brotli/enc/hash.h +462 -154
- data/vendor/brotli/enc/histogram.cc +6 -6
- data/vendor/brotli/enc/histogram.h +13 -13
- data/vendor/brotli/enc/literal_cost.cc +45 -45
- data/vendor/brotli/enc/metablock.cc +92 -89
- data/vendor/brotli/enc/metablock.h +12 -12
- data/vendor/brotli/enc/port.h +7 -16
- data/vendor/brotli/enc/prefix.h +23 -22
- data/vendor/brotli/enc/ringbuffer.h +75 -29
- data/vendor/brotli/enc/static_dict.cc +56 -48
- data/vendor/brotli/enc/static_dict.h +5 -5
- data/vendor/brotli/enc/streams.cc +1 -1
- data/vendor/brotli/enc/streams.h +5 -5
- data/vendor/brotli/enc/transform.h +40 -35
- data/vendor/brotli/enc/types.h +2 -0
- data/vendor/brotli/enc/utf8_util.cc +3 -2
- data/vendor/brotli/enc/write_bits.h +6 -6
- metadata +9 -5
- data/vendor/brotli/dec/streams.c +0 -102
- data/vendor/brotli/dec/streams.h +0 -95
@@ -23,7 +23,7 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
23
23
|
uint8_t prev_byte2,
|
24
24
|
const Command* cmds,
|
25
25
|
size_t num_commands,
|
26
|
-
|
26
|
+
ContextType literal_context_mode,
|
27
27
|
MetaBlockSplit* mb) {
|
28
28
|
SplitBlock(cmds, num_commands,
|
29
29
|
ringbuffer, pos, mask,
|
@@ -31,12 +31,12 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
31
31
|
&mb->command_split,
|
32
32
|
&mb->distance_split);
|
33
33
|
|
34
|
-
std::vector<
|
35
|
-
|
34
|
+
std::vector<ContextType> literal_context_modes(mb->literal_split.num_types,
|
35
|
+
literal_context_mode);
|
36
36
|
|
37
|
-
|
37
|
+
size_t num_literal_contexts =
|
38
38
|
mb->literal_split.num_types << kLiteralContextBits;
|
39
|
-
|
39
|
+
size_t num_distance_contexts =
|
40
40
|
mb->distance_split.num_types << kDistanceContextBits;
|
41
41
|
std::vector<HistogramLiteral> literal_histograms(num_literal_contexts);
|
42
42
|
mb->command_histograms.resize(mb->command_split.num_types);
|
@@ -58,17 +58,15 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
58
58
|
// Histogram ids need to fit in one byte.
|
59
59
|
static const size_t kMaxNumberOfHistograms = 256;
|
60
60
|
|
61
|
-
mb->literal_histograms = literal_histograms;
|
62
61
|
ClusterHistograms(literal_histograms,
|
63
|
-
|
62
|
+
1u << kLiteralContextBits,
|
64
63
|
mb->literal_split.num_types,
|
65
64
|
kMaxNumberOfHistograms,
|
66
65
|
&mb->literal_histograms,
|
67
66
|
&mb->literal_context_map);
|
68
67
|
|
69
|
-
mb->distance_histograms = distance_histograms;
|
70
68
|
ClusterHistograms(distance_histograms,
|
71
|
-
|
69
|
+
1u << kDistanceContextBits,
|
72
70
|
mb->distance_split.num_types,
|
73
71
|
kMaxNumberOfHistograms,
|
74
72
|
&mb->distance_histograms,
|
@@ -79,10 +77,10 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
79
77
|
template<typename HistogramType>
|
80
78
|
class BlockSplitter {
|
81
79
|
public:
|
82
|
-
BlockSplitter(
|
83
|
-
|
80
|
+
BlockSplitter(size_t alphabet_size,
|
81
|
+
size_t min_block_size,
|
84
82
|
double split_threshold,
|
85
|
-
|
83
|
+
size_t num_symbols,
|
86
84
|
BlockSplit* split,
|
87
85
|
std::vector<HistogramType>* histograms)
|
88
86
|
: alphabet_size_(alphabet_size),
|
@@ -95,10 +93,10 @@ class BlockSplitter {
|
|
95
93
|
block_size_(0),
|
96
94
|
curr_histogram_ix_(0),
|
97
95
|
merge_last_count_(0) {
|
98
|
-
|
96
|
+
size_t max_num_blocks = num_symbols / min_block_size + 1;
|
99
97
|
// We have to allocate one more histogram than the maximum number of block
|
100
98
|
// types for the current histogram when the meta-block is too big.
|
101
|
-
|
99
|
+
size_t max_num_types = std::min<size_t>(max_num_blocks, kMaxBlockTypes + 1);
|
102
100
|
split_->lengths.resize(max_num_blocks);
|
103
101
|
split_->types.resize(max_num_blocks);
|
104
102
|
histograms_->resize(max_num_types);
|
@@ -107,7 +105,7 @@ class BlockSplitter {
|
|
107
105
|
|
108
106
|
// Adds the next symbol to the current histogram. When the current histogram
|
109
107
|
// reaches the target size, decides on merging the block.
|
110
|
-
void AddSymbol(
|
108
|
+
void AddSymbol(size_t symbol) {
|
111
109
|
(*histograms_)[curr_histogram_ix_].Add(symbol);
|
112
110
|
++block_size_;
|
113
111
|
if (block_size_ == target_block_size_) {
|
@@ -125,7 +123,7 @@ class BlockSplitter {
|
|
125
123
|
}
|
126
124
|
if (num_blocks_ == 0) {
|
127
125
|
// Create first block.
|
128
|
-
split_->lengths[0] = block_size_;
|
126
|
+
split_->lengths[0] = static_cast<uint32_t>(block_size_);
|
129
127
|
split_->types[0] = 0;
|
130
128
|
last_entropy_[0] =
|
131
129
|
BitsEntropy(&(*histograms_)[0].data_[0], alphabet_size_);
|
@@ -140,8 +138,8 @@ class BlockSplitter {
|
|
140
138
|
HistogramType combined_histo[2];
|
141
139
|
double combined_entropy[2];
|
142
140
|
double diff[2];
|
143
|
-
for (
|
144
|
-
|
141
|
+
for (size_t j = 0; j < 2; ++j) {
|
142
|
+
size_t last_histogram_ix = last_histogram_ix_[j];
|
145
143
|
combined_histo[j] = (*histograms_)[curr_histogram_ix_];
|
146
144
|
combined_histo[j].AddHistogram((*histograms_)[last_histogram_ix]);
|
147
145
|
combined_entropy[j] = BitsEntropy(
|
@@ -153,10 +151,10 @@ class BlockSplitter {
|
|
153
151
|
diff[0] > split_threshold_ &&
|
154
152
|
diff[1] > split_threshold_) {
|
155
153
|
// Create new block.
|
156
|
-
split_->lengths[num_blocks_] = block_size_;
|
157
|
-
split_->types[num_blocks_] = split_->num_types;
|
154
|
+
split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
|
155
|
+
split_->types[num_blocks_] = static_cast<uint8_t>(split_->num_types);
|
158
156
|
last_histogram_ix_[1] = last_histogram_ix_[0];
|
159
|
-
last_histogram_ix_[0] = split_->num_types;
|
157
|
+
last_histogram_ix_[0] = static_cast<uint8_t>(split_->num_types);
|
160
158
|
last_entropy_[1] = last_entropy_[0];
|
161
159
|
last_entropy_[0] = entropy;
|
162
160
|
++num_blocks_;
|
@@ -167,7 +165,7 @@ class BlockSplitter {
|
|
167
165
|
target_block_size_ = min_block_size_;
|
168
166
|
} else if (diff[1] < diff[0] - 20.0) {
|
169
167
|
// Combine this block with second last block.
|
170
|
-
split_->lengths[num_blocks_] = block_size_;
|
168
|
+
split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
|
171
169
|
split_->types[num_blocks_] = split_->types[num_blocks_ - 2];
|
172
170
|
std::swap(last_histogram_ix_[0], last_histogram_ix_[1]);
|
173
171
|
(*histograms_)[last_histogram_ix_[0]] = combined_histo[1];
|
@@ -180,7 +178,7 @@ class BlockSplitter {
|
|
180
178
|
target_block_size_ = min_block_size_;
|
181
179
|
} else {
|
182
180
|
// Combine this block with last block.
|
183
|
-
split_->lengths[num_blocks_ - 1] += block_size_;
|
181
|
+
split_->lengths[num_blocks_ - 1] += static_cast<uint32_t>(block_size_);
|
184
182
|
(*histograms_)[last_histogram_ix_[0]] = combined_histo[0];
|
185
183
|
last_entropy_[0] = combined_entropy[0];
|
186
184
|
if (split_->num_types == 1) {
|
@@ -201,35 +199,35 @@ class BlockSplitter {
|
|
201
199
|
}
|
202
200
|
|
203
201
|
private:
|
204
|
-
static const
|
202
|
+
static const uint16_t kMaxBlockTypes = 256;
|
205
203
|
|
206
204
|
// Alphabet size of particular block category.
|
207
|
-
const
|
205
|
+
const size_t alphabet_size_;
|
208
206
|
// We collect at least this many symbols for each block.
|
209
|
-
const
|
207
|
+
const size_t min_block_size_;
|
210
208
|
// We merge histograms A and B if
|
211
209
|
// entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,
|
212
210
|
// where A is the current histogram and B is the histogram of the last or the
|
213
211
|
// second last block type.
|
214
212
|
const double split_threshold_;
|
215
213
|
|
216
|
-
|
214
|
+
size_t num_blocks_;
|
217
215
|
BlockSplit* split_; // not owned
|
218
216
|
std::vector<HistogramType>* histograms_; // not owned
|
219
217
|
|
220
218
|
// The number of symbols that we want to collect before deciding on whether
|
221
219
|
// or not to merge the block with a previous one or emit a new block.
|
222
|
-
|
220
|
+
size_t target_block_size_;
|
223
221
|
// The number of symbols in the current histogram.
|
224
|
-
|
222
|
+
size_t block_size_;
|
225
223
|
// Offset of the current histogram.
|
226
|
-
|
224
|
+
size_t curr_histogram_ix_;
|
227
225
|
// Offset of the histograms of the previous two block types.
|
228
|
-
|
226
|
+
size_t last_histogram_ix_[2];
|
229
227
|
// Entropy of the previous two block types.
|
230
228
|
double last_entropy_[2];
|
231
229
|
// The number of times we merged the current block with the last one.
|
232
|
-
|
230
|
+
size_t merge_last_count_;
|
233
231
|
};
|
234
232
|
|
235
233
|
void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
|
@@ -238,7 +236,7 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
|
|
238
236
|
const Command *commands,
|
239
237
|
size_t n_commands,
|
240
238
|
MetaBlockSplit* mb) {
|
241
|
-
|
239
|
+
size_t num_literals = 0;
|
242
240
|
for (size_t i = 0; i < n_commands; ++i) {
|
243
241
|
num_literals += commands[i].insert_len_;
|
244
242
|
}
|
@@ -247,21 +245,21 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
|
|
247
245
|
256, 512, 400.0, num_literals,
|
248
246
|
&mb->literal_split, &mb->literal_histograms);
|
249
247
|
BlockSplitter<HistogramCommand> cmd_blocks(
|
250
|
-
kNumCommandPrefixes, 1024, 500.0,
|
248
|
+
kNumCommandPrefixes, 1024, 500.0, n_commands,
|
251
249
|
&mb->command_split, &mb->command_histograms);
|
252
250
|
BlockSplitter<HistogramDistance> dist_blocks(
|
253
|
-
64, 512, 100.0,
|
251
|
+
64, 512, 100.0, n_commands,
|
254
252
|
&mb->distance_split, &mb->distance_histograms);
|
255
253
|
|
256
254
|
for (size_t i = 0; i < n_commands; ++i) {
|
257
255
|
const Command cmd = commands[i];
|
258
256
|
cmd_blocks.AddSymbol(cmd.cmd_prefix_);
|
259
|
-
for (
|
257
|
+
for (size_t j = cmd.insert_len_; j != 0; --j) {
|
260
258
|
lit_blocks.AddSymbol(ringbuffer[pos & mask]);
|
261
259
|
++pos;
|
262
260
|
}
|
263
|
-
pos += cmd.
|
264
|
-
if (cmd.
|
261
|
+
pos += cmd.copy_len();
|
262
|
+
if (cmd.copy_len() && cmd.cmd_prefix_ >= 128) {
|
265
263
|
dist_blocks.AddSymbol(cmd.dist_prefix_);
|
266
264
|
}
|
267
265
|
}
|
@@ -276,11 +274,11 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
|
|
276
274
|
template<typename HistogramType>
|
277
275
|
class ContextBlockSplitter {
|
278
276
|
public:
|
279
|
-
ContextBlockSplitter(
|
280
|
-
|
281
|
-
|
277
|
+
ContextBlockSplitter(size_t alphabet_size,
|
278
|
+
size_t num_contexts,
|
279
|
+
size_t min_block_size,
|
282
280
|
double split_threshold,
|
283
|
-
|
281
|
+
size_t num_symbols,
|
284
282
|
BlockSplit* split,
|
285
283
|
std::vector<HistogramType>* histograms)
|
286
284
|
: alphabet_size_(alphabet_size),
|
@@ -296,10 +294,10 @@ class ContextBlockSplitter {
|
|
296
294
|
curr_histogram_ix_(0),
|
297
295
|
last_entropy_(2 * num_contexts),
|
298
296
|
merge_last_count_(0) {
|
299
|
-
|
297
|
+
size_t max_num_blocks = num_symbols / min_block_size + 1;
|
300
298
|
// We have to allocate one more histogram than the maximum number of block
|
301
299
|
// types for the current histogram when the meta-block is too big.
|
302
|
-
|
300
|
+
size_t max_num_types = std::min(max_num_blocks, max_block_types_ + 1);
|
303
301
|
split_->lengths.resize(max_num_blocks);
|
304
302
|
split_->types.resize(max_num_blocks);
|
305
303
|
histograms_->resize(max_num_types * num_contexts);
|
@@ -308,7 +306,7 @@ class ContextBlockSplitter {
|
|
308
306
|
|
309
307
|
// Adds the next symbol to the current block type and context. When the
|
310
308
|
// current block reaches the target size, decides on merging the block.
|
311
|
-
void AddSymbol(
|
309
|
+
void AddSymbol(size_t symbol, size_t context) {
|
312
310
|
(*histograms_)[curr_histogram_ix_ + context].Add(symbol);
|
313
311
|
++block_size_;
|
314
312
|
if (block_size_ == target_block_size_) {
|
@@ -326,9 +324,9 @@ class ContextBlockSplitter {
|
|
326
324
|
}
|
327
325
|
if (num_blocks_ == 0) {
|
328
326
|
// Create first block.
|
329
|
-
split_->lengths[0] = block_size_;
|
327
|
+
split_->lengths[0] = static_cast<uint32_t>(block_size_);
|
330
328
|
split_->types[0] = 0;
|
331
|
-
for (
|
329
|
+
for (size_t i = 0; i < num_contexts_; ++i) {
|
332
330
|
last_entropy_[i] =
|
333
331
|
BitsEntropy(&(*histograms_)[i].data_[0], alphabet_size_);
|
334
332
|
last_entropy_[num_contexts_ + i] = last_entropy_[i];
|
@@ -346,13 +344,13 @@ class ContextBlockSplitter {
|
|
346
344
|
std::vector<HistogramType> combined_histo(2 * num_contexts_);
|
347
345
|
std::vector<double> combined_entropy(2 * num_contexts_);
|
348
346
|
double diff[2] = { 0.0 };
|
349
|
-
for (
|
350
|
-
|
347
|
+
for (size_t i = 0; i < num_contexts_; ++i) {
|
348
|
+
size_t curr_histo_ix = curr_histogram_ix_ + i;
|
351
349
|
entropy[i] = BitsEntropy(&(*histograms_)[curr_histo_ix].data_[0],
|
352
350
|
alphabet_size_);
|
353
|
-
for (
|
354
|
-
|
355
|
-
|
351
|
+
for (size_t j = 0; j < 2; ++j) {
|
352
|
+
size_t jx = j * num_contexts_ + i;
|
353
|
+
size_t last_histogram_ix = last_histogram_ix_[j] + i;
|
356
354
|
combined_histo[jx] = (*histograms_)[curr_histo_ix];
|
357
355
|
combined_histo[jx].AddHistogram((*histograms_)[last_histogram_ix]);
|
358
356
|
combined_entropy[jx] = BitsEntropy(
|
@@ -365,11 +363,11 @@ class ContextBlockSplitter {
|
|
365
363
|
diff[0] > split_threshold_ &&
|
366
364
|
diff[1] > split_threshold_) {
|
367
365
|
// Create new block.
|
368
|
-
split_->lengths[num_blocks_] = block_size_;
|
369
|
-
split_->types[num_blocks_] = split_->num_types;
|
366
|
+
split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
|
367
|
+
split_->types[num_blocks_] = static_cast<uint8_t>(split_->num_types);
|
370
368
|
last_histogram_ix_[1] = last_histogram_ix_[0];
|
371
369
|
last_histogram_ix_[0] = split_->num_types * num_contexts_;
|
372
|
-
for (
|
370
|
+
for (size_t i = 0; i < num_contexts_; ++i) {
|
373
371
|
last_entropy_[num_contexts_ + i] = last_entropy_[i];
|
374
372
|
last_entropy_[i] = entropy[i];
|
375
373
|
}
|
@@ -381,10 +379,10 @@ class ContextBlockSplitter {
|
|
381
379
|
target_block_size_ = min_block_size_;
|
382
380
|
} else if (diff[1] < diff[0] - 20.0) {
|
383
381
|
// Combine this block with second last block.
|
384
|
-
split_->lengths[num_blocks_] = block_size_;
|
382
|
+
split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
|
385
383
|
split_->types[num_blocks_] = split_->types[num_blocks_ - 2];
|
386
384
|
std::swap(last_histogram_ix_[0], last_histogram_ix_[1]);
|
387
|
-
for (
|
385
|
+
for (size_t i = 0; i < num_contexts_; ++i) {
|
388
386
|
(*histograms_)[last_histogram_ix_[0] + i] =
|
389
387
|
combined_histo[num_contexts_ + i];
|
390
388
|
last_entropy_[num_contexts_ + i] = last_entropy_[i];
|
@@ -397,8 +395,8 @@ class ContextBlockSplitter {
|
|
397
395
|
target_block_size_ = min_block_size_;
|
398
396
|
} else {
|
399
397
|
// Combine this block with last block.
|
400
|
-
split_->lengths[num_blocks_ - 1] += block_size_;
|
401
|
-
for (
|
398
|
+
split_->lengths[num_blocks_ - 1] += static_cast<uint32_t>(block_size_);
|
399
|
+
for (size_t i = 0; i < num_contexts_; ++i) {
|
402
400
|
(*histograms_)[last_histogram_ix_[0] + i] = combined_histo[i];
|
403
401
|
last_entropy_[i] = combined_entropy[i];
|
404
402
|
if (split_->num_types == 1) {
|
@@ -423,34 +421,34 @@ class ContextBlockSplitter {
|
|
423
421
|
static const int kMaxBlockTypes = 256;
|
424
422
|
|
425
423
|
// Alphabet size of particular block category.
|
426
|
-
const
|
427
|
-
const
|
428
|
-
const
|
424
|
+
const size_t alphabet_size_;
|
425
|
+
const size_t num_contexts_;
|
426
|
+
const size_t max_block_types_;
|
429
427
|
// We collect at least this many symbols for each block.
|
430
|
-
const
|
428
|
+
const size_t min_block_size_;
|
431
429
|
// We merge histograms A and B if
|
432
430
|
// entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,
|
433
431
|
// where A is the current histogram and B is the histogram of the last or the
|
434
432
|
// second last block type.
|
435
433
|
const double split_threshold_;
|
436
434
|
|
437
|
-
|
435
|
+
size_t num_blocks_;
|
438
436
|
BlockSplit* split_; // not owned
|
439
437
|
std::vector<HistogramType>* histograms_; // not owned
|
440
438
|
|
441
439
|
// The number of symbols that we want to collect before deciding on whether
|
442
440
|
// or not to merge the block with a previous one or emit a new block.
|
443
|
-
|
441
|
+
size_t target_block_size_;
|
444
442
|
// The number of symbols in the current histogram.
|
445
|
-
|
443
|
+
size_t block_size_;
|
446
444
|
// Offset of the current histogram.
|
447
|
-
|
445
|
+
size_t curr_histogram_ix_;
|
448
446
|
// Offset of the histograms of the previous two block types.
|
449
|
-
|
447
|
+
size_t last_histogram_ix_[2];
|
450
448
|
// Entropy of the previous two block types.
|
451
449
|
std::vector<double> last_entropy_;
|
452
450
|
// The number of times we merged the current block with the last one.
|
453
|
-
|
451
|
+
size_t merge_last_count_;
|
454
452
|
};
|
455
453
|
|
456
454
|
void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
@@ -458,13 +456,13 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
458
456
|
size_t mask,
|
459
457
|
uint8_t prev_byte,
|
460
458
|
uint8_t prev_byte2,
|
461
|
-
|
462
|
-
|
463
|
-
const
|
459
|
+
ContextType literal_context_mode,
|
460
|
+
size_t num_contexts,
|
461
|
+
const uint32_t* static_context_map,
|
464
462
|
const Command *commands,
|
465
463
|
size_t n_commands,
|
466
464
|
MetaBlockSplit* mb) {
|
467
|
-
|
465
|
+
size_t num_literals = 0;
|
468
466
|
for (size_t i = 0; i < n_commands; ++i) {
|
469
467
|
num_literals += commands[i].insert_len_;
|
470
468
|
}
|
@@ -473,25 +471,25 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
473
471
|
256, num_contexts, 512, 400.0, num_literals,
|
474
472
|
&mb->literal_split, &mb->literal_histograms);
|
475
473
|
BlockSplitter<HistogramCommand> cmd_blocks(
|
476
|
-
kNumCommandPrefixes, 1024, 500.0,
|
474
|
+
kNumCommandPrefixes, 1024, 500.0, n_commands,
|
477
475
|
&mb->command_split, &mb->command_histograms);
|
478
476
|
BlockSplitter<HistogramDistance> dist_blocks(
|
479
|
-
64, 512, 100.0,
|
477
|
+
64, 512, 100.0, n_commands,
|
480
478
|
&mb->distance_split, &mb->distance_histograms);
|
481
479
|
|
482
480
|
for (size_t i = 0; i < n_commands; ++i) {
|
483
481
|
const Command cmd = commands[i];
|
484
482
|
cmd_blocks.AddSymbol(cmd.cmd_prefix_);
|
485
|
-
for (
|
486
|
-
|
483
|
+
for (size_t j = cmd.insert_len_; j != 0; --j) {
|
484
|
+
size_t context = Context(prev_byte, prev_byte2, literal_context_mode);
|
487
485
|
uint8_t literal = ringbuffer[pos & mask];
|
488
486
|
lit_blocks.AddSymbol(literal, static_context_map[context]);
|
489
487
|
prev_byte2 = prev_byte;
|
490
488
|
prev_byte = literal;
|
491
489
|
++pos;
|
492
490
|
}
|
493
|
-
pos += cmd.
|
494
|
-
if (cmd.
|
491
|
+
pos += cmd.copy_len();
|
492
|
+
if (cmd.copy_len()) {
|
495
493
|
prev_byte2 = ringbuffer[(pos - 2) & mask];
|
496
494
|
prev_byte = ringbuffer[(pos - 1) & mask];
|
497
495
|
if (cmd.cmd_prefix_ >= 128) {
|
@@ -506,31 +504,36 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
506
504
|
|
507
505
|
mb->literal_context_map.resize(
|
508
506
|
mb->literal_split.num_types << kLiteralContextBits);
|
509
|
-
for (
|
510
|
-
for (
|
507
|
+
for (size_t i = 0; i < mb->literal_split.num_types; ++i) {
|
508
|
+
for (size_t j = 0; j < (1u << kLiteralContextBits); ++j) {
|
511
509
|
mb->literal_context_map[(i << kLiteralContextBits) + j] =
|
512
|
-
i * num_contexts + static_context_map[j];
|
510
|
+
static_cast<uint32_t>(i * num_contexts) + static_context_map[j];
|
513
511
|
}
|
514
512
|
}
|
515
513
|
}
|
516
514
|
|
517
|
-
void OptimizeHistograms(
|
518
|
-
|
515
|
+
void OptimizeHistograms(size_t num_direct_distance_codes,
|
516
|
+
size_t distance_postfix_bits,
|
519
517
|
MetaBlockSplit* mb) {
|
518
|
+
uint8_t* good_for_rle = new uint8_t[kNumCommandPrefixes];
|
520
519
|
for (size_t i = 0; i < mb->literal_histograms.size(); ++i) {
|
521
|
-
OptimizeHuffmanCountsForRle(256, &mb->literal_histograms[i].data_[0]
|
520
|
+
OptimizeHuffmanCountsForRle(256, &mb->literal_histograms[i].data_[0],
|
521
|
+
good_for_rle);
|
522
522
|
}
|
523
523
|
for (size_t i = 0; i < mb->command_histograms.size(); ++i) {
|
524
524
|
OptimizeHuffmanCountsForRle(kNumCommandPrefixes,
|
525
|
-
&mb->command_histograms[i].data_[0]
|
525
|
+
&mb->command_histograms[i].data_[0],
|
526
|
+
good_for_rle);
|
526
527
|
}
|
527
|
-
|
528
|
+
size_t num_distance_codes =
|
528
529
|
kNumDistanceShortCodes + num_direct_distance_codes +
|
529
|
-
(
|
530
|
+
(48u << distance_postfix_bits);
|
530
531
|
for (size_t i = 0; i < mb->distance_histograms.size(); ++i) {
|
531
532
|
OptimizeHuffmanCountsForRle(num_distance_codes,
|
532
|
-
&mb->distance_histograms[i].data_[0]
|
533
|
+
&mb->distance_histograms[i].data_[0],
|
534
|
+
good_for_rle);
|
533
535
|
}
|
536
|
+
delete[] good_for_rle;
|
534
537
|
}
|
535
538
|
|
536
539
|
} // namespace brotli
|
@@ -18,19 +18,19 @@
|
|
18
18
|
namespace brotli {
|
19
19
|
|
20
20
|
struct BlockSplit {
|
21
|
-
BlockSplit() : num_types(0) {}
|
21
|
+
BlockSplit(void) : num_types(0) {}
|
22
22
|
|
23
|
-
|
24
|
-
std::vector<
|
25
|
-
std::vector<
|
23
|
+
size_t num_types;
|
24
|
+
std::vector<uint8_t> types;
|
25
|
+
std::vector<uint32_t> lengths;
|
26
26
|
};
|
27
27
|
|
28
28
|
struct MetaBlockSplit {
|
29
29
|
BlockSplit literal_split;
|
30
30
|
BlockSplit command_split;
|
31
31
|
BlockSplit distance_split;
|
32
|
-
std::vector<
|
33
|
-
std::vector<
|
32
|
+
std::vector<uint32_t> literal_context_map;
|
33
|
+
std::vector<uint32_t> distance_context_map;
|
34
34
|
std::vector<HistogramLiteral> literal_histograms;
|
35
35
|
std::vector<HistogramCommand> command_histograms;
|
36
36
|
std::vector<HistogramDistance> distance_histograms;
|
@@ -44,7 +44,7 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
44
44
|
uint8_t prev_byte2,
|
45
45
|
const Command* cmds,
|
46
46
|
size_t num_commands,
|
47
|
-
|
47
|
+
ContextType literal_context_mode,
|
48
48
|
MetaBlockSplit* mb);
|
49
49
|
|
50
50
|
// Uses a fast greedy block splitter that tries to merge current block with the
|
@@ -64,15 +64,15 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
64
64
|
size_t mask,
|
65
65
|
uint8_t prev_byte,
|
66
66
|
uint8_t prev_byte2,
|
67
|
-
|
68
|
-
|
69
|
-
const
|
67
|
+
ContextType literal_context_mode,
|
68
|
+
size_t num_contexts,
|
69
|
+
const uint32_t* static_context_map,
|
70
70
|
const Command *commands,
|
71
71
|
size_t n_commands,
|
72
72
|
MetaBlockSplit* mb);
|
73
73
|
|
74
|
-
void OptimizeHistograms(
|
75
|
-
|
74
|
+
void OptimizeHistograms(size_t num_direct_distance_codes,
|
75
|
+
size_t distance_postfix_bits,
|
76
76
|
MetaBlockSplit* mb);
|
77
77
|
|
78
78
|
} // namespace brotli
|
data/vendor/brotli/enc/port.h
CHANGED
@@ -22,10 +22,9 @@
|
|
22
22
|
/* Let's try and follow the Linux convention */
|
23
23
|
#define __BYTE_ORDER BYTE_ORDER
|
24
24
|
#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
25
|
-
#define __BIG_ENDIAN BIG_ENDIAN
|
26
25
|
#endif
|
27
26
|
|
28
|
-
// define the
|
27
|
+
// define the macro IS_LITTLE_ENDIAN
|
29
28
|
// using the above endian definitions from endian.h if
|
30
29
|
// endian.h was included
|
31
30
|
#ifdef __BYTE_ORDER
|
@@ -33,19 +32,17 @@
|
|
33
32
|
#define IS_LITTLE_ENDIAN
|
34
33
|
#endif
|
35
34
|
|
36
|
-
#if __BYTE_ORDER == __BIG_ENDIAN
|
37
|
-
#define IS_BIG_ENDIAN
|
38
|
-
#endif
|
39
|
-
|
40
35
|
#else
|
41
36
|
|
42
37
|
#if defined(__LITTLE_ENDIAN__)
|
43
38
|
#define IS_LITTLE_ENDIAN
|
44
|
-
#elif defined(__BIG_ENDIAN__)
|
45
|
-
#define IS_BIG_ENDIAN
|
46
39
|
#endif
|
47
40
|
#endif // __BYTE_ORDER
|
48
41
|
|
42
|
+
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
43
|
+
#define IS_LITTLE_ENDIAN
|
44
|
+
#endif
|
45
|
+
|
49
46
|
// Enable little-endian optimization for x64 architecture on Windows.
|
50
47
|
#if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
|
51
48
|
#define IS_LITTLE_ENDIAN
|
@@ -69,8 +66,8 @@
|
|
69
66
|
// On some platforms, like ARM, the copy functions can be more efficient
|
70
67
|
// then a load and a store.
|
71
68
|
|
72
|
-
#if defined(ARCH_PIII) ||
|
73
|
-
defined(ARCH_K8) || defined(_ARCH_PPC)
|
69
|
+
#if defined(ARCH_PIII) || \
|
70
|
+
defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC)
|
74
71
|
|
75
72
|
// x86 and x86-64 can perform unaligned loads/stores directly;
|
76
73
|
// modern PowerPC hardware can also do unaligned integer loads and stores;
|
@@ -142,10 +139,4 @@ inline void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
|
|
142
139
|
|
143
140
|
#endif
|
144
141
|
|
145
|
-
#ifdef BROTLI_ENCODE_DEBUG
|
146
|
-
#define BROTLI_DCHECK(x) assert(x)
|
147
|
-
#else
|
148
|
-
#define BROTLI_DCHECK(x)
|
149
|
-
#endif
|
150
|
-
|
151
142
|
#endif // BROTLI_ENC_PORT_H_
|
data/vendor/brotli/enc/prefix.h
CHANGED
@@ -15,18 +15,18 @@
|
|
15
15
|
|
16
16
|
namespace brotli {
|
17
17
|
|
18
|
-
static const
|
19
|
-
static const
|
20
|
-
static const
|
21
|
-
static const
|
22
|
-
static const
|
23
|
-
static const
|
18
|
+
static const uint32_t kNumInsertLenPrefixes = 24;
|
19
|
+
static const uint32_t kNumCopyLenPrefixes = 24;
|
20
|
+
static const uint32_t kNumCommandPrefixes = 704;
|
21
|
+
static const uint32_t kNumBlockLenPrefixes = 26;
|
22
|
+
static const uint32_t kNumDistanceShortCodes = 16;
|
23
|
+
static const uint32_t kNumDistancePrefixes = 520;
|
24
24
|
|
25
25
|
// Represents the range of values belonging to a prefix code:
|
26
26
|
// [offset, offset + 2^nbits)
|
27
27
|
struct PrefixCodeRange {
|
28
|
-
|
29
|
-
|
28
|
+
uint32_t offset;
|
29
|
+
uint32_t nbits;
|
30
30
|
};
|
31
31
|
|
32
32
|
static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
|
@@ -39,8 +39,8 @@ static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
|
|
39
39
|
{8433, 13}, {16625, 24}
|
40
40
|
};
|
41
41
|
|
42
|
-
inline void GetBlockLengthPrefixCode(
|
43
|
-
|
42
|
+
inline void GetBlockLengthPrefixCode(uint32_t len, uint32_t* code,
|
43
|
+
uint32_t* n_extra, uint32_t* extra) {
|
44
44
|
*code = 0;
|
45
45
|
while (*code < 25 && len >= kBlockLengthPrefixCode[*code + 1].offset) {
|
46
46
|
++(*code);
|
@@ -49,9 +49,9 @@ inline void GetBlockLengthPrefixCode(int len,
|
|
49
49
|
*extra = len - kBlockLengthPrefixCode[*code].offset;
|
50
50
|
}
|
51
51
|
|
52
|
-
inline void PrefixEncodeCopyDistance(
|
53
|
-
|
54
|
-
|
52
|
+
inline void PrefixEncodeCopyDistance(size_t distance_code,
|
53
|
+
size_t num_direct_codes,
|
54
|
+
size_t postfix_bits,
|
55
55
|
uint16_t* code,
|
56
56
|
uint32_t* extra_bits) {
|
57
57
|
if (distance_code < kNumDistanceShortCodes + num_direct_codes) {
|
@@ -59,18 +59,19 @@ inline void PrefixEncodeCopyDistance(int distance_code,
|
|
59
59
|
*extra_bits = 0;
|
60
60
|
return;
|
61
61
|
}
|
62
|
-
distance_code -= kNumDistanceShortCodes + num_direct_codes;
|
63
|
-
distance_code += (
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
62
|
+
distance_code -= kNumDistanceShortCodes + num_direct_codes; /* >= 0 */
|
63
|
+
distance_code += (1u << (postfix_bits + 2u)); /* > 0 */
|
64
|
+
size_t bucket = Log2FloorNonZero(distance_code) - 1;
|
65
|
+
size_t postfix_mask = (1 << postfix_bits) - 1;
|
66
|
+
size_t postfix = distance_code & postfix_mask;
|
67
|
+
size_t prefix = (distance_code >> bucket) & 1;
|
68
|
+
size_t offset = (2 + prefix) << bucket;
|
69
|
+
size_t nbits = bucket - postfix_bits;
|
70
70
|
*code = static_cast<uint16_t>(
|
71
71
|
(kNumDistanceShortCodes + num_direct_codes +
|
72
72
|
((2 * (nbits - 1) + prefix) << postfix_bits) + postfix));
|
73
|
-
*extra_bits = (
|
73
|
+
*extra_bits = static_cast<uint32_t>(
|
74
|
+
(nbits << 24) | ((distance_code - offset) >> postfix_bits));
|
74
75
|
}
|
75
76
|
|
76
77
|
} // namespace brotli
|