brotli 0.1.1 → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/ext/brotli/brotli.cc +114 -24
  3. data/ext/brotli/brotli.h +0 -1
  4. data/ext/brotli/extconf.rb +30 -23
  5. data/lib/brotli/version.rb +1 -1
  6. data/vendor/brotli/LICENSE +1 -1
  7. data/vendor/brotli/dec/Makefile +1 -1
  8. data/vendor/brotli/dec/bit_reader.c +3 -3
  9. data/vendor/brotli/dec/bit_reader.h +25 -27
  10. data/vendor/brotli/dec/context.h +4 -4
  11. data/vendor/brotli/dec/decode.c +410 -486
  12. data/vendor/brotli/dec/decode.h +101 -105
  13. data/vendor/brotli/dec/dictionary.c +1 -1
  14. data/vendor/brotli/dec/dictionary.h +7 -8
  15. data/vendor/brotli/dec/huffman.c +103 -105
  16. data/vendor/brotli/dec/huffman.h +18 -18
  17. data/vendor/brotli/dec/port.h +52 -40
  18. data/vendor/brotli/dec/prefix.h +2 -0
  19. data/vendor/brotli/dec/state.c +13 -19
  20. data/vendor/brotli/dec/state.h +25 -39
  21. data/vendor/brotli/dec/transform.h +38 -44
  22. data/vendor/brotli/dec/types.h +2 -2
  23. data/vendor/brotli/enc/Makefile +1 -1
  24. data/vendor/brotli/enc/backward_references.cc +455 -359
  25. data/vendor/brotli/enc/backward_references.h +79 -3
  26. data/vendor/brotli/enc/bit_cost.h +54 -32
  27. data/vendor/brotli/enc/block_splitter.cc +285 -193
  28. data/vendor/brotli/enc/block_splitter.h +4 -12
  29. data/vendor/brotli/enc/brotli_bit_stream.cc +623 -324
  30. data/vendor/brotli/enc/brotli_bit_stream.h +76 -37
  31. data/vendor/brotli/enc/cluster.h +161 -120
  32. data/vendor/brotli/enc/command.h +60 -37
  33. data/vendor/brotli/enc/compress_fragment.cc +701 -0
  34. data/vendor/brotli/enc/compress_fragment.h +47 -0
  35. data/vendor/brotli/enc/compress_fragment_two_pass.cc +524 -0
  36. data/vendor/brotli/enc/compress_fragment_two_pass.h +40 -0
  37. data/vendor/brotli/enc/compressor.h +15 -0
  38. data/vendor/brotli/enc/context.h +1 -1
  39. data/vendor/brotli/enc/dictionary.h +2 -2
  40. data/vendor/brotli/enc/encode.cc +819 -286
  41. data/vendor/brotli/enc/encode.h +38 -15
  42. data/vendor/brotli/enc/encode_parallel.cc +40 -42
  43. data/vendor/brotli/enc/entropy_encode.cc +144 -147
  44. data/vendor/brotli/enc/entropy_encode.h +32 -8
  45. data/vendor/brotli/enc/entropy_encode_static.h +572 -0
  46. data/vendor/brotli/enc/fast_log.h +7 -40
  47. data/vendor/brotli/enc/find_match_length.h +9 -9
  48. data/vendor/brotli/enc/hash.h +462 -154
  49. data/vendor/brotli/enc/histogram.cc +6 -6
  50. data/vendor/brotli/enc/histogram.h +13 -13
  51. data/vendor/brotli/enc/literal_cost.cc +45 -45
  52. data/vendor/brotli/enc/metablock.cc +92 -89
  53. data/vendor/brotli/enc/metablock.h +12 -12
  54. data/vendor/brotli/enc/port.h +7 -16
  55. data/vendor/brotli/enc/prefix.h +23 -22
  56. data/vendor/brotli/enc/ringbuffer.h +75 -29
  57. data/vendor/brotli/enc/static_dict.cc +56 -48
  58. data/vendor/brotli/enc/static_dict.h +5 -5
  59. data/vendor/brotli/enc/streams.cc +1 -1
  60. data/vendor/brotli/enc/streams.h +5 -5
  61. data/vendor/brotli/enc/transform.h +40 -35
  62. data/vendor/brotli/enc/types.h +2 -0
  63. data/vendor/brotli/enc/utf8_util.cc +3 -2
  64. data/vendor/brotli/enc/write_bits.h +6 -6
  65. metadata +9 -5
  66. data/vendor/brotli/dec/streams.c +0 -102
  67. data/vendor/brotli/dec/streams.h +0 -95
@@ -23,7 +23,7 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
23
23
  uint8_t prev_byte2,
24
24
  const Command* cmds,
25
25
  size_t num_commands,
26
- int literal_context_mode,
26
+ ContextType literal_context_mode,
27
27
  MetaBlockSplit* mb) {
28
28
  SplitBlock(cmds, num_commands,
29
29
  ringbuffer, pos, mask,
@@ -31,12 +31,12 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
31
31
  &mb->command_split,
32
32
  &mb->distance_split);
33
33
 
34
- std::vector<int> literal_context_modes(mb->literal_split.num_types,
35
- literal_context_mode);
34
+ std::vector<ContextType> literal_context_modes(mb->literal_split.num_types,
35
+ literal_context_mode);
36
36
 
37
- int num_literal_contexts =
37
+ size_t num_literal_contexts =
38
38
  mb->literal_split.num_types << kLiteralContextBits;
39
- int num_distance_contexts =
39
+ size_t num_distance_contexts =
40
40
  mb->distance_split.num_types << kDistanceContextBits;
41
41
  std::vector<HistogramLiteral> literal_histograms(num_literal_contexts);
42
42
  mb->command_histograms.resize(mb->command_split.num_types);
@@ -58,17 +58,15 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
58
58
  // Histogram ids need to fit in one byte.
59
59
  static const size_t kMaxNumberOfHistograms = 256;
60
60
 
61
- mb->literal_histograms = literal_histograms;
62
61
  ClusterHistograms(literal_histograms,
63
- 1 << kLiteralContextBits,
62
+ 1u << kLiteralContextBits,
64
63
  mb->literal_split.num_types,
65
64
  kMaxNumberOfHistograms,
66
65
  &mb->literal_histograms,
67
66
  &mb->literal_context_map);
68
67
 
69
- mb->distance_histograms = distance_histograms;
70
68
  ClusterHistograms(distance_histograms,
71
- 1 << kDistanceContextBits,
69
+ 1u << kDistanceContextBits,
72
70
  mb->distance_split.num_types,
73
71
  kMaxNumberOfHistograms,
74
72
  &mb->distance_histograms,
@@ -79,10 +77,10 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
79
77
  template<typename HistogramType>
80
78
  class BlockSplitter {
81
79
  public:
82
- BlockSplitter(int alphabet_size,
83
- int min_block_size,
80
+ BlockSplitter(size_t alphabet_size,
81
+ size_t min_block_size,
84
82
  double split_threshold,
85
- int num_symbols,
83
+ size_t num_symbols,
86
84
  BlockSplit* split,
87
85
  std::vector<HistogramType>* histograms)
88
86
  : alphabet_size_(alphabet_size),
@@ -95,10 +93,10 @@ class BlockSplitter {
95
93
  block_size_(0),
96
94
  curr_histogram_ix_(0),
97
95
  merge_last_count_(0) {
98
- int max_num_blocks = num_symbols / min_block_size + 1;
96
+ size_t max_num_blocks = num_symbols / min_block_size + 1;
99
97
  // We have to allocate one more histogram than the maximum number of block
100
98
  // types for the current histogram when the meta-block is too big.
101
- int max_num_types = std::min(max_num_blocks, kMaxBlockTypes + 1);
99
+ size_t max_num_types = std::min<size_t>(max_num_blocks, kMaxBlockTypes + 1);
102
100
  split_->lengths.resize(max_num_blocks);
103
101
  split_->types.resize(max_num_blocks);
104
102
  histograms_->resize(max_num_types);
@@ -107,7 +105,7 @@ class BlockSplitter {
107
105
 
108
106
  // Adds the next symbol to the current histogram. When the current histogram
109
107
  // reaches the target size, decides on merging the block.
110
- void AddSymbol(int symbol) {
108
+ void AddSymbol(size_t symbol) {
111
109
  (*histograms_)[curr_histogram_ix_].Add(symbol);
112
110
  ++block_size_;
113
111
  if (block_size_ == target_block_size_) {
@@ -125,7 +123,7 @@ class BlockSplitter {
125
123
  }
126
124
  if (num_blocks_ == 0) {
127
125
  // Create first block.
128
- split_->lengths[0] = block_size_;
126
+ split_->lengths[0] = static_cast<uint32_t>(block_size_);
129
127
  split_->types[0] = 0;
130
128
  last_entropy_[0] =
131
129
  BitsEntropy(&(*histograms_)[0].data_[0], alphabet_size_);
@@ -140,8 +138,8 @@ class BlockSplitter {
140
138
  HistogramType combined_histo[2];
141
139
  double combined_entropy[2];
142
140
  double diff[2];
143
- for (int j = 0; j < 2; ++j) {
144
- int last_histogram_ix = last_histogram_ix_[j];
141
+ for (size_t j = 0; j < 2; ++j) {
142
+ size_t last_histogram_ix = last_histogram_ix_[j];
145
143
  combined_histo[j] = (*histograms_)[curr_histogram_ix_];
146
144
  combined_histo[j].AddHistogram((*histograms_)[last_histogram_ix]);
147
145
  combined_entropy[j] = BitsEntropy(
@@ -153,10 +151,10 @@ class BlockSplitter {
153
151
  diff[0] > split_threshold_ &&
154
152
  diff[1] > split_threshold_) {
155
153
  // Create new block.
156
- split_->lengths[num_blocks_] = block_size_;
157
- split_->types[num_blocks_] = split_->num_types;
154
+ split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
155
+ split_->types[num_blocks_] = static_cast<uint8_t>(split_->num_types);
158
156
  last_histogram_ix_[1] = last_histogram_ix_[0];
159
- last_histogram_ix_[0] = split_->num_types;
157
+ last_histogram_ix_[0] = static_cast<uint8_t>(split_->num_types);
160
158
  last_entropy_[1] = last_entropy_[0];
161
159
  last_entropy_[0] = entropy;
162
160
  ++num_blocks_;
@@ -167,7 +165,7 @@ class BlockSplitter {
167
165
  target_block_size_ = min_block_size_;
168
166
  } else if (diff[1] < diff[0] - 20.0) {
169
167
  // Combine this block with second last block.
170
- split_->lengths[num_blocks_] = block_size_;
168
+ split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
171
169
  split_->types[num_blocks_] = split_->types[num_blocks_ - 2];
172
170
  std::swap(last_histogram_ix_[0], last_histogram_ix_[1]);
173
171
  (*histograms_)[last_histogram_ix_[0]] = combined_histo[1];
@@ -180,7 +178,7 @@ class BlockSplitter {
180
178
  target_block_size_ = min_block_size_;
181
179
  } else {
182
180
  // Combine this block with last block.
183
- split_->lengths[num_blocks_ - 1] += block_size_;
181
+ split_->lengths[num_blocks_ - 1] += static_cast<uint32_t>(block_size_);
184
182
  (*histograms_)[last_histogram_ix_[0]] = combined_histo[0];
185
183
  last_entropy_[0] = combined_entropy[0];
186
184
  if (split_->num_types == 1) {
@@ -201,35 +199,35 @@ class BlockSplitter {
201
199
  }
202
200
 
203
201
  private:
204
- static const int kMaxBlockTypes = 256;
202
+ static const uint16_t kMaxBlockTypes = 256;
205
203
 
206
204
  // Alphabet size of particular block category.
207
- const int alphabet_size_;
205
+ const size_t alphabet_size_;
208
206
  // We collect at least this many symbols for each block.
209
- const int min_block_size_;
207
+ const size_t min_block_size_;
210
208
  // We merge histograms A and B if
211
209
  // entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,
212
210
  // where A is the current histogram and B is the histogram of the last or the
213
211
  // second last block type.
214
212
  const double split_threshold_;
215
213
 
216
- int num_blocks_;
214
+ size_t num_blocks_;
217
215
  BlockSplit* split_; // not owned
218
216
  std::vector<HistogramType>* histograms_; // not owned
219
217
 
220
218
  // The number of symbols that we want to collect before deciding on whether
221
219
  // or not to merge the block with a previous one or emit a new block.
222
- int target_block_size_;
220
+ size_t target_block_size_;
223
221
  // The number of symbols in the current histogram.
224
- int block_size_;
222
+ size_t block_size_;
225
223
  // Offset of the current histogram.
226
- int curr_histogram_ix_;
224
+ size_t curr_histogram_ix_;
227
225
  // Offset of the histograms of the previous two block types.
228
- int last_histogram_ix_[2];
226
+ size_t last_histogram_ix_[2];
229
227
  // Entropy of the previous two block types.
230
228
  double last_entropy_[2];
231
229
  // The number of times we merged the current block with the last one.
232
- int merge_last_count_;
230
+ size_t merge_last_count_;
233
231
  };
234
232
 
235
233
  void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
@@ -238,7 +236,7 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
238
236
  const Command *commands,
239
237
  size_t n_commands,
240
238
  MetaBlockSplit* mb) {
241
- int num_literals = 0;
239
+ size_t num_literals = 0;
242
240
  for (size_t i = 0; i < n_commands; ++i) {
243
241
  num_literals += commands[i].insert_len_;
244
242
  }
@@ -247,21 +245,21 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
247
245
  256, 512, 400.0, num_literals,
248
246
  &mb->literal_split, &mb->literal_histograms);
249
247
  BlockSplitter<HistogramCommand> cmd_blocks(
250
- kNumCommandPrefixes, 1024, 500.0, static_cast<int>(n_commands),
248
+ kNumCommandPrefixes, 1024, 500.0, n_commands,
251
249
  &mb->command_split, &mb->command_histograms);
252
250
  BlockSplitter<HistogramDistance> dist_blocks(
253
- 64, 512, 100.0, static_cast<int>(n_commands),
251
+ 64, 512, 100.0, n_commands,
254
252
  &mb->distance_split, &mb->distance_histograms);
255
253
 
256
254
  for (size_t i = 0; i < n_commands; ++i) {
257
255
  const Command cmd = commands[i];
258
256
  cmd_blocks.AddSymbol(cmd.cmd_prefix_);
259
- for (int j = 0; j < cmd.insert_len_; ++j) {
257
+ for (size_t j = cmd.insert_len_; j != 0; --j) {
260
258
  lit_blocks.AddSymbol(ringbuffer[pos & mask]);
261
259
  ++pos;
262
260
  }
263
- pos += cmd.copy_len_;
264
- if (cmd.copy_len_ > 0 && cmd.cmd_prefix_ >= 128) {
261
+ pos += cmd.copy_len();
262
+ if (cmd.copy_len() && cmd.cmd_prefix_ >= 128) {
265
263
  dist_blocks.AddSymbol(cmd.dist_prefix_);
266
264
  }
267
265
  }
@@ -276,11 +274,11 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
276
274
  template<typename HistogramType>
277
275
  class ContextBlockSplitter {
278
276
  public:
279
- ContextBlockSplitter(int alphabet_size,
280
- int num_contexts,
281
- int min_block_size,
277
+ ContextBlockSplitter(size_t alphabet_size,
278
+ size_t num_contexts,
279
+ size_t min_block_size,
282
280
  double split_threshold,
283
- int num_symbols,
281
+ size_t num_symbols,
284
282
  BlockSplit* split,
285
283
  std::vector<HistogramType>* histograms)
286
284
  : alphabet_size_(alphabet_size),
@@ -296,10 +294,10 @@ class ContextBlockSplitter {
296
294
  curr_histogram_ix_(0),
297
295
  last_entropy_(2 * num_contexts),
298
296
  merge_last_count_(0) {
299
- int max_num_blocks = num_symbols / min_block_size + 1;
297
+ size_t max_num_blocks = num_symbols / min_block_size + 1;
300
298
  // We have to allocate one more histogram than the maximum number of block
301
299
  // types for the current histogram when the meta-block is too big.
302
- int max_num_types = std::min(max_num_blocks, max_block_types_ + 1);
300
+ size_t max_num_types = std::min(max_num_blocks, max_block_types_ + 1);
303
301
  split_->lengths.resize(max_num_blocks);
304
302
  split_->types.resize(max_num_blocks);
305
303
  histograms_->resize(max_num_types * num_contexts);
@@ -308,7 +306,7 @@ class ContextBlockSplitter {
308
306
 
309
307
  // Adds the next symbol to the current block type and context. When the
310
308
  // current block reaches the target size, decides on merging the block.
311
- void AddSymbol(int symbol, int context) {
309
+ void AddSymbol(size_t symbol, size_t context) {
312
310
  (*histograms_)[curr_histogram_ix_ + context].Add(symbol);
313
311
  ++block_size_;
314
312
  if (block_size_ == target_block_size_) {
@@ -326,9 +324,9 @@ class ContextBlockSplitter {
326
324
  }
327
325
  if (num_blocks_ == 0) {
328
326
  // Create first block.
329
- split_->lengths[0] = block_size_;
327
+ split_->lengths[0] = static_cast<uint32_t>(block_size_);
330
328
  split_->types[0] = 0;
331
- for (int i = 0; i < num_contexts_; ++i) {
329
+ for (size_t i = 0; i < num_contexts_; ++i) {
332
330
  last_entropy_[i] =
333
331
  BitsEntropy(&(*histograms_)[i].data_[0], alphabet_size_);
334
332
  last_entropy_[num_contexts_ + i] = last_entropy_[i];
@@ -346,13 +344,13 @@ class ContextBlockSplitter {
346
344
  std::vector<HistogramType> combined_histo(2 * num_contexts_);
347
345
  std::vector<double> combined_entropy(2 * num_contexts_);
348
346
  double diff[2] = { 0.0 };
349
- for (int i = 0; i < num_contexts_; ++i) {
350
- int curr_histo_ix = curr_histogram_ix_ + i;
347
+ for (size_t i = 0; i < num_contexts_; ++i) {
348
+ size_t curr_histo_ix = curr_histogram_ix_ + i;
351
349
  entropy[i] = BitsEntropy(&(*histograms_)[curr_histo_ix].data_[0],
352
350
  alphabet_size_);
353
- for (int j = 0; j < 2; ++j) {
354
- int jx = j * num_contexts_ + i;
355
- int last_histogram_ix = last_histogram_ix_[j] + i;
351
+ for (size_t j = 0; j < 2; ++j) {
352
+ size_t jx = j * num_contexts_ + i;
353
+ size_t last_histogram_ix = last_histogram_ix_[j] + i;
356
354
  combined_histo[jx] = (*histograms_)[curr_histo_ix];
357
355
  combined_histo[jx].AddHistogram((*histograms_)[last_histogram_ix]);
358
356
  combined_entropy[jx] = BitsEntropy(
@@ -365,11 +363,11 @@ class ContextBlockSplitter {
365
363
  diff[0] > split_threshold_ &&
366
364
  diff[1] > split_threshold_) {
367
365
  // Create new block.
368
- split_->lengths[num_blocks_] = block_size_;
369
- split_->types[num_blocks_] = split_->num_types;
366
+ split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
367
+ split_->types[num_blocks_] = static_cast<uint8_t>(split_->num_types);
370
368
  last_histogram_ix_[1] = last_histogram_ix_[0];
371
369
  last_histogram_ix_[0] = split_->num_types * num_contexts_;
372
- for (int i = 0; i < num_contexts_; ++i) {
370
+ for (size_t i = 0; i < num_contexts_; ++i) {
373
371
  last_entropy_[num_contexts_ + i] = last_entropy_[i];
374
372
  last_entropy_[i] = entropy[i];
375
373
  }
@@ -381,10 +379,10 @@ class ContextBlockSplitter {
381
379
  target_block_size_ = min_block_size_;
382
380
  } else if (diff[1] < diff[0] - 20.0) {
383
381
  // Combine this block with second last block.
384
- split_->lengths[num_blocks_] = block_size_;
382
+ split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
385
383
  split_->types[num_blocks_] = split_->types[num_blocks_ - 2];
386
384
  std::swap(last_histogram_ix_[0], last_histogram_ix_[1]);
387
- for (int i = 0; i < num_contexts_; ++i) {
385
+ for (size_t i = 0; i < num_contexts_; ++i) {
388
386
  (*histograms_)[last_histogram_ix_[0] + i] =
389
387
  combined_histo[num_contexts_ + i];
390
388
  last_entropy_[num_contexts_ + i] = last_entropy_[i];
@@ -397,8 +395,8 @@ class ContextBlockSplitter {
397
395
  target_block_size_ = min_block_size_;
398
396
  } else {
399
397
  // Combine this block with last block.
400
- split_->lengths[num_blocks_ - 1] += block_size_;
401
- for (int i = 0; i < num_contexts_; ++i) {
398
+ split_->lengths[num_blocks_ - 1] += static_cast<uint32_t>(block_size_);
399
+ for (size_t i = 0; i < num_contexts_; ++i) {
402
400
  (*histograms_)[last_histogram_ix_[0] + i] = combined_histo[i];
403
401
  last_entropy_[i] = combined_entropy[i];
404
402
  if (split_->num_types == 1) {
@@ -423,34 +421,34 @@ class ContextBlockSplitter {
423
421
  static const int kMaxBlockTypes = 256;
424
422
 
425
423
  // Alphabet size of particular block category.
426
- const int alphabet_size_;
427
- const int num_contexts_;
428
- const int max_block_types_;
424
+ const size_t alphabet_size_;
425
+ const size_t num_contexts_;
426
+ const size_t max_block_types_;
429
427
  // We collect at least this many symbols for each block.
430
- const int min_block_size_;
428
+ const size_t min_block_size_;
431
429
  // We merge histograms A and B if
432
430
  // entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,
433
431
  // where A is the current histogram and B is the histogram of the last or the
434
432
  // second last block type.
435
433
  const double split_threshold_;
436
434
 
437
- int num_blocks_;
435
+ size_t num_blocks_;
438
436
  BlockSplit* split_; // not owned
439
437
  std::vector<HistogramType>* histograms_; // not owned
440
438
 
441
439
  // The number of symbols that we want to collect before deciding on whether
442
440
  // or not to merge the block with a previous one or emit a new block.
443
- int target_block_size_;
441
+ size_t target_block_size_;
444
442
  // The number of symbols in the current histogram.
445
- int block_size_;
443
+ size_t block_size_;
446
444
  // Offset of the current histogram.
447
- int curr_histogram_ix_;
445
+ size_t curr_histogram_ix_;
448
446
  // Offset of the histograms of the previous two block types.
449
- int last_histogram_ix_[2];
447
+ size_t last_histogram_ix_[2];
450
448
  // Entropy of the previous two block types.
451
449
  std::vector<double> last_entropy_;
452
450
  // The number of times we merged the current block with the last one.
453
- int merge_last_count_;
451
+ size_t merge_last_count_;
454
452
  };
455
453
 
456
454
  void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
@@ -458,13 +456,13 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
458
456
  size_t mask,
459
457
  uint8_t prev_byte,
460
458
  uint8_t prev_byte2,
461
- int literal_context_mode,
462
- int num_contexts,
463
- const int* static_context_map,
459
+ ContextType literal_context_mode,
460
+ size_t num_contexts,
461
+ const uint32_t* static_context_map,
464
462
  const Command *commands,
465
463
  size_t n_commands,
466
464
  MetaBlockSplit* mb) {
467
- int num_literals = 0;
465
+ size_t num_literals = 0;
468
466
  for (size_t i = 0; i < n_commands; ++i) {
469
467
  num_literals += commands[i].insert_len_;
470
468
  }
@@ -473,25 +471,25 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
473
471
  256, num_contexts, 512, 400.0, num_literals,
474
472
  &mb->literal_split, &mb->literal_histograms);
475
473
  BlockSplitter<HistogramCommand> cmd_blocks(
476
- kNumCommandPrefixes, 1024, 500.0, static_cast<int>(n_commands),
474
+ kNumCommandPrefixes, 1024, 500.0, n_commands,
477
475
  &mb->command_split, &mb->command_histograms);
478
476
  BlockSplitter<HistogramDistance> dist_blocks(
479
- 64, 512, 100.0, static_cast<int>(n_commands),
477
+ 64, 512, 100.0, n_commands,
480
478
  &mb->distance_split, &mb->distance_histograms);
481
479
 
482
480
  for (size_t i = 0; i < n_commands; ++i) {
483
481
  const Command cmd = commands[i];
484
482
  cmd_blocks.AddSymbol(cmd.cmd_prefix_);
485
- for (int j = 0; j < cmd.insert_len_; ++j) {
486
- int context = Context(prev_byte, prev_byte2, literal_context_mode);
483
+ for (size_t j = cmd.insert_len_; j != 0; --j) {
484
+ size_t context = Context(prev_byte, prev_byte2, literal_context_mode);
487
485
  uint8_t literal = ringbuffer[pos & mask];
488
486
  lit_blocks.AddSymbol(literal, static_context_map[context]);
489
487
  prev_byte2 = prev_byte;
490
488
  prev_byte = literal;
491
489
  ++pos;
492
490
  }
493
- pos += cmd.copy_len_;
494
- if (cmd.copy_len_ > 0) {
491
+ pos += cmd.copy_len();
492
+ if (cmd.copy_len()) {
495
493
  prev_byte2 = ringbuffer[(pos - 2) & mask];
496
494
  prev_byte = ringbuffer[(pos - 1) & mask];
497
495
  if (cmd.cmd_prefix_ >= 128) {
@@ -506,31 +504,36 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
506
504
 
507
505
  mb->literal_context_map.resize(
508
506
  mb->literal_split.num_types << kLiteralContextBits);
509
- for (int i = 0; i < mb->literal_split.num_types; ++i) {
510
- for (int j = 0; j < (1 << kLiteralContextBits); ++j) {
507
+ for (size_t i = 0; i < mb->literal_split.num_types; ++i) {
508
+ for (size_t j = 0; j < (1u << kLiteralContextBits); ++j) {
511
509
  mb->literal_context_map[(i << kLiteralContextBits) + j] =
512
- i * num_contexts + static_context_map[j];
510
+ static_cast<uint32_t>(i * num_contexts) + static_context_map[j];
513
511
  }
514
512
  }
515
513
  }
516
514
 
517
- void OptimizeHistograms(int num_direct_distance_codes,
518
- int distance_postfix_bits,
515
+ void OptimizeHistograms(size_t num_direct_distance_codes,
516
+ size_t distance_postfix_bits,
519
517
  MetaBlockSplit* mb) {
518
+ uint8_t* good_for_rle = new uint8_t[kNumCommandPrefixes];
520
519
  for (size_t i = 0; i < mb->literal_histograms.size(); ++i) {
521
- OptimizeHuffmanCountsForRle(256, &mb->literal_histograms[i].data_[0]);
520
+ OptimizeHuffmanCountsForRle(256, &mb->literal_histograms[i].data_[0],
521
+ good_for_rle);
522
522
  }
523
523
  for (size_t i = 0; i < mb->command_histograms.size(); ++i) {
524
524
  OptimizeHuffmanCountsForRle(kNumCommandPrefixes,
525
- &mb->command_histograms[i].data_[0]);
525
+ &mb->command_histograms[i].data_[0],
526
+ good_for_rle);
526
527
  }
527
- int num_distance_codes =
528
+ size_t num_distance_codes =
528
529
  kNumDistanceShortCodes + num_direct_distance_codes +
529
- (48 << distance_postfix_bits);
530
+ (48u << distance_postfix_bits);
530
531
  for (size_t i = 0; i < mb->distance_histograms.size(); ++i) {
531
532
  OptimizeHuffmanCountsForRle(num_distance_codes,
532
- &mb->distance_histograms[i].data_[0]);
533
+ &mb->distance_histograms[i].data_[0],
534
+ good_for_rle);
533
535
  }
536
+ delete[] good_for_rle;
534
537
  }
535
538
 
536
539
  } // namespace brotli
@@ -18,19 +18,19 @@
18
18
  namespace brotli {
19
19
 
20
20
  struct BlockSplit {
21
- BlockSplit() : num_types(0) {}
21
+ BlockSplit(void) : num_types(0) {}
22
22
 
23
- int num_types;
24
- std::vector<int> types;
25
- std::vector<int> lengths;
23
+ size_t num_types;
24
+ std::vector<uint8_t> types;
25
+ std::vector<uint32_t> lengths;
26
26
  };
27
27
 
28
28
  struct MetaBlockSplit {
29
29
  BlockSplit literal_split;
30
30
  BlockSplit command_split;
31
31
  BlockSplit distance_split;
32
- std::vector<int> literal_context_map;
33
- std::vector<int> distance_context_map;
32
+ std::vector<uint32_t> literal_context_map;
33
+ std::vector<uint32_t> distance_context_map;
34
34
  std::vector<HistogramLiteral> literal_histograms;
35
35
  std::vector<HistogramCommand> command_histograms;
36
36
  std::vector<HistogramDistance> distance_histograms;
@@ -44,7 +44,7 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
44
44
  uint8_t prev_byte2,
45
45
  const Command* cmds,
46
46
  size_t num_commands,
47
- int literal_context_mode,
47
+ ContextType literal_context_mode,
48
48
  MetaBlockSplit* mb);
49
49
 
50
50
  // Uses a fast greedy block splitter that tries to merge current block with the
@@ -64,15 +64,15 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
64
64
  size_t mask,
65
65
  uint8_t prev_byte,
66
66
  uint8_t prev_byte2,
67
- int literal_context_mode,
68
- int num_contexts,
69
- const int* static_context_map,
67
+ ContextType literal_context_mode,
68
+ size_t num_contexts,
69
+ const uint32_t* static_context_map,
70
70
  const Command *commands,
71
71
  size_t n_commands,
72
72
  MetaBlockSplit* mb);
73
73
 
74
- void OptimizeHistograms(int num_direct_distance_codes,
75
- int distance_postfix_bits,
74
+ void OptimizeHistograms(size_t num_direct_distance_codes,
75
+ size_t distance_postfix_bits,
76
76
  MetaBlockSplit* mb);
77
77
 
78
78
  } // namespace brotli
@@ -22,10 +22,9 @@
22
22
  /* Let's try and follow the Linux convention */
23
23
  #define __BYTE_ORDER BYTE_ORDER
24
24
  #define __LITTLE_ENDIAN LITTLE_ENDIAN
25
- #define __BIG_ENDIAN BIG_ENDIAN
26
25
  #endif
27
26
 
28
- // define the macros IS_LITTLE_ENDIAN or IS_BIG_ENDIAN
27
+ // define the macro IS_LITTLE_ENDIAN
29
28
  // using the above endian definitions from endian.h if
30
29
  // endian.h was included
31
30
  #ifdef __BYTE_ORDER
@@ -33,19 +32,17 @@
33
32
  #define IS_LITTLE_ENDIAN
34
33
  #endif
35
34
 
36
- #if __BYTE_ORDER == __BIG_ENDIAN
37
- #define IS_BIG_ENDIAN
38
- #endif
39
-
40
35
  #else
41
36
 
42
37
  #if defined(__LITTLE_ENDIAN__)
43
38
  #define IS_LITTLE_ENDIAN
44
- #elif defined(__BIG_ENDIAN__)
45
- #define IS_BIG_ENDIAN
46
39
  #endif
47
40
  #endif // __BYTE_ORDER
48
41
 
42
+ #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
43
+ #define IS_LITTLE_ENDIAN
44
+ #endif
45
+
49
46
  // Enable little-endian optimization for x64 architecture on Windows.
50
47
  #if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
51
48
  #define IS_LITTLE_ENDIAN
@@ -69,8 +66,8 @@
69
66
  // On some platforms, like ARM, the copy functions can be more efficient
70
67
  // then a load and a store.
71
68
 
72
- #if defined(ARCH_PIII) || defined(ARCH_ATHLON) || \
73
- defined(ARCH_K8) || defined(_ARCH_PPC)
69
+ #if defined(ARCH_PIII) || \
70
+ defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC)
74
71
 
75
72
  // x86 and x86-64 can perform unaligned loads/stores directly;
76
73
  // modern PowerPC hardware can also do unaligned integer loads and stores;
@@ -142,10 +139,4 @@ inline void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
142
139
 
143
140
  #endif
144
141
 
145
- #ifdef BROTLI_ENCODE_DEBUG
146
- #define BROTLI_DCHECK(x) assert(x)
147
- #else
148
- #define BROTLI_DCHECK(x)
149
- #endif
150
-
151
142
  #endif // BROTLI_ENC_PORT_H_
@@ -15,18 +15,18 @@
15
15
 
16
16
  namespace brotli {
17
17
 
18
- static const int kNumInsertLenPrefixes = 24;
19
- static const int kNumCopyLenPrefixes = 24;
20
- static const int kNumCommandPrefixes = 704;
21
- static const int kNumBlockLenPrefixes = 26;
22
- static const int kNumDistanceShortCodes = 16;
23
- static const int kNumDistancePrefixes = 520;
18
+ static const uint32_t kNumInsertLenPrefixes = 24;
19
+ static const uint32_t kNumCopyLenPrefixes = 24;
20
+ static const uint32_t kNumCommandPrefixes = 704;
21
+ static const uint32_t kNumBlockLenPrefixes = 26;
22
+ static const uint32_t kNumDistanceShortCodes = 16;
23
+ static const uint32_t kNumDistancePrefixes = 520;
24
24
 
25
25
  // Represents the range of values belonging to a prefix code:
26
26
  // [offset, offset + 2^nbits)
27
27
  struct PrefixCodeRange {
28
- int offset;
29
- int nbits;
28
+ uint32_t offset;
29
+ uint32_t nbits;
30
30
  };
31
31
 
32
32
  static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
@@ -39,8 +39,8 @@ static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
39
39
  {8433, 13}, {16625, 24}
40
40
  };
41
41
 
42
- inline void GetBlockLengthPrefixCode(int len,
43
- int* code, int* n_extra, int* extra) {
42
+ inline void GetBlockLengthPrefixCode(uint32_t len, uint32_t* code,
43
+ uint32_t* n_extra, uint32_t* extra) {
44
44
  *code = 0;
45
45
  while (*code < 25 && len >= kBlockLengthPrefixCode[*code + 1].offset) {
46
46
  ++(*code);
@@ -49,9 +49,9 @@ inline void GetBlockLengthPrefixCode(int len,
49
49
  *extra = len - kBlockLengthPrefixCode[*code].offset;
50
50
  }
51
51
 
52
- inline void PrefixEncodeCopyDistance(int distance_code,
53
- int num_direct_codes,
54
- int postfix_bits,
52
+ inline void PrefixEncodeCopyDistance(size_t distance_code,
53
+ size_t num_direct_codes,
54
+ size_t postfix_bits,
55
55
  uint16_t* code,
56
56
  uint32_t* extra_bits) {
57
57
  if (distance_code < kNumDistanceShortCodes + num_direct_codes) {
@@ -59,18 +59,19 @@ inline void PrefixEncodeCopyDistance(int distance_code,
59
59
  *extra_bits = 0;
60
60
  return;
61
61
  }
62
- distance_code -= kNumDistanceShortCodes + num_direct_codes;
63
- distance_code += (1 << (postfix_bits + 2));
64
- int bucket = Log2Floor(distance_code) - 1;
65
- int postfix_mask = (1 << postfix_bits) - 1;
66
- int postfix = distance_code & postfix_mask;
67
- int prefix = (distance_code >> bucket) & 1;
68
- int offset = (2 + prefix) << bucket;
69
- int nbits = bucket - postfix_bits;
62
+ distance_code -= kNumDistanceShortCodes + num_direct_codes; /* >= 0 */
63
+ distance_code += (1u << (postfix_bits + 2u)); /* > 0 */
64
+ size_t bucket = Log2FloorNonZero(distance_code) - 1;
65
+ size_t postfix_mask = (1 << postfix_bits) - 1;
66
+ size_t postfix = distance_code & postfix_mask;
67
+ size_t prefix = (distance_code >> bucket) & 1;
68
+ size_t offset = (2 + prefix) << bucket;
69
+ size_t nbits = bucket - postfix_bits;
70
70
  *code = static_cast<uint16_t>(
71
71
  (kNumDistanceShortCodes + num_direct_codes +
72
72
  ((2 * (nbits - 1) + prefix) << postfix_bits) + postfix));
73
- *extra_bits = (nbits << 24) | ((distance_code - offset) >> postfix_bits);
73
+ *extra_bits = static_cast<uint32_t>(
74
+ (nbits << 24) | ((distance_code - offset) >> postfix_bits));
74
75
  }
75
76
 
76
77
  } // namespace brotli