brotli 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/ext/brotli/brotli.cc +114 -24
  3. data/ext/brotli/brotli.h +0 -1
  4. data/ext/brotli/extconf.rb +30 -23
  5. data/lib/brotli/version.rb +1 -1
  6. data/vendor/brotli/LICENSE +1 -1
  7. data/vendor/brotli/dec/Makefile +1 -1
  8. data/vendor/brotli/dec/bit_reader.c +3 -3
  9. data/vendor/brotli/dec/bit_reader.h +25 -27
  10. data/vendor/brotli/dec/context.h +4 -4
  11. data/vendor/brotli/dec/decode.c +410 -486
  12. data/vendor/brotli/dec/decode.h +101 -105
  13. data/vendor/brotli/dec/dictionary.c +1 -1
  14. data/vendor/brotli/dec/dictionary.h +7 -8
  15. data/vendor/brotli/dec/huffman.c +103 -105
  16. data/vendor/brotli/dec/huffman.h +18 -18
  17. data/vendor/brotli/dec/port.h +52 -40
  18. data/vendor/brotli/dec/prefix.h +2 -0
  19. data/vendor/brotli/dec/state.c +13 -19
  20. data/vendor/brotli/dec/state.h +25 -39
  21. data/vendor/brotli/dec/transform.h +38 -44
  22. data/vendor/brotli/dec/types.h +2 -2
  23. data/vendor/brotli/enc/Makefile +1 -1
  24. data/vendor/brotli/enc/backward_references.cc +455 -359
  25. data/vendor/brotli/enc/backward_references.h +79 -3
  26. data/vendor/brotli/enc/bit_cost.h +54 -32
  27. data/vendor/brotli/enc/block_splitter.cc +285 -193
  28. data/vendor/brotli/enc/block_splitter.h +4 -12
  29. data/vendor/brotli/enc/brotli_bit_stream.cc +623 -324
  30. data/vendor/brotli/enc/brotli_bit_stream.h +76 -37
  31. data/vendor/brotli/enc/cluster.h +161 -120
  32. data/vendor/brotli/enc/command.h +60 -37
  33. data/vendor/brotli/enc/compress_fragment.cc +701 -0
  34. data/vendor/brotli/enc/compress_fragment.h +47 -0
  35. data/vendor/brotli/enc/compress_fragment_two_pass.cc +524 -0
  36. data/vendor/brotli/enc/compress_fragment_two_pass.h +40 -0
  37. data/vendor/brotli/enc/compressor.h +15 -0
  38. data/vendor/brotli/enc/context.h +1 -1
  39. data/vendor/brotli/enc/dictionary.h +2 -2
  40. data/vendor/brotli/enc/encode.cc +819 -286
  41. data/vendor/brotli/enc/encode.h +38 -15
  42. data/vendor/brotli/enc/encode_parallel.cc +40 -42
  43. data/vendor/brotli/enc/entropy_encode.cc +144 -147
  44. data/vendor/brotli/enc/entropy_encode.h +32 -8
  45. data/vendor/brotli/enc/entropy_encode_static.h +572 -0
  46. data/vendor/brotli/enc/fast_log.h +7 -40
  47. data/vendor/brotli/enc/find_match_length.h +9 -9
  48. data/vendor/brotli/enc/hash.h +462 -154
  49. data/vendor/brotli/enc/histogram.cc +6 -6
  50. data/vendor/brotli/enc/histogram.h +13 -13
  51. data/vendor/brotli/enc/literal_cost.cc +45 -45
  52. data/vendor/brotli/enc/metablock.cc +92 -89
  53. data/vendor/brotli/enc/metablock.h +12 -12
  54. data/vendor/brotli/enc/port.h +7 -16
  55. data/vendor/brotli/enc/prefix.h +23 -22
  56. data/vendor/brotli/enc/ringbuffer.h +75 -29
  57. data/vendor/brotli/enc/static_dict.cc +56 -48
  58. data/vendor/brotli/enc/static_dict.h +5 -5
  59. data/vendor/brotli/enc/streams.cc +1 -1
  60. data/vendor/brotli/enc/streams.h +5 -5
  61. data/vendor/brotli/enc/transform.h +40 -35
  62. data/vendor/brotli/enc/types.h +2 -0
  63. data/vendor/brotli/enc/utf8_util.cc +3 -2
  64. data/vendor/brotli/enc/write_bits.h +6 -6
  65. metadata +9 -5
  66. data/vendor/brotli/dec/streams.c +0 -102
  67. data/vendor/brotli/dec/streams.h +0 -95
@@ -23,7 +23,7 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
23
23
  uint8_t prev_byte2,
24
24
  const Command* cmds,
25
25
  size_t num_commands,
26
- int literal_context_mode,
26
+ ContextType literal_context_mode,
27
27
  MetaBlockSplit* mb) {
28
28
  SplitBlock(cmds, num_commands,
29
29
  ringbuffer, pos, mask,
@@ -31,12 +31,12 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
31
31
  &mb->command_split,
32
32
  &mb->distance_split);
33
33
 
34
- std::vector<int> literal_context_modes(mb->literal_split.num_types,
35
- literal_context_mode);
34
+ std::vector<ContextType> literal_context_modes(mb->literal_split.num_types,
35
+ literal_context_mode);
36
36
 
37
- int num_literal_contexts =
37
+ size_t num_literal_contexts =
38
38
  mb->literal_split.num_types << kLiteralContextBits;
39
- int num_distance_contexts =
39
+ size_t num_distance_contexts =
40
40
  mb->distance_split.num_types << kDistanceContextBits;
41
41
  std::vector<HistogramLiteral> literal_histograms(num_literal_contexts);
42
42
  mb->command_histograms.resize(mb->command_split.num_types);
@@ -58,17 +58,15 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
58
58
  // Histogram ids need to fit in one byte.
59
59
  static const size_t kMaxNumberOfHistograms = 256;
60
60
 
61
- mb->literal_histograms = literal_histograms;
62
61
  ClusterHistograms(literal_histograms,
63
- 1 << kLiteralContextBits,
62
+ 1u << kLiteralContextBits,
64
63
  mb->literal_split.num_types,
65
64
  kMaxNumberOfHistograms,
66
65
  &mb->literal_histograms,
67
66
  &mb->literal_context_map);
68
67
 
69
- mb->distance_histograms = distance_histograms;
70
68
  ClusterHistograms(distance_histograms,
71
- 1 << kDistanceContextBits,
69
+ 1u << kDistanceContextBits,
72
70
  mb->distance_split.num_types,
73
71
  kMaxNumberOfHistograms,
74
72
  &mb->distance_histograms,
@@ -79,10 +77,10 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
79
77
  template<typename HistogramType>
80
78
  class BlockSplitter {
81
79
  public:
82
- BlockSplitter(int alphabet_size,
83
- int min_block_size,
80
+ BlockSplitter(size_t alphabet_size,
81
+ size_t min_block_size,
84
82
  double split_threshold,
85
- int num_symbols,
83
+ size_t num_symbols,
86
84
  BlockSplit* split,
87
85
  std::vector<HistogramType>* histograms)
88
86
  : alphabet_size_(alphabet_size),
@@ -95,10 +93,10 @@ class BlockSplitter {
95
93
  block_size_(0),
96
94
  curr_histogram_ix_(0),
97
95
  merge_last_count_(0) {
98
- int max_num_blocks = num_symbols / min_block_size + 1;
96
+ size_t max_num_blocks = num_symbols / min_block_size + 1;
99
97
  // We have to allocate one more histogram than the maximum number of block
100
98
  // types for the current histogram when the meta-block is too big.
101
- int max_num_types = std::min(max_num_blocks, kMaxBlockTypes + 1);
99
+ size_t max_num_types = std::min<size_t>(max_num_blocks, kMaxBlockTypes + 1);
102
100
  split_->lengths.resize(max_num_blocks);
103
101
  split_->types.resize(max_num_blocks);
104
102
  histograms_->resize(max_num_types);
@@ -107,7 +105,7 @@ class BlockSplitter {
107
105
 
108
106
  // Adds the next symbol to the current histogram. When the current histogram
109
107
  // reaches the target size, decides on merging the block.
110
- void AddSymbol(int symbol) {
108
+ void AddSymbol(size_t symbol) {
111
109
  (*histograms_)[curr_histogram_ix_].Add(symbol);
112
110
  ++block_size_;
113
111
  if (block_size_ == target_block_size_) {
@@ -125,7 +123,7 @@ class BlockSplitter {
125
123
  }
126
124
  if (num_blocks_ == 0) {
127
125
  // Create first block.
128
- split_->lengths[0] = block_size_;
126
+ split_->lengths[0] = static_cast<uint32_t>(block_size_);
129
127
  split_->types[0] = 0;
130
128
  last_entropy_[0] =
131
129
  BitsEntropy(&(*histograms_)[0].data_[0], alphabet_size_);
@@ -140,8 +138,8 @@ class BlockSplitter {
140
138
  HistogramType combined_histo[2];
141
139
  double combined_entropy[2];
142
140
  double diff[2];
143
- for (int j = 0; j < 2; ++j) {
144
- int last_histogram_ix = last_histogram_ix_[j];
141
+ for (size_t j = 0; j < 2; ++j) {
142
+ size_t last_histogram_ix = last_histogram_ix_[j];
145
143
  combined_histo[j] = (*histograms_)[curr_histogram_ix_];
146
144
  combined_histo[j].AddHistogram((*histograms_)[last_histogram_ix]);
147
145
  combined_entropy[j] = BitsEntropy(
@@ -153,10 +151,10 @@ class BlockSplitter {
153
151
  diff[0] > split_threshold_ &&
154
152
  diff[1] > split_threshold_) {
155
153
  // Create new block.
156
- split_->lengths[num_blocks_] = block_size_;
157
- split_->types[num_blocks_] = split_->num_types;
154
+ split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
155
+ split_->types[num_blocks_] = static_cast<uint8_t>(split_->num_types);
158
156
  last_histogram_ix_[1] = last_histogram_ix_[0];
159
- last_histogram_ix_[0] = split_->num_types;
157
+ last_histogram_ix_[0] = static_cast<uint8_t>(split_->num_types);
160
158
  last_entropy_[1] = last_entropy_[0];
161
159
  last_entropy_[0] = entropy;
162
160
  ++num_blocks_;
@@ -167,7 +165,7 @@ class BlockSplitter {
167
165
  target_block_size_ = min_block_size_;
168
166
  } else if (diff[1] < diff[0] - 20.0) {
169
167
  // Combine this block with second last block.
170
- split_->lengths[num_blocks_] = block_size_;
168
+ split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
171
169
  split_->types[num_blocks_] = split_->types[num_blocks_ - 2];
172
170
  std::swap(last_histogram_ix_[0], last_histogram_ix_[1]);
173
171
  (*histograms_)[last_histogram_ix_[0]] = combined_histo[1];
@@ -180,7 +178,7 @@ class BlockSplitter {
180
178
  target_block_size_ = min_block_size_;
181
179
  } else {
182
180
  // Combine this block with last block.
183
- split_->lengths[num_blocks_ - 1] += block_size_;
181
+ split_->lengths[num_blocks_ - 1] += static_cast<uint32_t>(block_size_);
184
182
  (*histograms_)[last_histogram_ix_[0]] = combined_histo[0];
185
183
  last_entropy_[0] = combined_entropy[0];
186
184
  if (split_->num_types == 1) {
@@ -201,35 +199,35 @@ class BlockSplitter {
201
199
  }
202
200
 
203
201
  private:
204
- static const int kMaxBlockTypes = 256;
202
+ static const uint16_t kMaxBlockTypes = 256;
205
203
 
206
204
  // Alphabet size of particular block category.
207
- const int alphabet_size_;
205
+ const size_t alphabet_size_;
208
206
  // We collect at least this many symbols for each block.
209
- const int min_block_size_;
207
+ const size_t min_block_size_;
210
208
  // We merge histograms A and B if
211
209
  // entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,
212
210
  // where A is the current histogram and B is the histogram of the last or the
213
211
  // second last block type.
214
212
  const double split_threshold_;
215
213
 
216
- int num_blocks_;
214
+ size_t num_blocks_;
217
215
  BlockSplit* split_; // not owned
218
216
  std::vector<HistogramType>* histograms_; // not owned
219
217
 
220
218
  // The number of symbols that we want to collect before deciding on whether
221
219
  // or not to merge the block with a previous one or emit a new block.
222
- int target_block_size_;
220
+ size_t target_block_size_;
223
221
  // The number of symbols in the current histogram.
224
- int block_size_;
222
+ size_t block_size_;
225
223
  // Offset of the current histogram.
226
- int curr_histogram_ix_;
224
+ size_t curr_histogram_ix_;
227
225
  // Offset of the histograms of the previous two block types.
228
- int last_histogram_ix_[2];
226
+ size_t last_histogram_ix_[2];
229
227
  // Entropy of the previous two block types.
230
228
  double last_entropy_[2];
231
229
  // The number of times we merged the current block with the last one.
232
- int merge_last_count_;
230
+ size_t merge_last_count_;
233
231
  };
234
232
 
235
233
  void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
@@ -238,7 +236,7 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
238
236
  const Command *commands,
239
237
  size_t n_commands,
240
238
  MetaBlockSplit* mb) {
241
- int num_literals = 0;
239
+ size_t num_literals = 0;
242
240
  for (size_t i = 0; i < n_commands; ++i) {
243
241
  num_literals += commands[i].insert_len_;
244
242
  }
@@ -247,21 +245,21 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
247
245
  256, 512, 400.0, num_literals,
248
246
  &mb->literal_split, &mb->literal_histograms);
249
247
  BlockSplitter<HistogramCommand> cmd_blocks(
250
- kNumCommandPrefixes, 1024, 500.0, static_cast<int>(n_commands),
248
+ kNumCommandPrefixes, 1024, 500.0, n_commands,
251
249
  &mb->command_split, &mb->command_histograms);
252
250
  BlockSplitter<HistogramDistance> dist_blocks(
253
- 64, 512, 100.0, static_cast<int>(n_commands),
251
+ 64, 512, 100.0, n_commands,
254
252
  &mb->distance_split, &mb->distance_histograms);
255
253
 
256
254
  for (size_t i = 0; i < n_commands; ++i) {
257
255
  const Command cmd = commands[i];
258
256
  cmd_blocks.AddSymbol(cmd.cmd_prefix_);
259
- for (int j = 0; j < cmd.insert_len_; ++j) {
257
+ for (size_t j = cmd.insert_len_; j != 0; --j) {
260
258
  lit_blocks.AddSymbol(ringbuffer[pos & mask]);
261
259
  ++pos;
262
260
  }
263
- pos += cmd.copy_len_;
264
- if (cmd.copy_len_ > 0 && cmd.cmd_prefix_ >= 128) {
261
+ pos += cmd.copy_len();
262
+ if (cmd.copy_len() && cmd.cmd_prefix_ >= 128) {
265
263
  dist_blocks.AddSymbol(cmd.dist_prefix_);
266
264
  }
267
265
  }
@@ -276,11 +274,11 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
276
274
  template<typename HistogramType>
277
275
  class ContextBlockSplitter {
278
276
  public:
279
- ContextBlockSplitter(int alphabet_size,
280
- int num_contexts,
281
- int min_block_size,
277
+ ContextBlockSplitter(size_t alphabet_size,
278
+ size_t num_contexts,
279
+ size_t min_block_size,
282
280
  double split_threshold,
283
- int num_symbols,
281
+ size_t num_symbols,
284
282
  BlockSplit* split,
285
283
  std::vector<HistogramType>* histograms)
286
284
  : alphabet_size_(alphabet_size),
@@ -296,10 +294,10 @@ class ContextBlockSplitter {
296
294
  curr_histogram_ix_(0),
297
295
  last_entropy_(2 * num_contexts),
298
296
  merge_last_count_(0) {
299
- int max_num_blocks = num_symbols / min_block_size + 1;
297
+ size_t max_num_blocks = num_symbols / min_block_size + 1;
300
298
  // We have to allocate one more histogram than the maximum number of block
301
299
  // types for the current histogram when the meta-block is too big.
302
- int max_num_types = std::min(max_num_blocks, max_block_types_ + 1);
300
+ size_t max_num_types = std::min(max_num_blocks, max_block_types_ + 1);
303
301
  split_->lengths.resize(max_num_blocks);
304
302
  split_->types.resize(max_num_blocks);
305
303
  histograms_->resize(max_num_types * num_contexts);
@@ -308,7 +306,7 @@ class ContextBlockSplitter {
308
306
 
309
307
  // Adds the next symbol to the current block type and context. When the
310
308
  // current block reaches the target size, decides on merging the block.
311
- void AddSymbol(int symbol, int context) {
309
+ void AddSymbol(size_t symbol, size_t context) {
312
310
  (*histograms_)[curr_histogram_ix_ + context].Add(symbol);
313
311
  ++block_size_;
314
312
  if (block_size_ == target_block_size_) {
@@ -326,9 +324,9 @@ class ContextBlockSplitter {
326
324
  }
327
325
  if (num_blocks_ == 0) {
328
326
  // Create first block.
329
- split_->lengths[0] = block_size_;
327
+ split_->lengths[0] = static_cast<uint32_t>(block_size_);
330
328
  split_->types[0] = 0;
331
- for (int i = 0; i < num_contexts_; ++i) {
329
+ for (size_t i = 0; i < num_contexts_; ++i) {
332
330
  last_entropy_[i] =
333
331
  BitsEntropy(&(*histograms_)[i].data_[0], alphabet_size_);
334
332
  last_entropy_[num_contexts_ + i] = last_entropy_[i];
@@ -346,13 +344,13 @@ class ContextBlockSplitter {
346
344
  std::vector<HistogramType> combined_histo(2 * num_contexts_);
347
345
  std::vector<double> combined_entropy(2 * num_contexts_);
348
346
  double diff[2] = { 0.0 };
349
- for (int i = 0; i < num_contexts_; ++i) {
350
- int curr_histo_ix = curr_histogram_ix_ + i;
347
+ for (size_t i = 0; i < num_contexts_; ++i) {
348
+ size_t curr_histo_ix = curr_histogram_ix_ + i;
351
349
  entropy[i] = BitsEntropy(&(*histograms_)[curr_histo_ix].data_[0],
352
350
  alphabet_size_);
353
- for (int j = 0; j < 2; ++j) {
354
- int jx = j * num_contexts_ + i;
355
- int last_histogram_ix = last_histogram_ix_[j] + i;
351
+ for (size_t j = 0; j < 2; ++j) {
352
+ size_t jx = j * num_contexts_ + i;
353
+ size_t last_histogram_ix = last_histogram_ix_[j] + i;
356
354
  combined_histo[jx] = (*histograms_)[curr_histo_ix];
357
355
  combined_histo[jx].AddHistogram((*histograms_)[last_histogram_ix]);
358
356
  combined_entropy[jx] = BitsEntropy(
@@ -365,11 +363,11 @@ class ContextBlockSplitter {
365
363
  diff[0] > split_threshold_ &&
366
364
  diff[1] > split_threshold_) {
367
365
  // Create new block.
368
- split_->lengths[num_blocks_] = block_size_;
369
- split_->types[num_blocks_] = split_->num_types;
366
+ split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
367
+ split_->types[num_blocks_] = static_cast<uint8_t>(split_->num_types);
370
368
  last_histogram_ix_[1] = last_histogram_ix_[0];
371
369
  last_histogram_ix_[0] = split_->num_types * num_contexts_;
372
- for (int i = 0; i < num_contexts_; ++i) {
370
+ for (size_t i = 0; i < num_contexts_; ++i) {
373
371
  last_entropy_[num_contexts_ + i] = last_entropy_[i];
374
372
  last_entropy_[i] = entropy[i];
375
373
  }
@@ -381,10 +379,10 @@ class ContextBlockSplitter {
381
379
  target_block_size_ = min_block_size_;
382
380
  } else if (diff[1] < diff[0] - 20.0) {
383
381
  // Combine this block with second last block.
384
- split_->lengths[num_blocks_] = block_size_;
382
+ split_->lengths[num_blocks_] = static_cast<uint32_t>(block_size_);
385
383
  split_->types[num_blocks_] = split_->types[num_blocks_ - 2];
386
384
  std::swap(last_histogram_ix_[0], last_histogram_ix_[1]);
387
- for (int i = 0; i < num_contexts_; ++i) {
385
+ for (size_t i = 0; i < num_contexts_; ++i) {
388
386
  (*histograms_)[last_histogram_ix_[0] + i] =
389
387
  combined_histo[num_contexts_ + i];
390
388
  last_entropy_[num_contexts_ + i] = last_entropy_[i];
@@ -397,8 +395,8 @@ class ContextBlockSplitter {
397
395
  target_block_size_ = min_block_size_;
398
396
  } else {
399
397
  // Combine this block with last block.
400
- split_->lengths[num_blocks_ - 1] += block_size_;
401
- for (int i = 0; i < num_contexts_; ++i) {
398
+ split_->lengths[num_blocks_ - 1] += static_cast<uint32_t>(block_size_);
399
+ for (size_t i = 0; i < num_contexts_; ++i) {
402
400
  (*histograms_)[last_histogram_ix_[0] + i] = combined_histo[i];
403
401
  last_entropy_[i] = combined_entropy[i];
404
402
  if (split_->num_types == 1) {
@@ -423,34 +421,34 @@ class ContextBlockSplitter {
423
421
  static const int kMaxBlockTypes = 256;
424
422
 
425
423
  // Alphabet size of particular block category.
426
- const int alphabet_size_;
427
- const int num_contexts_;
428
- const int max_block_types_;
424
+ const size_t alphabet_size_;
425
+ const size_t num_contexts_;
426
+ const size_t max_block_types_;
429
427
  // We collect at least this many symbols for each block.
430
- const int min_block_size_;
428
+ const size_t min_block_size_;
431
429
  // We merge histograms A and B if
432
430
  // entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,
433
431
  // where A is the current histogram and B is the histogram of the last or the
434
432
  // second last block type.
435
433
  const double split_threshold_;
436
434
 
437
- int num_blocks_;
435
+ size_t num_blocks_;
438
436
  BlockSplit* split_; // not owned
439
437
  std::vector<HistogramType>* histograms_; // not owned
440
438
 
441
439
  // The number of symbols that we want to collect before deciding on whether
442
440
  // or not to merge the block with a previous one or emit a new block.
443
- int target_block_size_;
441
+ size_t target_block_size_;
444
442
  // The number of symbols in the current histogram.
445
- int block_size_;
443
+ size_t block_size_;
446
444
  // Offset of the current histogram.
447
- int curr_histogram_ix_;
445
+ size_t curr_histogram_ix_;
448
446
  // Offset of the histograms of the previous two block types.
449
- int last_histogram_ix_[2];
447
+ size_t last_histogram_ix_[2];
450
448
  // Entropy of the previous two block types.
451
449
  std::vector<double> last_entropy_;
452
450
  // The number of times we merged the current block with the last one.
453
- int merge_last_count_;
451
+ size_t merge_last_count_;
454
452
  };
455
453
 
456
454
  void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
@@ -458,13 +456,13 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
458
456
  size_t mask,
459
457
  uint8_t prev_byte,
460
458
  uint8_t prev_byte2,
461
- int literal_context_mode,
462
- int num_contexts,
463
- const int* static_context_map,
459
+ ContextType literal_context_mode,
460
+ size_t num_contexts,
461
+ const uint32_t* static_context_map,
464
462
  const Command *commands,
465
463
  size_t n_commands,
466
464
  MetaBlockSplit* mb) {
467
- int num_literals = 0;
465
+ size_t num_literals = 0;
468
466
  for (size_t i = 0; i < n_commands; ++i) {
469
467
  num_literals += commands[i].insert_len_;
470
468
  }
@@ -473,25 +471,25 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
473
471
  256, num_contexts, 512, 400.0, num_literals,
474
472
  &mb->literal_split, &mb->literal_histograms);
475
473
  BlockSplitter<HistogramCommand> cmd_blocks(
476
- kNumCommandPrefixes, 1024, 500.0, static_cast<int>(n_commands),
474
+ kNumCommandPrefixes, 1024, 500.0, n_commands,
477
475
  &mb->command_split, &mb->command_histograms);
478
476
  BlockSplitter<HistogramDistance> dist_blocks(
479
- 64, 512, 100.0, static_cast<int>(n_commands),
477
+ 64, 512, 100.0, n_commands,
480
478
  &mb->distance_split, &mb->distance_histograms);
481
479
 
482
480
  for (size_t i = 0; i < n_commands; ++i) {
483
481
  const Command cmd = commands[i];
484
482
  cmd_blocks.AddSymbol(cmd.cmd_prefix_);
485
- for (int j = 0; j < cmd.insert_len_; ++j) {
486
- int context = Context(prev_byte, prev_byte2, literal_context_mode);
483
+ for (size_t j = cmd.insert_len_; j != 0; --j) {
484
+ size_t context = Context(prev_byte, prev_byte2, literal_context_mode);
487
485
  uint8_t literal = ringbuffer[pos & mask];
488
486
  lit_blocks.AddSymbol(literal, static_context_map[context]);
489
487
  prev_byte2 = prev_byte;
490
488
  prev_byte = literal;
491
489
  ++pos;
492
490
  }
493
- pos += cmd.copy_len_;
494
- if (cmd.copy_len_ > 0) {
491
+ pos += cmd.copy_len();
492
+ if (cmd.copy_len()) {
495
493
  prev_byte2 = ringbuffer[(pos - 2) & mask];
496
494
  prev_byte = ringbuffer[(pos - 1) & mask];
497
495
  if (cmd.cmd_prefix_ >= 128) {
@@ -506,31 +504,36 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
506
504
 
507
505
  mb->literal_context_map.resize(
508
506
  mb->literal_split.num_types << kLiteralContextBits);
509
- for (int i = 0; i < mb->literal_split.num_types; ++i) {
510
- for (int j = 0; j < (1 << kLiteralContextBits); ++j) {
507
+ for (size_t i = 0; i < mb->literal_split.num_types; ++i) {
508
+ for (size_t j = 0; j < (1u << kLiteralContextBits); ++j) {
511
509
  mb->literal_context_map[(i << kLiteralContextBits) + j] =
512
- i * num_contexts + static_context_map[j];
510
+ static_cast<uint32_t>(i * num_contexts) + static_context_map[j];
513
511
  }
514
512
  }
515
513
  }
516
514
 
517
- void OptimizeHistograms(int num_direct_distance_codes,
518
- int distance_postfix_bits,
515
+ void OptimizeHistograms(size_t num_direct_distance_codes,
516
+ size_t distance_postfix_bits,
519
517
  MetaBlockSplit* mb) {
518
+ uint8_t* good_for_rle = new uint8_t[kNumCommandPrefixes];
520
519
  for (size_t i = 0; i < mb->literal_histograms.size(); ++i) {
521
- OptimizeHuffmanCountsForRle(256, &mb->literal_histograms[i].data_[0]);
520
+ OptimizeHuffmanCountsForRle(256, &mb->literal_histograms[i].data_[0],
521
+ good_for_rle);
522
522
  }
523
523
  for (size_t i = 0; i < mb->command_histograms.size(); ++i) {
524
524
  OptimizeHuffmanCountsForRle(kNumCommandPrefixes,
525
- &mb->command_histograms[i].data_[0]);
525
+ &mb->command_histograms[i].data_[0],
526
+ good_for_rle);
526
527
  }
527
- int num_distance_codes =
528
+ size_t num_distance_codes =
528
529
  kNumDistanceShortCodes + num_direct_distance_codes +
529
- (48 << distance_postfix_bits);
530
+ (48u << distance_postfix_bits);
530
531
  for (size_t i = 0; i < mb->distance_histograms.size(); ++i) {
531
532
  OptimizeHuffmanCountsForRle(num_distance_codes,
532
- &mb->distance_histograms[i].data_[0]);
533
+ &mb->distance_histograms[i].data_[0],
534
+ good_for_rle);
533
535
  }
536
+ delete[] good_for_rle;
534
537
  }
535
538
 
536
539
  } // namespace brotli
@@ -18,19 +18,19 @@
18
18
  namespace brotli {
19
19
 
20
20
  struct BlockSplit {
21
- BlockSplit() : num_types(0) {}
21
+ BlockSplit(void) : num_types(0) {}
22
22
 
23
- int num_types;
24
- std::vector<int> types;
25
- std::vector<int> lengths;
23
+ size_t num_types;
24
+ std::vector<uint8_t> types;
25
+ std::vector<uint32_t> lengths;
26
26
  };
27
27
 
28
28
  struct MetaBlockSplit {
29
29
  BlockSplit literal_split;
30
30
  BlockSplit command_split;
31
31
  BlockSplit distance_split;
32
- std::vector<int> literal_context_map;
33
- std::vector<int> distance_context_map;
32
+ std::vector<uint32_t> literal_context_map;
33
+ std::vector<uint32_t> distance_context_map;
34
34
  std::vector<HistogramLiteral> literal_histograms;
35
35
  std::vector<HistogramCommand> command_histograms;
36
36
  std::vector<HistogramDistance> distance_histograms;
@@ -44,7 +44,7 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
44
44
  uint8_t prev_byte2,
45
45
  const Command* cmds,
46
46
  size_t num_commands,
47
- int literal_context_mode,
47
+ ContextType literal_context_mode,
48
48
  MetaBlockSplit* mb);
49
49
 
50
50
  // Uses a fast greedy block splitter that tries to merge current block with the
@@ -64,15 +64,15 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
64
64
  size_t mask,
65
65
  uint8_t prev_byte,
66
66
  uint8_t prev_byte2,
67
- int literal_context_mode,
68
- int num_contexts,
69
- const int* static_context_map,
67
+ ContextType literal_context_mode,
68
+ size_t num_contexts,
69
+ const uint32_t* static_context_map,
70
70
  const Command *commands,
71
71
  size_t n_commands,
72
72
  MetaBlockSplit* mb);
73
73
 
74
- void OptimizeHistograms(int num_direct_distance_codes,
75
- int distance_postfix_bits,
74
+ void OptimizeHistograms(size_t num_direct_distance_codes,
75
+ size_t distance_postfix_bits,
76
76
  MetaBlockSplit* mb);
77
77
 
78
78
  } // namespace brotli
@@ -22,10 +22,9 @@
22
22
  /* Let's try and follow the Linux convention */
23
23
  #define __BYTE_ORDER BYTE_ORDER
24
24
  #define __LITTLE_ENDIAN LITTLE_ENDIAN
25
- #define __BIG_ENDIAN BIG_ENDIAN
26
25
  #endif
27
26
 
28
- // define the macros IS_LITTLE_ENDIAN or IS_BIG_ENDIAN
27
+ // define the macro IS_LITTLE_ENDIAN
29
28
  // using the above endian definitions from endian.h if
30
29
  // endian.h was included
31
30
  #ifdef __BYTE_ORDER
@@ -33,19 +32,17 @@
33
32
  #define IS_LITTLE_ENDIAN
34
33
  #endif
35
34
 
36
- #if __BYTE_ORDER == __BIG_ENDIAN
37
- #define IS_BIG_ENDIAN
38
- #endif
39
-
40
35
  #else
41
36
 
42
37
  #if defined(__LITTLE_ENDIAN__)
43
38
  #define IS_LITTLE_ENDIAN
44
- #elif defined(__BIG_ENDIAN__)
45
- #define IS_BIG_ENDIAN
46
39
  #endif
47
40
  #endif // __BYTE_ORDER
48
41
 
42
+ #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
43
+ #define IS_LITTLE_ENDIAN
44
+ #endif
45
+
49
46
  // Enable little-endian optimization for x64 architecture on Windows.
50
47
  #if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
51
48
  #define IS_LITTLE_ENDIAN
@@ -69,8 +66,8 @@
69
66
  // On some platforms, like ARM, the copy functions can be more efficient
70
67
  // then a load and a store.
71
68
 
72
- #if defined(ARCH_PIII) || defined(ARCH_ATHLON) || \
73
- defined(ARCH_K8) || defined(_ARCH_PPC)
69
+ #if defined(ARCH_PIII) || \
70
+ defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC)
74
71
 
75
72
  // x86 and x86-64 can perform unaligned loads/stores directly;
76
73
  // modern PowerPC hardware can also do unaligned integer loads and stores;
@@ -142,10 +139,4 @@ inline void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
142
139
 
143
140
  #endif
144
141
 
145
- #ifdef BROTLI_ENCODE_DEBUG
146
- #define BROTLI_DCHECK(x) assert(x)
147
- #else
148
- #define BROTLI_DCHECK(x)
149
- #endif
150
-
151
142
  #endif // BROTLI_ENC_PORT_H_
@@ -15,18 +15,18 @@
15
15
 
16
16
  namespace brotli {
17
17
 
18
- static const int kNumInsertLenPrefixes = 24;
19
- static const int kNumCopyLenPrefixes = 24;
20
- static const int kNumCommandPrefixes = 704;
21
- static const int kNumBlockLenPrefixes = 26;
22
- static const int kNumDistanceShortCodes = 16;
23
- static const int kNumDistancePrefixes = 520;
18
+ static const uint32_t kNumInsertLenPrefixes = 24;
19
+ static const uint32_t kNumCopyLenPrefixes = 24;
20
+ static const uint32_t kNumCommandPrefixes = 704;
21
+ static const uint32_t kNumBlockLenPrefixes = 26;
22
+ static const uint32_t kNumDistanceShortCodes = 16;
23
+ static const uint32_t kNumDistancePrefixes = 520;
24
24
 
25
25
  // Represents the range of values belonging to a prefix code:
26
26
  // [offset, offset + 2^nbits)
27
27
  struct PrefixCodeRange {
28
- int offset;
29
- int nbits;
28
+ uint32_t offset;
29
+ uint32_t nbits;
30
30
  };
31
31
 
32
32
  static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
@@ -39,8 +39,8 @@ static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
39
39
  {8433, 13}, {16625, 24}
40
40
  };
41
41
 
42
- inline void GetBlockLengthPrefixCode(int len,
43
- int* code, int* n_extra, int* extra) {
42
+ inline void GetBlockLengthPrefixCode(uint32_t len, uint32_t* code,
43
+ uint32_t* n_extra, uint32_t* extra) {
44
44
  *code = 0;
45
45
  while (*code < 25 && len >= kBlockLengthPrefixCode[*code + 1].offset) {
46
46
  ++(*code);
@@ -49,9 +49,9 @@ inline void GetBlockLengthPrefixCode(int len,
49
49
  *extra = len - kBlockLengthPrefixCode[*code].offset;
50
50
  }
51
51
 
52
- inline void PrefixEncodeCopyDistance(int distance_code,
53
- int num_direct_codes,
54
- int postfix_bits,
52
+ inline void PrefixEncodeCopyDistance(size_t distance_code,
53
+ size_t num_direct_codes,
54
+ size_t postfix_bits,
55
55
  uint16_t* code,
56
56
  uint32_t* extra_bits) {
57
57
  if (distance_code < kNumDistanceShortCodes + num_direct_codes) {
@@ -59,18 +59,19 @@ inline void PrefixEncodeCopyDistance(int distance_code,
59
59
  *extra_bits = 0;
60
60
  return;
61
61
  }
62
- distance_code -= kNumDistanceShortCodes + num_direct_codes;
63
- distance_code += (1 << (postfix_bits + 2));
64
- int bucket = Log2Floor(distance_code) - 1;
65
- int postfix_mask = (1 << postfix_bits) - 1;
66
- int postfix = distance_code & postfix_mask;
67
- int prefix = (distance_code >> bucket) & 1;
68
- int offset = (2 + prefix) << bucket;
69
- int nbits = bucket - postfix_bits;
62
+ distance_code -= kNumDistanceShortCodes + num_direct_codes; /* >= 0 */
63
+ distance_code += (1u << (postfix_bits + 2u)); /* > 0 */
64
+ size_t bucket = Log2FloorNonZero(distance_code) - 1;
65
+ size_t postfix_mask = (1 << postfix_bits) - 1;
66
+ size_t postfix = distance_code & postfix_mask;
67
+ size_t prefix = (distance_code >> bucket) & 1;
68
+ size_t offset = (2 + prefix) << bucket;
69
+ size_t nbits = bucket - postfix_bits;
70
70
  *code = static_cast<uint16_t>(
71
71
  (kNumDistanceShortCodes + num_direct_codes +
72
72
  ((2 * (nbits - 1) + prefix) << postfix_bits) + postfix));
73
- *extra_bits = (nbits << 24) | ((distance_code - offset) >> postfix_bits);
73
+ *extra_bits = static_cast<uint32_t>(
74
+ (nbits << 24) | ((distance_code - offset) >> postfix_bits));
74
75
  }
75
76
 
76
77
  } // namespace brotli