brotli 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/main.yml +34 -0
  3. data/.github/workflows/publish.yml +34 -0
  4. data/Gemfile +6 -2
  5. data/Rakefile +18 -6
  6. data/bin/before_install.sh +9 -0
  7. data/brotli.gemspec +7 -13
  8. data/ext/brotli/brotli.c +209 -11
  9. data/ext/brotli/buffer.c +1 -7
  10. data/ext/brotli/buffer.h +1 -1
  11. data/ext/brotli/extconf.rb +45 -26
  12. data/lib/brotli/version.rb +1 -1
  13. data/smoke.sh +1 -1
  14. data/test/brotli_test.rb +104 -0
  15. data/test/brotli_writer_test.rb +36 -0
  16. data/test/test_helper.rb +8 -0
  17. data/vendor/brotli/c/common/constants.c +15 -0
  18. data/vendor/brotli/c/common/constants.h +149 -6
  19. data/vendor/brotli/c/{dec/context.h → common/context.c} +91 -186
  20. data/vendor/brotli/c/common/context.h +113 -0
  21. data/vendor/brotli/c/common/dictionary.bin +0 -0
  22. data/vendor/brotli/c/common/dictionary.bin.br +0 -0
  23. data/vendor/brotli/c/common/dictionary.c +11 -2
  24. data/vendor/brotli/c/common/dictionary.h +4 -4
  25. data/vendor/brotli/c/common/platform.c +22 -0
  26. data/vendor/brotli/c/common/platform.h +594 -0
  27. data/vendor/brotli/c/common/transform.c +291 -0
  28. data/vendor/brotli/c/common/transform.h +85 -0
  29. data/vendor/brotli/c/common/version.h +8 -1
  30. data/vendor/brotli/c/dec/bit_reader.c +29 -1
  31. data/vendor/brotli/c/dec/bit_reader.h +91 -100
  32. data/vendor/brotli/c/dec/decode.c +665 -437
  33. data/vendor/brotli/c/dec/huffman.c +65 -84
  34. data/vendor/brotli/c/dec/huffman.h +67 -14
  35. data/vendor/brotli/c/dec/prefix.h +1 -20
  36. data/vendor/brotli/c/dec/state.c +32 -45
  37. data/vendor/brotli/c/dec/state.h +173 -55
  38. data/vendor/brotli/c/enc/backward_references.c +27 -16
  39. data/vendor/brotli/c/enc/backward_references.h +7 -7
  40. data/vendor/brotli/c/enc/backward_references_hq.c +155 -116
  41. data/vendor/brotli/c/enc/backward_references_hq.h +22 -23
  42. data/vendor/brotli/c/enc/backward_references_inc.h +32 -22
  43. data/vendor/brotli/c/enc/bit_cost.c +1 -1
  44. data/vendor/brotli/c/enc/bit_cost.h +5 -5
  45. data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
  46. data/vendor/brotli/c/enc/block_splitter.c +5 -6
  47. data/vendor/brotli/c/enc/block_splitter.h +1 -1
  48. data/vendor/brotli/c/enc/block_splitter_inc.h +26 -17
  49. data/vendor/brotli/c/enc/brotli_bit_stream.c +107 -123
  50. data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
  51. data/vendor/brotli/c/enc/cluster.c +1 -1
  52. data/vendor/brotli/c/enc/cluster.h +1 -1
  53. data/vendor/brotli/c/enc/cluster_inc.h +6 -3
  54. data/vendor/brotli/c/enc/command.c +28 -0
  55. data/vendor/brotli/c/enc/command.h +52 -42
  56. data/vendor/brotli/c/enc/compress_fragment.c +21 -22
  57. data/vendor/brotli/c/enc/compress_fragment.h +1 -1
  58. data/vendor/brotli/c/enc/compress_fragment_two_pass.c +102 -69
  59. data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
  60. data/vendor/brotli/c/enc/dictionary_hash.c +1827 -1101
  61. data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
  62. data/vendor/brotli/c/enc/encode.c +358 -195
  63. data/vendor/brotli/c/enc/encoder_dict.c +33 -0
  64. data/vendor/brotli/c/enc/encoder_dict.h +43 -0
  65. data/vendor/brotli/c/enc/entropy_encode.c +16 -14
  66. data/vendor/brotli/c/enc/entropy_encode.h +7 -7
  67. data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
  68. data/vendor/brotli/c/enc/fast_log.c +105 -0
  69. data/vendor/brotli/c/enc/fast_log.h +20 -99
  70. data/vendor/brotli/c/enc/find_match_length.h +5 -6
  71. data/vendor/brotli/c/enc/hash.h +145 -103
  72. data/vendor/brotli/c/enc/hash_composite_inc.h +125 -0
  73. data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +93 -53
  74. data/vendor/brotli/c/enc/hash_longest_match64_inc.h +54 -53
  75. data/vendor/brotli/c/enc/hash_longest_match_inc.h +58 -54
  76. data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +95 -63
  77. data/vendor/brotli/c/enc/hash_rolling_inc.h +212 -0
  78. data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +46 -43
  79. data/vendor/brotli/c/enc/histogram.c +9 -6
  80. data/vendor/brotli/c/enc/histogram.h +6 -3
  81. data/vendor/brotli/c/enc/histogram_inc.h +1 -1
  82. data/vendor/brotli/c/enc/literal_cost.c +5 -5
  83. data/vendor/brotli/c/enc/literal_cost.h +2 -2
  84. data/vendor/brotli/c/enc/memory.c +5 -16
  85. data/vendor/brotli/c/enc/memory.h +52 -1
  86. data/vendor/brotli/c/enc/metablock.c +171 -36
  87. data/vendor/brotli/c/enc/metablock.h +13 -8
  88. data/vendor/brotli/c/enc/metablock_inc.h +2 -2
  89. data/vendor/brotli/c/enc/params.h +46 -0
  90. data/vendor/brotli/c/enc/prefix.h +3 -4
  91. data/vendor/brotli/c/enc/quality.h +29 -24
  92. data/vendor/brotli/c/enc/ringbuffer.h +19 -12
  93. data/vendor/brotli/c/enc/static_dict.c +49 -45
  94. data/vendor/brotli/c/enc/static_dict.h +4 -3
  95. data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
  96. data/vendor/brotli/c/enc/utf8_util.c +21 -21
  97. data/vendor/brotli/c/enc/utf8_util.h +1 -1
  98. data/vendor/brotli/c/enc/write_bits.h +35 -38
  99. data/vendor/brotli/c/include/brotli/decode.h +13 -8
  100. data/vendor/brotli/c/include/brotli/encode.h +54 -8
  101. data/vendor/brotli/c/include/brotli/port.h +225 -83
  102. data/vendor/brotli/c/include/brotli/types.h +0 -7
  103. metadata +28 -87
  104. data/.travis.yml +0 -30
  105. data/spec/brotli_spec.rb +0 -88
  106. data/spec/inflate_spec.rb +0 -75
  107. data/spec/spec_helper.rb +0 -4
  108. data/vendor/brotli/c/dec/port.h +0 -168
  109. data/vendor/brotli/c/dec/transform.h +0 -300
  110. data/vendor/brotli/c/enc/context.h +0 -184
  111. data/vendor/brotli/c/enc/port.h +0 -184
@@ -13,13 +13,14 @@
13
13
  #include <string.h> /* memcpy, memset */
14
14
 
15
15
  #include "../common/constants.h"
16
+ #include "../common/context.h"
17
+ #include "../common/platform.h"
16
18
  #include <brotli/types.h>
17
- #include "./context.h"
18
19
  #include "./entropy_encode.h"
19
20
  #include "./entropy_encode_static.h"
20
21
  #include "./fast_log.h"
22
+ #include "./histogram.h"
21
23
  #include "./memory.h"
22
- #include "./port.h"
23
24
  #include "./write_bits.h"
24
25
 
25
26
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -27,40 +28,24 @@ extern "C" {
27
28
  #endif
28
29
 
29
30
  #define MAX_HUFFMAN_TREE_SIZE (2 * BROTLI_NUM_COMMAND_SYMBOLS + 1)
30
- /* The size of Huffman dictionary for distances assuming that NPOSTFIX = 0 and
31
- NDIRECT = 0. */
32
- #define SIMPLE_DISTANCE_ALPHABET_SIZE (BROTLI_NUM_DISTANCE_SHORT_CODES + \
33
- (2 * BROTLI_MAX_DISTANCE_BITS))
34
- /* SIMPLE_DISTANCE_ALPHABET_SIZE == 64 */
35
- #define SIMPLE_DISTANCE_ALPHABET_BITS 6
36
-
37
- /* Represents the range of values belonging to a prefix code:
38
- [offset, offset + 2^nbits) */
39
- typedef struct PrefixCodeRange {
40
- uint32_t offset;
41
- uint32_t nbits;
42
- } PrefixCodeRange;
43
-
44
- static const PrefixCodeRange
45
- kBlockLengthPrefixCode[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = {
46
- { 1, 2}, { 5, 2}, { 9, 2}, {13, 2}, {17, 3}, { 25, 3}, { 33, 3},
47
- {41, 3}, {49, 4}, {65, 4}, {81, 4}, {97, 4}, {113, 5}, {145, 5},
48
- {177, 5}, { 209, 5}, { 241, 6}, { 305, 6}, { 369, 7}, { 497, 8},
49
- {753, 9}, {1265, 10}, {2289, 11}, {4337, 12}, {8433, 13}, {16625, 24}
50
- };
31
+ /* The maximum size of Huffman dictionary for distances assuming that
32
+ NPOSTFIX = 0 and NDIRECT = 0. */
33
+ #define MAX_SIMPLE_DISTANCE_ALPHABET_SIZE \
34
+ BROTLI_DISTANCE_ALPHABET_SIZE(0, 0, BROTLI_LARGE_MAX_DISTANCE_BITS)
35
+ /* MAX_SIMPLE_DISTANCE_ALPHABET_SIZE == 140 */
51
36
 
52
37
  static BROTLI_INLINE uint32_t BlockLengthPrefixCode(uint32_t len) {
53
38
  uint32_t code = (len >= 177) ? (len >= 753 ? 20 : 14) : (len >= 41 ? 7 : 0);
54
39
  while (code < (BROTLI_NUM_BLOCK_LEN_SYMBOLS - 1) &&
55
- len >= kBlockLengthPrefixCode[code + 1].offset) ++code;
40
+ len >= _kBrotliPrefixCodeRanges[code + 1].offset) ++code;
56
41
  return code;
57
42
  }
58
43
 
59
44
  static BROTLI_INLINE void GetBlockLengthPrefixCode(uint32_t len, size_t* code,
60
45
  uint32_t* n_extra, uint32_t* extra) {
61
46
  *code = BlockLengthPrefixCode(len);
62
- *n_extra = kBlockLengthPrefixCode[*code].nbits;
63
- *extra = len - kBlockLengthPrefixCode[*code].offset;
47
+ *n_extra = _kBrotliPrefixCodeRanges[*code].nbits;
48
+ *extra = len - _kBrotliPrefixCodeRanges[*code].offset;
64
49
  }
65
50
 
66
51
  typedef struct BlockTypeCodeCalculator {
@@ -89,9 +74,9 @@ static void BrotliEncodeMlen(size_t length, uint64_t* bits,
89
74
  size_t* numbits, uint64_t* nibblesbits) {
90
75
  size_t lg = (length == 1) ? 1 : Log2FloorNonZero((uint32_t)(length - 1)) + 1;
91
76
  size_t mnibbles = (lg < 16 ? 16 : (lg + 3)) / 4;
92
- assert(length > 0);
93
- assert(length <= (1 << 24));
94
- assert(lg <= 24);
77
+ BROTLI_DCHECK(length > 0);
78
+ BROTLI_DCHECK(length <= (1 << 24));
79
+ BROTLI_DCHECK(lg <= 24);
95
80
  *nibblesbits = mnibbles - 4;
96
81
  *numbits = mnibbles * 4;
97
82
  *bits = length - 1;
@@ -258,7 +243,7 @@ static void StoreSimpleHuffmanTree(const uint8_t* depths,
258
243
  size_t symbols[4],
259
244
  size_t num_symbols,
260
245
  size_t max_bits,
261
- size_t *storage_ix, uint8_t *storage) {
246
+ size_t* storage_ix, uint8_t* storage) {
262
247
  /* value of 1 indicates a simple Huffman code */
263
248
  BrotliWriteBits(2, 1, storage_ix, storage);
264
249
  BrotliWriteBits(2, num_symbols - 1, storage_ix, storage); /* NSYM - 1 */
@@ -297,7 +282,7 @@ static void StoreSimpleHuffmanTree(const uint8_t* depths,
297
282
  depths = symbol depths */
298
283
  void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
299
284
  HuffmanTree* tree,
300
- size_t *storage_ix, uint8_t *storage) {
285
+ size_t* storage_ix, uint8_t* storage) {
301
286
  /* Write the Huffman tree into the brotli-representation.
302
287
  The command alphabet is the largest, so this allocation will fit all
303
288
  alphabets. */
@@ -311,7 +296,7 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
311
296
  int num_codes = 0;
312
297
  size_t code = 0;
313
298
 
314
- assert(num <= BROTLI_NUM_COMMAND_SYMBOLS);
299
+ BROTLI_DCHECK(num <= BROTLI_NUM_COMMAND_SYMBOLS);
315
300
 
316
301
  BrotliWriteHuffmanTree(depths, num, &huffman_tree_size, huffman_tree,
317
302
  huffman_tree_extra_bits);
@@ -360,8 +345,9 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
360
345
 
361
346
  /* Builds a Huffman tree from histogram[0:length] into depth[0:length] and
362
347
  bits[0:length] and stores the encoded tree to the bit stream. */
363
- static void BuildAndStoreHuffmanTree(const uint32_t *histogram,
364
- const size_t length,
348
+ static void BuildAndStoreHuffmanTree(const uint32_t* histogram,
349
+ const size_t histogram_length,
350
+ const size_t alphabet_size,
365
351
  HuffmanTree* tree,
366
352
  uint8_t* depth,
367
353
  uint16_t* bits,
@@ -371,7 +357,7 @@ static void BuildAndStoreHuffmanTree(const uint32_t *histogram,
371
357
  size_t s4[4] = { 0 };
372
358
  size_t i;
373
359
  size_t max_bits = 0;
374
- for (i = 0; i < length; i++) {
360
+ for (i = 0; i < histogram_length; i++) {
375
361
  if (histogram[i]) {
376
362
  if (count < 4) {
377
363
  s4[count] = i;
@@ -383,7 +369,7 @@ static void BuildAndStoreHuffmanTree(const uint32_t *histogram,
383
369
  }
384
370
 
385
371
  {
386
- size_t max_bits_counter = length - 1;
372
+ size_t max_bits_counter = alphabet_size - 1;
387
373
  while (max_bits_counter) {
388
374
  max_bits_counter >>= 1;
389
375
  ++max_bits;
@@ -398,14 +384,14 @@ static void BuildAndStoreHuffmanTree(const uint32_t *histogram,
398
384
  return;
399
385
  }
400
386
 
401
- memset(depth, 0, length * sizeof(depth[0]));
402
- BrotliCreateHuffmanTree(histogram, length, 15, tree, depth);
403
- BrotliConvertBitDepthsToSymbols(depth, length, bits);
387
+ memset(depth, 0, histogram_length * sizeof(depth[0]));
388
+ BrotliCreateHuffmanTree(histogram, histogram_length, 15, tree, depth);
389
+ BrotliConvertBitDepthsToSymbols(depth, histogram_length, bits);
404
390
 
405
391
  if (count <= 4) {
406
392
  StoreSimpleHuffmanTree(depth, s4, count, max_bits, storage_ix, storage);
407
393
  } else {
408
- BrotliStoreHuffmanTree(depth, length, tree, storage_ix, storage);
394
+ BrotliStoreHuffmanTree(depth, histogram_length, tree, storage_ix, storage);
409
395
  }
410
396
  }
411
397
 
@@ -449,7 +435,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
449
435
  const size_t max_tree_size = 2 * length + 1;
450
436
  HuffmanTree* tree = BROTLI_ALLOC(m, HuffmanTree, max_tree_size);
451
437
  uint32_t count_limit;
452
- if (BROTLI_IS_OOM(m)) return;
438
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
453
439
  for (count_limit = 1; ; count_limit *= 2) {
454
440
  HuffmanTree* node = tree;
455
441
  size_t l;
@@ -619,7 +605,7 @@ static void MoveToFrontTransform(const uint32_t* BROTLI_RESTRICT v_in,
619
605
  for (i = 1; i < v_size; ++i) {
620
606
  if (v_in[i] > max_value) max_value = v_in[i];
621
607
  }
622
- assert(max_value < 256u);
608
+ BROTLI_DCHECK(max_value < 256u);
623
609
  for (i = 0; i <= max_value; ++i) {
624
610
  mtf[i] = (uint8_t)i;
625
611
  }
@@ -627,7 +613,7 @@ static void MoveToFrontTransform(const uint32_t* BROTLI_RESTRICT v_in,
627
613
  size_t mtf_size = max_value + 1;
628
614
  for (i = 0; i < v_size; ++i) {
629
615
  size_t index = IndexOf(mtf, mtf_size, (uint8_t)v_in[i]);
630
- assert(index < mtf_size);
616
+ BROTLI_DCHECK(index < mtf_size);
631
617
  v_out[i] = (uint32_t)index;
632
618
  MoveToFront(mtf, index);
633
619
  }
@@ -659,7 +645,7 @@ static void RunLengthCodeZeros(const size_t in_size,
659
645
  *max_run_length_prefix = max_prefix;
660
646
  *out_size = 0;
661
647
  for (i = 0; i < in_size;) {
662
- assert(*out_size <= i);
648
+ BROTLI_DCHECK(*out_size <= i);
663
649
  if (v[i] != 0) {
664
650
  v[*out_size] = v[i] + *max_run_length_prefix;
665
651
  ++i;
@@ -713,7 +699,7 @@ static void EncodeContextMap(MemoryManager* m,
713
699
  }
714
700
 
715
701
  rle_symbols = BROTLI_ALLOC(m, uint32_t, context_map_size);
716
- if (BROTLI_IS_OOM(m)) return;
702
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(rle_symbols)) return;
717
703
  MoveToFrontTransform(context_map, context_map_size, rle_symbols);
718
704
  RunLengthCodeZeros(context_map_size, rle_symbols,
719
705
  &num_rle_symbols, &max_run_length_prefix);
@@ -729,6 +715,7 @@ static void EncodeContextMap(MemoryManager* m,
729
715
  }
730
716
  }
731
717
  BuildAndStoreHuffmanTree(histogram, num_clusters + max_run_length_prefix,
718
+ num_clusters + max_run_length_prefix,
732
719
  tree, depths, bits, storage_ix, storage);
733
720
  for (i = 0; i < num_rle_symbols; ++i) {
734
721
  const uint32_t rle_symbol = rle_symbols[i] & kSymbolMask;
@@ -788,10 +775,11 @@ static void BuildAndStoreBlockSplitCode(const uint8_t* types,
788
775
  }
789
776
  StoreVarLenUint8(num_types - 1, storage_ix, storage);
790
777
  if (num_types > 1) { /* TODO: else? could StoreBlockSwitch occur? */
791
- BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, tree,
778
+ BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, num_types + 2, tree,
792
779
  &code->type_depths[0], &code->type_bits[0],
793
780
  storage_ix, storage);
794
781
  BuildAndStoreHuffmanTree(&length_histo[0], BROTLI_NUM_BLOCK_LEN_SYMBOLS,
782
+ BROTLI_NUM_BLOCK_LEN_SYMBOLS,
795
783
  tree, &code->length_depths[0],
796
784
  &code->length_bits[0], storage_ix, storage);
797
785
  StoreBlockSwitch(code, lengths[0], types[0], 1, storage_ix, storage);
@@ -822,8 +810,8 @@ static void StoreTrivialContextMap(size_t num_types,
822
810
  for (i = context_bits; i < alphabet_size; ++i) {
823
811
  histogram[i] = 1;
824
812
  }
825
- BuildAndStoreHuffmanTree(histogram, alphabet_size, tree,
826
- depths, bits, storage_ix, storage);
813
+ BuildAndStoreHuffmanTree(histogram, alphabet_size, alphabet_size,
814
+ tree, depths, bits, storage_ix, storage);
827
815
  for (i = 0; i < num_types; ++i) {
828
816
  size_t code = (i == 0 ? 0 : i + context_bits - 1);
829
817
  BrotliWriteBits(depths[code], bits[code], storage_ix, storage);
@@ -838,7 +826,7 @@ static void StoreTrivialContextMap(size_t num_types,
838
826
 
839
827
  /* Manages the encoding of one block category (literal, command or distance). */
840
828
  typedef struct BlockEncoder {
841
- size_t alphabet_size_;
829
+ size_t histogram_length_;
842
830
  size_t num_block_types_;
843
831
  const uint8_t* block_types_; /* Not owned. */
844
832
  const uint32_t* block_lengths_; /* Not owned. */
@@ -851,10 +839,10 @@ typedef struct BlockEncoder {
851
839
  uint16_t* bits_;
852
840
  } BlockEncoder;
853
841
 
854
- static void InitBlockEncoder(BlockEncoder* self, size_t alphabet_size,
842
+ static void InitBlockEncoder(BlockEncoder* self, size_t histogram_length,
855
843
  size_t num_block_types, const uint8_t* block_types,
856
844
  const uint32_t* block_lengths, const size_t num_blocks) {
857
- self->alphabet_size_ = alphabet_size;
845
+ self->histogram_length_ = histogram_length;
858
846
  self->num_block_types_ = num_block_types;
859
847
  self->block_types_ = block_types;
860
848
  self->block_lengths_ = block_lengths;
@@ -890,7 +878,7 @@ static void StoreSymbol(BlockEncoder* self, size_t symbol, size_t* storage_ix,
890
878
  uint32_t block_len = self->block_lengths_[block_ix];
891
879
  uint8_t block_type = self->block_types_[block_ix];
892
880
  self->block_len_ = block_len;
893
- self->entropy_ix_ = block_type * self->alphabet_size_;
881
+ self->entropy_ix_ = block_type * self->histogram_length_;
894
882
  StoreBlockSwitch(&self->block_split_code_, block_len, block_type, 0,
895
883
  storage_ix, storage);
896
884
  }
@@ -919,7 +907,7 @@ static void StoreSymbolWithContext(BlockEncoder* self, size_t symbol,
919
907
  --self->block_len_;
920
908
  {
921
909
  size_t histo_ix = context_map[self->entropy_ix_ + context];
922
- size_t ix = histo_ix * self->alphabet_size_ + symbol;
910
+ size_t ix = histo_ix * self->histogram_length_ + symbol;
923
911
  BrotliWriteBits(self->depths_[ix], self->bits_[ix], storage_ix, storage);
924
912
  }
925
913
  }
@@ -945,42 +933,36 @@ static void JumpToByteBoundary(size_t* storage_ix, uint8_t* storage) {
945
933
  }
946
934
 
947
935
  void BrotliStoreMetaBlock(MemoryManager* m,
948
- const uint8_t* input,
949
- size_t start_pos,
950
- size_t length,
951
- size_t mask,
952
- uint8_t prev_byte,
953
- uint8_t prev_byte2,
954
- BROTLI_BOOL is_last,
955
- uint32_t num_direct_distance_codes,
956
- uint32_t distance_postfix_bits,
957
- ContextType literal_context_mode,
958
- const Command *commands,
959
- size_t n_commands,
960
- const MetaBlockSplit* mb,
961
- size_t *storage_ix,
962
- uint8_t *storage) {
936
+ const uint8_t* input, size_t start_pos, size_t length, size_t mask,
937
+ uint8_t prev_byte, uint8_t prev_byte2, BROTLI_BOOL is_last,
938
+ const BrotliEncoderParams* params, ContextType literal_context_mode,
939
+ const Command* commands, size_t n_commands, const MetaBlockSplit* mb,
940
+ size_t* storage_ix, uint8_t* storage) {
941
+
963
942
  size_t pos = start_pos;
964
943
  size_t i;
965
- size_t num_distance_codes =
966
- BROTLI_NUM_DISTANCE_SHORT_CODES + num_direct_distance_codes +
967
- (48u << distance_postfix_bits);
944
+ uint32_t num_distance_symbols = params->dist.alphabet_size_max;
945
+ uint32_t num_effective_distance_symbols = params->dist.alphabet_size_limit;
968
946
  HuffmanTree* tree;
947
+ ContextLut literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
969
948
  BlockEncoder literal_enc;
970
949
  BlockEncoder command_enc;
971
950
  BlockEncoder distance_enc;
951
+ const BrotliDistanceParams* dist = &params->dist;
952
+ BROTLI_DCHECK(
953
+ num_effective_distance_symbols <= BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS);
972
954
 
973
955
  StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
974
956
 
975
957
  tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
976
- if (BROTLI_IS_OOM(m)) return;
977
- InitBlockEncoder(&literal_enc, 256, mb->literal_split.num_types,
978
- mb->literal_split.types, mb->literal_split.lengths,
979
- mb->literal_split.num_blocks);
958
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
959
+ InitBlockEncoder(&literal_enc, BROTLI_NUM_LITERAL_SYMBOLS,
960
+ mb->literal_split.num_types, mb->literal_split.types,
961
+ mb->literal_split.lengths, mb->literal_split.num_blocks);
980
962
  InitBlockEncoder(&command_enc, BROTLI_NUM_COMMAND_SYMBOLS,
981
963
  mb->command_split.num_types, mb->command_split.types,
982
964
  mb->command_split.lengths, mb->command_split.num_blocks);
983
- InitBlockEncoder(&distance_enc, num_distance_codes,
965
+ InitBlockEncoder(&distance_enc, num_effective_distance_symbols,
984
966
  mb->distance_split.num_types, mb->distance_split.types,
985
967
  mb->distance_split.lengths, mb->distance_split.num_blocks);
986
968
 
@@ -989,9 +971,10 @@ void BrotliStoreMetaBlock(MemoryManager* m,
989
971
  BuildAndStoreBlockSwitchEntropyCodes(
990
972
  &distance_enc, tree, storage_ix, storage);
991
973
 
992
- BrotliWriteBits(2, distance_postfix_bits, storage_ix, storage);
993
- BrotliWriteBits(4, num_direct_distance_codes >> distance_postfix_bits,
994
- storage_ix, storage);
974
+ BrotliWriteBits(2, dist->distance_postfix_bits, storage_ix, storage);
975
+ BrotliWriteBits(
976
+ 4, dist->num_direct_distance_codes >> dist->distance_postfix_bits,
977
+ storage_ix, storage);
995
978
  for (i = 0; i < mb->literal_split.num_types; ++i) {
996
979
  BrotliWriteBits(2, literal_context_mode, storage_ix, storage);
997
980
  }
@@ -1017,13 +1000,16 @@ void BrotliStoreMetaBlock(MemoryManager* m,
1017
1000
  }
1018
1001
 
1019
1002
  BuildAndStoreEntropyCodesLiteral(m, &literal_enc, mb->literal_histograms,
1020
- mb->literal_histograms_size, tree, storage_ix, storage);
1003
+ mb->literal_histograms_size, BROTLI_NUM_LITERAL_SYMBOLS, tree,
1004
+ storage_ix, storage);
1021
1005
  if (BROTLI_IS_OOM(m)) return;
1022
1006
  BuildAndStoreEntropyCodesCommand(m, &command_enc, mb->command_histograms,
1023
- mb->command_histograms_size, tree, storage_ix, storage);
1007
+ mb->command_histograms_size, BROTLI_NUM_COMMAND_SYMBOLS, tree,
1008
+ storage_ix, storage);
1024
1009
  if (BROTLI_IS_OOM(m)) return;
1025
1010
  BuildAndStoreEntropyCodesDistance(m, &distance_enc, mb->distance_histograms,
1026
- mb->distance_histograms_size, tree, storage_ix, storage);
1011
+ mb->distance_histograms_size, num_distance_symbols, tree,
1012
+ storage_ix, storage);
1027
1013
  if (BROTLI_IS_OOM(m)) return;
1028
1014
  BROTLI_FREE(m, tree);
1029
1015
 
@@ -1041,7 +1027,8 @@ void BrotliStoreMetaBlock(MemoryManager* m,
1041
1027
  } else {
1042
1028
  size_t j;
1043
1029
  for (j = cmd.insert_len_; j != 0; --j) {
1044
- size_t context = Context(prev_byte, prev_byte2, literal_context_mode);
1030
+ size_t context =
1031
+ BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut);
1045
1032
  uint8_t literal = input[pos & mask];
1046
1033
  StoreSymbolWithContext(&literal_enc, literal, context,
1047
1034
  mb->literal_context_map, storage_ix, storage,
@@ -1056,9 +1043,9 @@ void BrotliStoreMetaBlock(MemoryManager* m,
1056
1043
  prev_byte2 = input[(pos - 2) & mask];
1057
1044
  prev_byte = input[(pos - 1) & mask];
1058
1045
  if (cmd.cmd_prefix_ >= 128) {
1059
- size_t dist_code = cmd.dist_prefix_;
1060
- uint32_t distnumextra = cmd.dist_extra_ >> 24;
1061
- uint64_t distextra = cmd.dist_extra_ & 0xffffff;
1046
+ size_t dist_code = cmd.dist_prefix_ & 0x3FF;
1047
+ uint32_t distnumextra = cmd.dist_prefix_ >> 10;
1048
+ uint64_t distextra = cmd.dist_extra_;
1062
1049
  if (mb->distance_context_map_size == 0) {
1063
1050
  StoreSymbol(&distance_enc, dist_code, storage_ix, storage);
1064
1051
  } else {
@@ -1082,7 +1069,7 @@ void BrotliStoreMetaBlock(MemoryManager* m,
1082
1069
  static void BuildHistograms(const uint8_t* input,
1083
1070
  size_t start_pos,
1084
1071
  size_t mask,
1085
- const Command *commands,
1072
+ const Command* commands,
1086
1073
  size_t n_commands,
1087
1074
  HistogramLiteral* lit_histo,
1088
1075
  HistogramCommand* cmd_histo,
@@ -1099,7 +1086,7 @@ static void BuildHistograms(const uint8_t* input,
1099
1086
  }
1100
1087
  pos += CommandCopyLen(&cmd);
1101
1088
  if (CommandCopyLen(&cmd) && cmd.cmd_prefix_ >= 128) {
1102
- HistogramAddDistance(dist_histo, cmd.dist_prefix_);
1089
+ HistogramAddDistance(dist_histo, cmd.dist_prefix_ & 0x3FF);
1103
1090
  }
1104
1091
  }
1105
1092
  }
@@ -1107,7 +1094,7 @@ static void BuildHistograms(const uint8_t* input,
1107
1094
  static void StoreDataWithHuffmanCodes(const uint8_t* input,
1108
1095
  size_t start_pos,
1109
1096
  size_t mask,
1110
- const Command *commands,
1097
+ const Command* commands,
1111
1098
  size_t n_commands,
1112
1099
  const uint8_t* lit_depth,
1113
1100
  const uint16_t* lit_bits,
@@ -1134,9 +1121,9 @@ static void StoreDataWithHuffmanCodes(const uint8_t* input,
1134
1121
  }
1135
1122
  pos += CommandCopyLen(&cmd);
1136
1123
  if (CommandCopyLen(&cmd) && cmd.cmd_prefix_ >= 128) {
1137
- const size_t dist_code = cmd.dist_prefix_;
1138
- const uint32_t distnumextra = cmd.dist_extra_ >> 24;
1139
- const uint32_t distextra = cmd.dist_extra_ & 0xffffff;
1124
+ const size_t dist_code = cmd.dist_prefix_ & 0x3FF;
1125
+ const uint32_t distnumextra = cmd.dist_prefix_ >> 10;
1126
+ const uint32_t distextra = cmd.dist_extra_;
1140
1127
  BrotliWriteBits(dist_depth[dist_code], dist_bits[dist_code],
1141
1128
  storage_ix, storage);
1142
1129
  BrotliWriteBits(distnumextra, distextra, storage_ix, storage);
@@ -1145,15 +1132,10 @@ static void StoreDataWithHuffmanCodes(const uint8_t* input,
1145
1132
  }
1146
1133
 
1147
1134
  void BrotliStoreMetaBlockTrivial(MemoryManager* m,
1148
- const uint8_t* input,
1149
- size_t start_pos,
1150
- size_t length,
1151
- size_t mask,
1152
- BROTLI_BOOL is_last,
1153
- const Command *commands,
1154
- size_t n_commands,
1155
- size_t *storage_ix,
1156
- uint8_t *storage) {
1135
+ const uint8_t* input, size_t start_pos, size_t length, size_t mask,
1136
+ BROTLI_BOOL is_last, const BrotliEncoderParams* params,
1137
+ const Command* commands, size_t n_commands,
1138
+ size_t* storage_ix, uint8_t* storage) {
1157
1139
  HistogramLiteral lit_histo;
1158
1140
  HistogramCommand cmd_histo;
1159
1141
  HistogramDistance dist_histo;
@@ -1161,9 +1143,10 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
1161
1143
  uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
1162
1144
  uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
1163
1145
  uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
1164
- uint8_t dist_depth[SIMPLE_DISTANCE_ALPHABET_SIZE];
1165
- uint16_t dist_bits[SIMPLE_DISTANCE_ALPHABET_SIZE];
1146
+ uint8_t dist_depth[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
1147
+ uint16_t dist_bits[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
1166
1148
  HuffmanTree* tree;
1149
+ uint32_t num_distance_symbols = params->dist.alphabet_size_max;
1167
1150
 
1168
1151
  StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
1169
1152
 
@@ -1177,15 +1160,17 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
1177
1160
  BrotliWriteBits(13, 0, storage_ix, storage);
1178
1161
 
1179
1162
  tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
1180
- if (BROTLI_IS_OOM(m)) return;
1181
- BuildAndStoreHuffmanTree(lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS, tree,
1163
+ if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
1164
+ BuildAndStoreHuffmanTree(lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS,
1165
+ BROTLI_NUM_LITERAL_SYMBOLS, tree,
1182
1166
  lit_depth, lit_bits,
1183
1167
  storage_ix, storage);
1184
- BuildAndStoreHuffmanTree(cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS, tree,
1168
+ BuildAndStoreHuffmanTree(cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS,
1169
+ BROTLI_NUM_COMMAND_SYMBOLS, tree,
1185
1170
  cmd_depth, cmd_bits,
1186
1171
  storage_ix, storage);
1187
- BuildAndStoreHuffmanTree(dist_histo.data_, SIMPLE_DISTANCE_ALPHABET_SIZE,
1188
- tree,
1172
+ BuildAndStoreHuffmanTree(dist_histo.data_, MAX_SIMPLE_DISTANCE_ALPHABET_SIZE,
1173
+ num_distance_symbols, tree,
1189
1174
  dist_depth, dist_bits,
1190
1175
  storage_ix, storage);
1191
1176
  BROTLI_FREE(m, tree);
@@ -1200,15 +1185,14 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
1200
1185
  }
1201
1186
 
1202
1187
  void BrotliStoreMetaBlockFast(MemoryManager* m,
1203
- const uint8_t* input,
1204
- size_t start_pos,
1205
- size_t length,
1206
- size_t mask,
1207
- BROTLI_BOOL is_last,
1208
- const Command *commands,
1209
- size_t n_commands,
1210
- size_t *storage_ix,
1211
- uint8_t *storage) {
1188
+ const uint8_t* input, size_t start_pos, size_t length, size_t mask,
1189
+ BROTLI_BOOL is_last, const BrotliEncoderParams* params,
1190
+ const Command* commands, size_t n_commands,
1191
+ size_t* storage_ix, uint8_t* storage) {
1192
+ uint32_t num_distance_symbols = params->dist.alphabet_size_max;
1193
+ uint32_t distance_alphabet_bits =
1194
+ Log2FloorNonZero(num_distance_symbols - 1) + 1;
1195
+
1212
1196
  StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
1213
1197
 
1214
1198
  BrotliWriteBits(13, 0, storage_ix, storage);
@@ -1252,8 +1236,8 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
1252
1236
  uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
1253
1237
  uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
1254
1238
  uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
1255
- uint8_t dist_depth[SIMPLE_DISTANCE_ALPHABET_SIZE];
1256
- uint16_t dist_bits[SIMPLE_DISTANCE_ALPHABET_SIZE];
1239
+ uint8_t dist_depth[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
1240
+ uint16_t dist_bits[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
1257
1241
  HistogramClearLiteral(&lit_histo);
1258
1242
  HistogramClearCommand(&cmd_histo);
1259
1243
  HistogramClearDistance(&dist_histo);
@@ -1274,7 +1258,7 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
1274
1258
  BrotliBuildAndStoreHuffmanTreeFast(m, dist_histo.data_,
1275
1259
  dist_histo.total_count_,
1276
1260
  /* max_bits = */
1277
- SIMPLE_DISTANCE_ALPHABET_BITS,
1261
+ distance_alphabet_bits,
1278
1262
  dist_depth, dist_bits,
1279
1263
  storage_ix, storage);
1280
1264
  if (BROTLI_IS_OOM(m)) return;
@@ -1293,11 +1277,11 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
1293
1277
  /* This is for storing uncompressed blocks (simple raw storage of
1294
1278
  bytes-as-bytes). */
1295
1279
  void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
1296
- const uint8_t * BROTLI_RESTRICT input,
1280
+ const uint8_t* BROTLI_RESTRICT input,
1297
1281
  size_t position, size_t mask,
1298
1282
  size_t len,
1299
- size_t * BROTLI_RESTRICT storage_ix,
1300
- uint8_t * BROTLI_RESTRICT storage) {
1283
+ size_t* BROTLI_RESTRICT storage_ix,
1284
+ uint8_t* BROTLI_RESTRICT storage) {
1301
1285
  size_t masked_pos = position & mask;
1302
1286
  BrotliStoreUncompressedMetaBlockHeader(len, storage_ix, storage);
1303
1287
  JumpToByteBoundary(storage_ix, storage);