brotli 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +7 -3
  4. data/brotli.gemspec +1 -1
  5. data/ext/brotli/brotli.c +4 -4
  6. data/ext/brotli/brotli.h +2 -2
  7. data/ext/brotli/extconf.rb +9 -16
  8. data/lib/brotli/version.rb +1 -1
  9. data/vendor/brotli/{common → c/common}/constants.h +11 -1
  10. data/vendor/brotli/c/common/dictionary.bin +432 -0
  11. data/vendor/brotli/c/common/dictionary.c +5905 -0
  12. data/vendor/brotli/c/common/dictionary.h +64 -0
  13. data/vendor/brotli/c/common/version.h +19 -0
  14. data/vendor/brotli/{dec → c/dec}/bit_reader.c +2 -2
  15. data/vendor/brotli/{dec → c/dec}/bit_reader.h +11 -34
  16. data/vendor/brotli/{dec → c/dec}/context.h +1 -1
  17. data/vendor/brotli/{dec → c/dec}/decode.c +389 -356
  18. data/vendor/brotli/{dec → c/dec}/huffman.c +24 -23
  19. data/vendor/brotli/{dec → c/dec}/huffman.h +1 -1
  20. data/vendor/brotli/{dec → c/dec}/port.h +19 -10
  21. data/vendor/brotli/{dec → c/dec}/prefix.h +1 -1
  22. data/vendor/brotli/{dec → c/dec}/state.c +23 -19
  23. data/vendor/brotli/{dec → c/dec}/state.h +18 -17
  24. data/vendor/brotli/{dec → c/dec}/transform.h +2 -2
  25. data/vendor/brotli/c/enc/backward_references.c +134 -0
  26. data/vendor/brotli/c/enc/backward_references.h +39 -0
  27. data/vendor/brotli/{enc/backward_references.c → c/enc/backward_references_hq.c} +144 -232
  28. data/vendor/brotli/{enc/backward_references.h → c/enc/backward_references_hq.h} +28 -31
  29. data/vendor/brotli/{enc → c/enc}/backward_references_inc.h +37 -31
  30. data/vendor/brotli/{enc → c/enc}/bit_cost.c +1 -1
  31. data/vendor/brotli/{enc → c/enc}/bit_cost.h +1 -1
  32. data/vendor/brotli/{enc → c/enc}/bit_cost_inc.h +0 -0
  33. data/vendor/brotli/{enc → c/enc}/block_encoder_inc.h +0 -0
  34. data/vendor/brotli/{enc → c/enc}/block_splitter.c +2 -4
  35. data/vendor/brotli/{enc → c/enc}/block_splitter.h +1 -1
  36. data/vendor/brotli/{enc → c/enc}/block_splitter_inc.h +6 -7
  37. data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.c +22 -26
  38. data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.h +1 -5
  39. data/vendor/brotli/{enc → c/enc}/cluster.c +1 -1
  40. data/vendor/brotli/{enc → c/enc}/cluster.h +1 -1
  41. data/vendor/brotli/{enc → c/enc}/cluster_inc.h +2 -0
  42. data/vendor/brotli/{enc → c/enc}/command.h +34 -17
  43. data/vendor/brotli/{enc → c/enc}/compress_fragment.c +97 -53
  44. data/vendor/brotli/{enc → c/enc}/compress_fragment.h +5 -2
  45. data/vendor/brotli/{enc → c/enc}/compress_fragment_two_pass.c +106 -51
  46. data/vendor/brotli/{enc → c/enc}/compress_fragment_two_pass.h +5 -2
  47. data/vendor/brotli/{enc → c/enc}/context.h +3 -3
  48. data/vendor/brotli/c/enc/dictionary_hash.c +1120 -0
  49. data/vendor/brotli/c/enc/dictionary_hash.h +24 -0
  50. data/vendor/brotli/{enc → c/enc}/encode.c +442 -240
  51. data/vendor/brotli/{enc → c/enc}/entropy_encode.c +9 -9
  52. data/vendor/brotli/{enc → c/enc}/entropy_encode.h +4 -4
  53. data/vendor/brotli/{enc → c/enc}/entropy_encode_static.h +4 -4
  54. data/vendor/brotli/{enc → c/enc}/fast_log.h +3 -3
  55. data/vendor/brotli/{enc → c/enc}/find_match_length.h +8 -8
  56. data/vendor/brotli/c/enc/hash.h +446 -0
  57. data/vendor/brotli/{enc → c/enc}/hash_forgetful_chain_inc.h +72 -68
  58. data/vendor/brotli/c/enc/hash_longest_match64_inc.h +266 -0
  59. data/vendor/brotli/c/enc/hash_longest_match_inc.h +258 -0
  60. data/vendor/brotli/{enc → c/enc}/hash_longest_match_quickly_inc.h +81 -77
  61. data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +326 -0
  62. data/vendor/brotli/{enc → c/enc}/histogram.c +4 -2
  63. data/vendor/brotli/{enc → c/enc}/histogram.h +1 -1
  64. data/vendor/brotli/{enc → c/enc}/histogram_inc.h +0 -0
  65. data/vendor/brotli/{enc → c/enc}/literal_cost.c +4 -7
  66. data/vendor/brotli/{enc → c/enc}/literal_cost.h +2 -2
  67. data/vendor/brotli/{enc → c/enc}/memory.c +1 -1
  68. data/vendor/brotli/{enc → c/enc}/memory.h +3 -2
  69. data/vendor/brotli/{enc → c/enc}/metablock.c +136 -123
  70. data/vendor/brotli/{enc → c/enc}/metablock.h +2 -12
  71. data/vendor/brotli/{enc → c/enc}/metablock_inc.h +0 -0
  72. data/vendor/brotli/{enc → c/enc}/port.h +49 -33
  73. data/vendor/brotli/{enc → c/enc}/prefix.h +4 -2
  74. data/vendor/brotli/{enc → c/enc}/quality.h +47 -17
  75. data/vendor/brotli/{enc → c/enc}/ringbuffer.h +6 -6
  76. data/vendor/brotli/{enc → c/enc}/static_dict.c +26 -22
  77. data/vendor/brotli/{enc → c/enc}/static_dict.h +3 -1
  78. data/vendor/brotli/c/enc/static_dict_lut.h +5864 -0
  79. data/vendor/brotli/{enc → c/enc}/utf8_util.c +1 -1
  80. data/vendor/brotli/{enc → c/enc}/utf8_util.h +2 -2
  81. data/vendor/brotli/{enc → c/enc}/write_bits.h +3 -3
  82. data/vendor/brotli/c/include/brotli/decode.h +339 -0
  83. data/vendor/brotli/c/include/brotli/encode.h +402 -0
  84. data/vendor/brotli/c/include/brotli/port.h +146 -0
  85. data/vendor/brotli/c/include/brotli/types.h +90 -0
  86. metadata +80 -79
  87. data/vendor/brotli/common/dictionary.c +0 -9474
  88. data/vendor/brotli/common/dictionary.h +0 -29
  89. data/vendor/brotli/common/port.h +0 -107
  90. data/vendor/brotli/common/types.h +0 -58
  91. data/vendor/brotli/dec/decode.h +0 -188
  92. data/vendor/brotli/enc/compressor.cc +0 -139
  93. data/vendor/brotli/enc/compressor.h +0 -161
  94. data/vendor/brotli/enc/dictionary_hash.h +0 -4121
  95. data/vendor/brotli/enc/encode.h +0 -221
  96. data/vendor/brotli/enc/encode_parallel.cc +0 -289
  97. data/vendor/brotli/enc/encode_parallel.h +0 -27
  98. data/vendor/brotli/enc/hash.h +0 -717
  99. data/vendor/brotli/enc/hash_longest_match_inc.h +0 -241
  100. data/vendor/brotli/enc/static_dict_lut.h +0 -11241
  101. data/vendor/brotli/enc/streams.cc +0 -114
  102. data/vendor/brotli/enc/streams.h +0 -121
@@ -11,7 +11,7 @@
11
11
  #include <string.h> /* memset */
12
12
 
13
13
  #include "../common/constants.h"
14
- #include "../common/types.h"
14
+ #include <brotli/types.h>
15
15
  #include "./port.h"
16
16
 
17
17
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -246,7 +246,7 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
246
246
  size_t limit;
247
247
  size_t sum;
248
248
  const size_t streak_limit = 1240;
249
- /* Let's make the Huffman code more compatible with rle encoding. */
249
+ /* Let's make the Huffman code more compatible with RLE encoding. */
250
250
  size_t i;
251
251
  for (i = 0; i < length; i++) {
252
252
  if (counts[i]) {
@@ -293,10 +293,10 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
293
293
  }
294
294
  }
295
295
  /* 2) Let's mark all population counts that already can be encoded
296
- with an rle code. */
296
+ with an RLE code. */
297
297
  memset(good_for_rle, 0, length);
298
298
  {
299
- /* Let's not spoil any of the existing good rle codes.
299
+ /* Let's not spoil any of the existing good RLE codes.
300
300
  Mark any seq of 0's that is longer as 5 as a good_for_rle.
301
301
  Mark any seq of non-0's that is longer as 7 as a good_for_rle. */
302
302
  uint32_t symbol = counts[0];
@@ -319,7 +319,7 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
319
319
  }
320
320
  }
321
321
  }
322
- /* 3) Let's replace those population counts that lead to more rle codes.
322
+ /* 3) Let's replace those population counts that lead to more RLE codes.
323
323
  Math here is in 24.8 fixed point representation. */
324
324
  stride = 0;
325
325
  limit = 256 * (counts[0] + counts[1] + counts[2]) / 3 + 420;
@@ -420,15 +420,15 @@ void BrotliWriteHuffmanTree(const uint8_t* depth,
420
420
  }
421
421
  }
422
422
 
423
- /* First gather statistics on if it is a good idea to do rle. */
423
+ /* First gather statistics on if it is a good idea to do RLE. */
424
424
  if (length > 50) {
425
- /* Find rle coding for longer codes.
426
- Shorter codes seem not to benefit from rle. */
425
+ /* Find RLE coding for longer codes.
426
+ Shorter codes seem not to benefit from RLE. */
427
427
  DecideOverRleUse(depth, new_length,
428
428
  &use_rle_for_non_zero, &use_rle_for_zero);
429
429
  }
430
430
 
431
- /* Actual rle coding. */
431
+ /* Actual RLE coding. */
432
432
  for (i = 0; i < new_length;) {
433
433
  const uint8_t value = depth[i];
434
434
  size_t reps = 1;
@@ -9,7 +9,7 @@
9
9
  #ifndef BROTLI_ENC_ENTROPY_ENCODE_H_
10
10
  #define BROTLI_ENC_ENTROPY_ENCODE_H_
11
11
 
12
- #include "../common/types.h"
12
+ #include <brotli/types.h>
13
13
  #include "./port.h"
14
14
 
15
15
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -30,7 +30,7 @@ static BROTLI_INLINE void InitHuffmanTree(HuffmanTree* self, uint32_t count,
30
30
  self->index_right_or_value_ = right;
31
31
  }
32
32
 
33
- /* Returns 1 is assignment of depths succeded, otherwise 0. */
33
+ /* Returns 1 is assignment of depths succeeded, otherwise 0. */
34
34
  BROTLI_INTERNAL BROTLI_BOOL BrotliSetDepth(
35
35
  int p, HuffmanTree* pool, uint8_t* depth, int max_depth);
36
36
 
@@ -53,7 +53,7 @@ BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t *data,
53
53
  uint8_t *depth);
54
54
 
55
55
  /* Change the population counts in a way that the consequent
56
- Huffman tree compression, especially its rle-part will be more
56
+ Huffman tree compression, especially its RLE-part will be more
57
57
  likely to compress this data more efficiently.
58
58
 
59
59
  length contains the size of the histogram.
@@ -62,7 +62,7 @@ BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t *data,
62
62
  BROTLI_INTERNAL void BrotliOptimizeHuffmanCountsForRle(
63
63
  size_t length, uint32_t* counts, uint8_t* good_for_rle);
64
64
 
65
- /* Write a Huffman tree from bit depths into the bitstream representation
65
+ /* Write a Huffman tree from bit depths into the bit-stream representation
66
66
  of a Huffman tree. The generated Huffman tree is to be compressed once
67
67
  more using a Huffman tree */
68
68
  BROTLI_INTERNAL void BrotliWriteHuffmanTree(const uint8_t* depth,
@@ -10,8 +10,8 @@
10
10
  #define BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_
11
11
 
12
12
  #include "../common/constants.h"
13
- #include "../common/port.h"
14
- #include "../common/types.h"
13
+ #include <brotli/port.h>
14
+ #include <brotli/types.h>
15
15
  #include "./write_bits.h"
16
16
 
17
17
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -83,7 +83,7 @@ static const uint32_t kCodeLengthBits[18] = {
83
83
  static BROTLI_INLINE void StoreStaticCodeLengthCode(
84
84
  size_t* storage_ix, uint8_t* storage) {
85
85
  BrotliWriteBits(
86
- 40, MAKE_UINT64_T(0x0000ffU, 0x55555554U), storage_ix, storage);
86
+ 40, BROTLI_MAKE_UINT64_T(0x0000ffU, 0x55555554U), storage_ix, storage);
87
87
  }
88
88
 
89
89
  static const uint64_t kZeroRepsBits[BROTLI_NUM_COMMAND_SYMBOLS] = {
@@ -516,7 +516,7 @@ static const uint16_t kStaticCommandCodeBits[BROTLI_NUM_COMMAND_SYMBOLS] = {
516
516
  static BROTLI_INLINE void StoreStaticCommandHuffmanTree(
517
517
  size_t* storage_ix, uint8_t* storage) {
518
518
  BrotliWriteBits(
519
- 56, MAKE_UINT64_T(0x926244U, 0x16307003U), storage_ix, storage);
519
+ 56, BROTLI_MAKE_UINT64_T(0x926244U, 0x16307003U), storage_ix, storage);
520
520
  BrotliWriteBits(3, 0x00000000U, storage_ix, storage);
521
521
  }
522
522
 
@@ -11,15 +11,15 @@
11
11
 
12
12
  #include <math.h>
13
13
 
14
- #include "../common/types.h"
15
- #include "../common/port.h"
14
+ #include <brotli/types.h>
15
+ #include <brotli/port.h>
16
16
 
17
17
  #if defined(__cplusplus) || defined(c_plusplus)
18
18
  extern "C" {
19
19
  #endif
20
20
 
21
21
  static BROTLI_INLINE uint32_t Log2FloorNonZero(size_t n) {
22
- #ifdef __GNUC__
22
+ #if BROTLI_MODERN_COMPILER || __has_builtin(__builtin_clz)
23
23
  return 31u ^ (uint32_t)__builtin_clz((uint32_t)n);
24
24
  #else
25
25
  uint32_t result = 0;
@@ -9,7 +9,7 @@
9
9
  #ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_
10
10
  #define BROTLI_ENC_FIND_MATCH_LENGTH_H_
11
11
 
12
- #include "../common/types.h"
12
+ #include <brotli/types.h>
13
13
  #include "./port.h"
14
14
 
15
15
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -17,21 +17,21 @@ extern "C" {
17
17
  #endif
18
18
 
19
19
  /* Separate implementation for little-endian 64-bit targets, for speed. */
20
- #if defined(__GNUC__) && defined(_LP64) && defined(IS_LITTLE_ENDIAN)
20
+ #if defined(__GNUC__) && defined(_LP64) && defined(BROTLI_LITTLE_ENDIAN)
21
21
 
22
22
  static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
23
23
  const uint8_t* s2,
24
24
  size_t limit) {
25
25
  size_t matched = 0;
26
26
  size_t limit2 = (limit >> 3) + 1; /* + 1 is for pre-decrement in while */
27
- while (PREDICT_TRUE(--limit2)) {
28
- if (PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64(s2) ==
29
- BROTLI_UNALIGNED_LOAD64(s1 + matched))) {
27
+ while (BROTLI_PREDICT_TRUE(--limit2)) {
28
+ if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64LE(s2) ==
29
+ BROTLI_UNALIGNED_LOAD64LE(s1 + matched))) {
30
30
  s2 += 8;
31
31
  matched += 8;
32
32
  } else {
33
- uint64_t x =
34
- BROTLI_UNALIGNED_LOAD64(s2) ^ BROTLI_UNALIGNED_LOAD64(s1 + matched);
33
+ uint64_t x = BROTLI_UNALIGNED_LOAD64LE(s2) ^
34
+ BROTLI_UNALIGNED_LOAD64LE(s1 + matched);
35
35
  size_t matching_bits = (size_t)__builtin_ctzll(x);
36
36
  matched += matching_bits >> 3;
37
37
  return matched;
@@ -39,7 +39,7 @@ static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
39
39
  }
40
40
  limit = (limit & 7) + 1; /* + 1 is for pre-decrement in while */
41
41
  while (--limit) {
42
- if (PREDICT_TRUE(s1[matched] == *s2)) {
42
+ if (BROTLI_PREDICT_TRUE(s1[matched] == *s2)) {
43
43
  ++s2;
44
44
  ++matched;
45
45
  } else {
@@ -0,0 +1,446 @@
1
+ /* Copyright 2010 Google Inc. All Rights Reserved.
2
+
3
+ Distributed under MIT license.
4
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
+ */
6
+
7
+ /* A (forgetful) hash table to the data seen by the compressor, to
8
+ help create backward references to previous data. */
9
+
10
+ #ifndef BROTLI_ENC_HASH_H_
11
+ #define BROTLI_ENC_HASH_H_
12
+
13
+ #include <string.h> /* memcmp, memset */
14
+
15
+ #include "../common/constants.h"
16
+ #include "../common/dictionary.h"
17
+ #include <brotli/types.h>
18
+ #include "./fast_log.h"
19
+ #include "./find_match_length.h"
20
+ #include "./memory.h"
21
+ #include "./port.h"
22
+ #include "./quality.h"
23
+ #include "./static_dict.h"
24
+
25
+ #if defined(__cplusplus) || defined(c_plusplus)
26
+ extern "C" {
27
+ #endif
28
+
29
+ /* Pointer to hasher data.
30
+ *
31
+ * Excluding initialization and destruction, hasher can be passed as
32
+ * HasherHandle by value.
33
+ *
34
+ * Typically hasher data consists of 3 sections:
35
+ * * HasherCommon structure
36
+ * * private structured hasher data, depending on hasher type
37
+ * * private dynamic hasher data, depending on hasher type and parameters
38
+ */
39
+ typedef uint8_t* HasherHandle;
40
+
41
+ typedef struct {
42
+ BrotliHasherParams params;
43
+
44
+ /* False if hasher needs to be "prepared" before use. */
45
+ BROTLI_BOOL is_prepared_;
46
+
47
+ size_t dict_num_lookups;
48
+ size_t dict_num_matches;
49
+ } HasherCommon;
50
+
51
+ static BROTLI_INLINE HasherCommon* GetHasherCommon(HasherHandle handle) {
52
+ return (HasherCommon*)handle;
53
+ }
54
+
55
+ #define score_t size_t
56
+
57
+ static const uint32_t kCutoffTransformsCount = 10;
58
+ /* 0, 12, 27, 23, 42, 63, 56, 48, 59, 64 */
59
+ /* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */
60
+ static const uint64_t kCutoffTransforms =
61
+ BROTLI_MAKE_UINT64_T(0x071B520A, 0xDA2D3200);
62
+
63
+ typedef struct HasherSearchResult {
64
+ size_t len;
65
+ size_t distance;
66
+ score_t score;
67
+ int len_code_delta; /* == len_code - len */
68
+ } HasherSearchResult;
69
+
70
+ /* kHashMul32 multiplier has these properties:
71
+ * The multiplier must be odd. Otherwise we may lose the highest bit.
72
+ * No long streaks of ones or zeros.
73
+ * There is no effort to ensure that it is a prime, the oddity is enough
74
+ for this use.
75
+ * The number has been tuned heuristically against compression benchmarks. */
76
+ static const uint32_t kHashMul32 = 0x1e35a7bd;
77
+ static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(0x1e35a7bd, 0x1e35a7bd);
78
+ static const uint64_t kHashMul64Long =
79
+ BROTLI_MAKE_UINT64_T(0x1fe35a7bU, 0xd3579bd3U);
80
+
81
+ static BROTLI_INLINE uint32_t Hash14(const uint8_t* data) {
82
+ uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
83
+ /* The higher bits contain more mixture from the multiplication,
84
+ so we take our results from there. */
85
+ return h >> (32 - 14);
86
+ }
87
+
88
+ static BROTLI_INLINE void PrepareDistanceCache(
89
+ int* BROTLI_RESTRICT distance_cache, const int num_distances) {
90
+ if (num_distances > 4) {
91
+ int last_distance = distance_cache[0];
92
+ distance_cache[4] = last_distance - 1;
93
+ distance_cache[5] = last_distance + 1;
94
+ distance_cache[6] = last_distance - 2;
95
+ distance_cache[7] = last_distance + 2;
96
+ distance_cache[8] = last_distance - 3;
97
+ distance_cache[9] = last_distance + 3;
98
+ if (num_distances > 10) {
99
+ int next_last_distance = distance_cache[1];
100
+ distance_cache[10] = next_last_distance - 1;
101
+ distance_cache[11] = next_last_distance + 1;
102
+ distance_cache[12] = next_last_distance - 2;
103
+ distance_cache[13] = next_last_distance + 2;
104
+ distance_cache[14] = next_last_distance - 3;
105
+ distance_cache[15] = next_last_distance + 3;
106
+ }
107
+ }
108
+ }
109
+
110
+ #define BROTLI_LITERAL_BYTE_SCORE 135
111
+ #define BROTLI_DISTANCE_BIT_PENALTY 30
112
+ /* Score must be positive after applying maximal penalty. */
113
+ #define BROTLI_SCORE_BASE (BROTLI_DISTANCE_BIT_PENALTY * 8 * sizeof(size_t))
114
+
115
+ /* Usually, we always choose the longest backward reference. This function
116
+ allows for the exception of that rule.
117
+
118
+ If we choose a backward reference that is further away, it will
119
+ usually be coded with more bits. We approximate this by assuming
120
+ log2(distance). If the distance can be expressed in terms of the
121
+ last four distances, we use some heuristic constants to estimate
122
+ the bits cost. For the first up to four literals we use the bit
123
+ cost of the literals from the literal cost model, after that we
124
+ use the average bit cost of the cost model.
125
+
126
+ This function is used to sometimes discard a longer backward reference
127
+ when it is not much longer and the bit cost for encoding it is more
128
+ than the saved literals.
129
+
130
+ backward_reference_offset MUST be positive. */
131
+ static BROTLI_INLINE score_t BackwardReferenceScore(
132
+ size_t copy_length, size_t backward_reference_offset) {
133
+ return BROTLI_SCORE_BASE + BROTLI_LITERAL_BYTE_SCORE * (score_t)copy_length -
134
+ BROTLI_DISTANCE_BIT_PENALTY * Log2FloorNonZero(backward_reference_offset);
135
+ }
136
+
137
+ static BROTLI_INLINE score_t BackwardReferenceScoreUsingLastDistance(
138
+ size_t copy_length) {
139
+ return BROTLI_LITERAL_BYTE_SCORE * (score_t)copy_length +
140
+ BROTLI_SCORE_BASE + 15;
141
+ }
142
+
143
+ static BROTLI_INLINE score_t BackwardReferencePenaltyUsingLastDistance(
144
+ size_t distance_short_code) {
145
+ return (score_t)39 + ((0x1CA10 >> (distance_short_code & 0xE)) & 0xE);
146
+ }
147
+
148
+ static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
149
+ const BrotliDictionary* dictionary, size_t item, const uint8_t* data,
150
+ size_t max_length, size_t max_backward, HasherSearchResult* out) {
151
+ size_t len;
152
+ size_t dist;
153
+ size_t offset;
154
+ size_t matchlen;
155
+ size_t backward;
156
+ score_t score;
157
+ len = item & 0x1F;
158
+ dist = item >> 5;
159
+ offset = dictionary->offsets_by_length[len] + len * dist;
160
+ if (len > max_length) {
161
+ return BROTLI_FALSE;
162
+ }
163
+
164
+ matchlen =
165
+ FindMatchLengthWithLimit(data, &dictionary->data[offset], len);
166
+ if (matchlen + kCutoffTransformsCount <= len || matchlen == 0) {
167
+ return BROTLI_FALSE;
168
+ }
169
+ {
170
+ size_t cut = len - matchlen;
171
+ size_t transform_id =
172
+ (cut << 2) + (size_t)((kCutoffTransforms >> (cut * 6)) & 0x3F);
173
+ backward = max_backward + dist + 1 +
174
+ (transform_id << dictionary->size_bits_by_length[len]);
175
+ }
176
+ if (backward >= BROTLI_MAX_DISTANCE) {
177
+ return BROTLI_FALSE;
178
+ }
179
+ score = BackwardReferenceScore(matchlen, backward);
180
+ if (score < out->score) {
181
+ return BROTLI_FALSE;
182
+ }
183
+ out->len = matchlen;
184
+ out->len_code_delta = (int)len - (int)matchlen;
185
+ out->distance = backward;
186
+ out->score = score;
187
+ return BROTLI_TRUE;
188
+ }
189
+
190
+ static BROTLI_INLINE void SearchInStaticDictionary(
191
+ const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
192
+ HasherHandle handle, const uint8_t* data, size_t max_length,
193
+ size_t max_backward, HasherSearchResult* out, BROTLI_BOOL shallow) {
194
+ size_t key;
195
+ size_t i;
196
+ HasherCommon* self = GetHasherCommon(handle);
197
+ if (self->dict_num_matches < (self->dict_num_lookups >> 7)) {
198
+ return;
199
+ }
200
+ key = Hash14(data) << 1;
201
+ for (i = 0; i < (shallow ? 1u : 2u); ++i, ++key) {
202
+ size_t item = dictionary_hash[key];
203
+ self->dict_num_lookups++;
204
+ if (item != 0) {
205
+ BROTLI_BOOL item_matches = TestStaticDictionaryItem(
206
+ dictionary, item, data, max_length, max_backward, out);
207
+ if (item_matches) {
208
+ self->dict_num_matches++;
209
+ }
210
+ }
211
+ }
212
+ }
213
+
214
+ typedef struct BackwardMatch {
215
+ uint32_t distance;
216
+ uint32_t length_and_code;
217
+ } BackwardMatch;
218
+
219
+ static BROTLI_INLINE void InitBackwardMatch(BackwardMatch* self,
220
+ size_t dist, size_t len) {
221
+ self->distance = (uint32_t)dist;
222
+ self->length_and_code = (uint32_t)(len << 5);
223
+ }
224
+
225
+ static BROTLI_INLINE void InitDictionaryBackwardMatch(BackwardMatch* self,
226
+ size_t dist, size_t len, size_t len_code) {
227
+ self->distance = (uint32_t)dist;
228
+ self->length_and_code =
229
+ (uint32_t)((len << 5) | (len == len_code ? 0 : len_code));
230
+ }
231
+
232
+ static BROTLI_INLINE size_t BackwardMatchLength(const BackwardMatch* self) {
233
+ return self->length_and_code >> 5;
234
+ }
235
+
236
+ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
237
+ size_t code = self->length_and_code & 31;
238
+ return code ? code : BackwardMatchLength(self);
239
+ }
240
+
241
+ #define EXPAND_CAT(a, b) CAT(a, b)
242
+ #define CAT(a, b) a ## b
243
+ #define FN(X) EXPAND_CAT(X, HASHER())
244
+
245
+ #define HASHER() H10
246
+ #define BUCKET_BITS 17
247
+ #define MAX_TREE_SEARCH_DEPTH 64
248
+ #define MAX_TREE_COMP_LENGTH 128
249
+ #include "./hash_to_binary_tree_inc.h" /* NOLINT(build/include) */
250
+ #undef MAX_TREE_SEARCH_DEPTH
251
+ #undef MAX_TREE_COMP_LENGTH
252
+ #undef BUCKET_BITS
253
+ #undef HASHER
254
+ /* MAX_NUM_MATCHES == 64 + MAX_TREE_SEARCH_DEPTH */
255
+ #define MAX_NUM_MATCHES_H10 128
256
+
257
+ /* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression
258
+ a little faster (0.5% - 1%) and it compresses 0.15% better on small text
259
+ and HTML inputs. */
260
+
261
+ #define HASHER() H2
262
+ #define BUCKET_BITS 16
263
+ #define BUCKET_SWEEP 1
264
+ #define HASH_LEN 5
265
+ #define USE_DICTIONARY 1
266
+ #include "./hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
267
+ #undef BUCKET_SWEEP
268
+ #undef USE_DICTIONARY
269
+ #undef HASHER
270
+
271
+ #define HASHER() H3
272
+ #define BUCKET_SWEEP 2
273
+ #define USE_DICTIONARY 0
274
+ #include "./hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
275
+ #undef USE_DICTIONARY
276
+ #undef BUCKET_SWEEP
277
+ #undef BUCKET_BITS
278
+ #undef HASHER
279
+
280
+ #define HASHER() H4
281
+ #define BUCKET_BITS 17
282
+ #define BUCKET_SWEEP 4
283
+ #define USE_DICTIONARY 1
284
+ #include "./hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
285
+ #undef USE_DICTIONARY
286
+ #undef HASH_LEN
287
+ #undef BUCKET_SWEEP
288
+ #undef BUCKET_BITS
289
+ #undef HASHER
290
+
291
+ #define HASHER() H5
292
+ #include "./hash_longest_match_inc.h" /* NOLINT(build/include) */
293
+ #undef HASHER
294
+
295
+ #define HASHER() H6
296
+ #include "./hash_longest_match64_inc.h" /* NOLINT(build/include) */
297
+ #undef HASHER
298
+
299
+ #define BUCKET_BITS 15
300
+
301
+ #define NUM_LAST_DISTANCES_TO_CHECK 4
302
+ #define NUM_BANKS 1
303
+ #define BANK_BITS 16
304
+ #define HASHER() H40
305
+ #include "./hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
306
+ #undef HASHER
307
+ #undef NUM_LAST_DISTANCES_TO_CHECK
308
+
309
+ #define NUM_LAST_DISTANCES_TO_CHECK 10
310
+ #define HASHER() H41
311
+ #include "./hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
312
+ #undef HASHER
313
+ #undef NUM_LAST_DISTANCES_TO_CHECK
314
+ #undef NUM_BANKS
315
+ #undef BANK_BITS
316
+
317
+ #define NUM_LAST_DISTANCES_TO_CHECK 16
318
+ #define NUM_BANKS 512
319
+ #define BANK_BITS 9
320
+ #define HASHER() H42
321
+ #include "./hash_forgetful_chain_inc.h" /* NOLINT(build/include) */
322
+ #undef HASHER
323
+ #undef NUM_LAST_DISTANCES_TO_CHECK
324
+ #undef NUM_BANKS
325
+ #undef BANK_BITS
326
+
327
+ #undef BUCKET_BITS
328
+
329
+ #define HASHER() H54
330
+ #define BUCKET_BITS 20
331
+ #define BUCKET_SWEEP 4
332
+ #define HASH_LEN 7
333
+ #define USE_DICTIONARY 0
334
+ #include "./hash_longest_match_quickly_inc.h" /* NOLINT(build/include) */
335
+ #undef USE_DICTIONARY
336
+ #undef HASH_LEN
337
+ #undef BUCKET_SWEEP
338
+ #undef BUCKET_BITS
339
+ #undef HASHER
340
+
341
+ #undef FN
342
+ #undef CAT
343
+ #undef EXPAND_CAT
344
+
345
+ #define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)
346
+ #define FOR_ALL_HASHERS(H) FOR_GENERIC_HASHERS(H) H(10)
347
+
348
+ static BROTLI_INLINE void DestroyHasher(
349
+ MemoryManager* m, HasherHandle* handle) {
350
+ if (*handle == NULL) return;
351
+ BROTLI_FREE(m, *handle);
352
+ }
353
+
354
+ static BROTLI_INLINE void HasherReset(HasherHandle handle) {
355
+ if (handle == NULL) return;
356
+ GetHasherCommon(handle)->is_prepared_ = BROTLI_FALSE;
357
+ }
358
+
359
+ static BROTLI_INLINE size_t HasherSize(const BrotliEncoderParams* params,
360
+ BROTLI_BOOL one_shot, const size_t input_size) {
361
+ size_t result = sizeof(HasherCommon);
362
+ switch (params->hasher.type) {
363
+ #define SIZE_(N) \
364
+ case N: \
365
+ result += HashMemAllocInBytesH ## N(params, one_shot, input_size); \
366
+ break;
367
+ FOR_ALL_HASHERS(SIZE_)
368
+ #undef SIZE_
369
+ default:
370
+ break;
371
+ }
372
+ return result;
373
+ }
374
+
375
+ static BROTLI_INLINE void HasherSetup(MemoryManager* m, HasherHandle* handle,
376
+ BrotliEncoderParams* params, const uint8_t* data, size_t position,
377
+ size_t input_size, BROTLI_BOOL is_last) {
378
+ HasherHandle self = NULL;
379
+ HasherCommon* common = NULL;
380
+ BROTLI_BOOL one_shot = (position == 0 && is_last);
381
+ if (*handle == NULL) {
382
+ size_t alloc_size;
383
+ ChooseHasher(params, &params->hasher);
384
+ alloc_size = HasherSize(params, one_shot, input_size);
385
+ self = BROTLI_ALLOC(m, uint8_t, alloc_size);
386
+ if (BROTLI_IS_OOM(m)) return;
387
+ *handle = self;
388
+ common = GetHasherCommon(self);
389
+ common->params = params->hasher;
390
+ switch (common->params.type) {
391
+ #define INITIALIZE_(N) \
392
+ case N: \
393
+ InitializeH ## N(*handle, params); \
394
+ break;
395
+ FOR_ALL_HASHERS(INITIALIZE_);
396
+ #undef INITIALIZE_
397
+ default:
398
+ break;
399
+ }
400
+ HasherReset(*handle);
401
+ }
402
+
403
+ self = *handle;
404
+ common = GetHasherCommon(self);
405
+ if (!common->is_prepared_) {
406
+ switch (common->params.type) {
407
+ #define PREPARE_(N) \
408
+ case N: \
409
+ PrepareH ## N(self, one_shot, input_size, data); \
410
+ break;
411
+ FOR_ALL_HASHERS(PREPARE_)
412
+ #undef PREPARE_
413
+ default: break;
414
+ }
415
+ if (position == 0) {
416
+ common->dict_num_lookups = 0;
417
+ common->dict_num_matches = 0;
418
+ }
419
+ common->is_prepared_ = BROTLI_TRUE;
420
+ }
421
+ }
422
+
423
+ static BROTLI_INLINE void InitOrStitchToPreviousBlock(
424
+ MemoryManager* m, HasherHandle* handle, const uint8_t* data, size_t mask,
425
+ BrotliEncoderParams* params, size_t position, size_t input_size,
426
+ BROTLI_BOOL is_last) {
427
+ HasherHandle self;
428
+ HasherSetup(m, handle, params, data, position, input_size, is_last);
429
+ if (BROTLI_IS_OOM(m)) return;
430
+ self = *handle;
431
+ switch (GetHasherCommon(self)->params.type) {
432
+ #define INIT_(N) \
433
+ case N: \
434
+ StitchToPreviousBlockH ## N(self, input_size, position, data, mask); \
435
+ break;
436
+ FOR_ALL_HASHERS(INIT_)
437
+ #undef INIT_
438
+ default: break;
439
+ }
440
+ }
441
+
442
+ #if defined(__cplusplus) || defined(c_plusplus)
443
+ } /* extern "C" */
444
+ #endif
445
+
446
+ #endif /* BROTLI_ENC_HASH_H_ */