brotli 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. checksums.yaml +5 -5
  2. data/Gemfile +1 -0
  3. data/Rakefile +6 -1
  4. data/brotli.gemspec +1 -1
  5. data/docs/Brotli.html +485 -0
  6. data/docs/Brotli/Error.html +124 -0
  7. data/docs/_index.html +122 -0
  8. data/docs/class_list.html +51 -0
  9. data/docs/css/common.css +1 -0
  10. data/docs/css/full_list.css +58 -0
  11. data/docs/css/style.css +496 -0
  12. data/docs/file.README.html +127 -0
  13. data/docs/file_list.html +56 -0
  14. data/docs/frames.html +17 -0
  15. data/docs/index.html +127 -0
  16. data/docs/js/app.js +292 -0
  17. data/docs/js/full_list.js +216 -0
  18. data/docs/js/jquery.js +4 -0
  19. data/docs/method_list.html +67 -0
  20. data/docs/top-level-namespace.html +110 -0
  21. data/ext/brotli/brotli.c +20 -0
  22. data/lib/brotli/version.rb +1 -1
  23. data/vendor/brotli/c/common/constants.h +13 -6
  24. data/vendor/brotli/c/{dec → common}/context.h +182 -172
  25. data/vendor/brotli/c/common/dictionary.bin +0 -0
  26. data/vendor/brotli/c/common/dictionary.bin.br +0 -0
  27. data/vendor/brotli/c/common/dictionary.c +1 -1
  28. data/vendor/brotli/c/common/dictionary.h +4 -4
  29. data/vendor/brotli/c/common/platform.h +509 -0
  30. data/vendor/brotli/c/common/transform.c +235 -0
  31. data/vendor/brotli/c/common/transform.h +80 -0
  32. data/vendor/brotli/c/common/version.h +8 -1
  33. data/vendor/brotli/c/dec/bit_reader.c +1 -1
  34. data/vendor/brotli/c/dec/bit_reader.h +35 -86
  35. data/vendor/brotli/c/dec/decode.c +322 -205
  36. data/vendor/brotli/c/dec/huffman.c +35 -37
  37. data/vendor/brotli/c/dec/huffman.h +13 -9
  38. data/vendor/brotli/c/dec/prefix.h +3 -4
  39. data/vendor/brotli/c/dec/state.c +26 -34
  40. data/vendor/brotli/c/dec/state.h +34 -23
  41. data/vendor/brotli/c/enc/backward_references.c +25 -15
  42. data/vendor/brotli/c/enc/backward_references.h +5 -6
  43. data/vendor/brotli/c/enc/backward_references_hq.c +94 -68
  44. data/vendor/brotli/c/enc/backward_references_hq.h +22 -25
  45. data/vendor/brotli/c/enc/backward_references_inc.h +10 -10
  46. data/vendor/brotli/c/enc/bit_cost.c +1 -1
  47. data/vendor/brotli/c/enc/bit_cost.h +5 -5
  48. data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
  49. data/vendor/brotli/c/enc/block_splitter.c +2 -3
  50. data/vendor/brotli/c/enc/block_splitter.h +1 -1
  51. data/vendor/brotli/c/enc/block_splitter_inc.h +11 -11
  52. data/vendor/brotli/c/enc/brotli_bit_stream.c +102 -101
  53. data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
  54. data/vendor/brotli/c/enc/cluster.c +1 -1
  55. data/vendor/brotli/c/enc/cluster.h +1 -1
  56. data/vendor/brotli/c/enc/command.h +40 -30
  57. data/vendor/brotli/c/enc/compress_fragment.c +21 -22
  58. data/vendor/brotli/c/enc/compress_fragment.h +1 -1
  59. data/vendor/brotli/c/enc/compress_fragment_two_pass.c +101 -68
  60. data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
  61. data/vendor/brotli/c/enc/dictionary_hash.c +1 -1
  62. data/vendor/brotli/c/enc/encode.c +262 -162
  63. data/vendor/brotli/c/enc/encoder_dict.c +32 -0
  64. data/vendor/brotli/c/enc/encoder_dict.h +41 -0
  65. data/vendor/brotli/c/enc/entropy_encode.c +14 -14
  66. data/vendor/brotli/c/enc/entropy_encode.h +5 -5
  67. data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
  68. data/vendor/brotli/c/enc/fast_log.h +4 -2
  69. data/vendor/brotli/c/enc/find_match_length.h +3 -3
  70. data/vendor/brotli/c/enc/hash.h +75 -24
  71. data/vendor/brotli/c/enc/hash_composite_inc.h +133 -0
  72. data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +9 -8
  73. data/vendor/brotli/c/enc/hash_longest_match64_inc.h +8 -8
  74. data/vendor/brotli/c/enc/hash_longest_match_inc.h +8 -8
  75. data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +10 -9
  76. data/vendor/brotli/c/enc/hash_rolling_inc.h +215 -0
  77. data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +9 -8
  78. data/vendor/brotli/c/enc/histogram.c +9 -6
  79. data/vendor/brotli/c/enc/histogram.h +6 -3
  80. data/vendor/brotli/c/enc/histogram_inc.h +1 -1
  81. data/vendor/brotli/c/enc/literal_cost.c +5 -5
  82. data/vendor/brotli/c/enc/literal_cost.h +2 -2
  83. data/vendor/brotli/c/enc/memory.c +5 -16
  84. data/vendor/brotli/c/enc/memory.h +40 -1
  85. data/vendor/brotli/c/enc/metablock.c +163 -25
  86. data/vendor/brotli/c/enc/metablock.h +13 -8
  87. data/vendor/brotli/c/enc/metablock_inc.h +1 -1
  88. data/vendor/brotli/c/enc/params.h +44 -0
  89. data/vendor/brotli/c/enc/prefix.h +3 -4
  90. data/vendor/brotli/c/enc/quality.h +29 -24
  91. data/vendor/brotli/c/enc/ringbuffer.h +15 -11
  92. data/vendor/brotli/c/enc/static_dict.c +49 -45
  93. data/vendor/brotli/c/enc/static_dict.h +4 -3
  94. data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
  95. data/vendor/brotli/c/enc/utf8_util.c +20 -20
  96. data/vendor/brotli/c/enc/utf8_util.h +1 -1
  97. data/vendor/brotli/c/enc/write_bits.h +16 -21
  98. data/vendor/brotli/c/include/brotli/decode.h +13 -8
  99. data/vendor/brotli/c/include/brotli/encode.h +33 -8
  100. data/vendor/brotli/c/include/brotli/port.h +211 -83
  101. data/vendor/brotli/c/include/brotli/types.h +0 -7
  102. metadata +33 -12
  103. data/vendor/brotli/c/dec/port.h +0 -168
  104. data/vendor/brotli/c/dec/transform.h +0 -300
  105. data/vendor/brotli/c/enc/context.h +0 -184
  106. data/vendor/brotli/c/enc/port.h +0 -184
@@ -0,0 +1,32 @@
1
+ /* Copyright 2017 Google Inc. All Rights Reserved.
2
+
3
+ Distributed under MIT license.
4
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
+ */
6
+
7
+ #include "./encoder_dict.h"
8
+
9
+ #include "../common/dictionary.h"
10
+ #include "../common/transform.h"
11
+ #include "./dictionary_hash.h"
12
+ #include "./hash.h"
13
+
14
+ #if defined(__cplusplus) || defined(c_plusplus)
15
+ extern "C" {
16
+ #endif
17
+
18
+ void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict) {
19
+ dict->words = BrotliGetDictionary();
20
+
21
+ dict->hash_table = kStaticDictionaryHash;
22
+ dict->buckets = kStaticDictionaryBuckets;
23
+ dict->dict_words = kStaticDictionaryWords;
24
+
25
+ dict->cutoffTransformsCount = kCutoffTransformsCount;
26
+ dict->cutoffTransforms = kCutoffTransforms;
27
+
28
+ }
29
+
30
+ #if defined(__cplusplus) || defined(c_plusplus)
31
+ } /* extern "C" */
32
+ #endif
@@ -0,0 +1,41 @@
1
+ /* Copyright 2017 Google Inc. All Rights Reserved.
2
+
3
+ Distributed under MIT license.
4
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
+ */
6
+
7
+ #ifndef BROTLI_ENC_ENCODER_DICT_H_
8
+ #define BROTLI_ENC_ENCODER_DICT_H_
9
+
10
+ #include "../common/dictionary.h"
11
+ #include "../common/platform.h"
12
+ #include <brotli/types.h>
13
+ #include "./static_dict_lut.h"
14
+
15
+ #if defined(__cplusplus) || defined(c_plusplus)
16
+ extern "C" {
17
+ #endif
18
+
19
+ /* Dictionary data (words and transforms) for 1 possible context */
20
+ typedef struct BrotliEncoderDictionary {
21
+ const BrotliDictionary* words;
22
+
23
+ /* cut off for fast encoder */
24
+ uint32_t cutoffTransformsCount;
25
+ uint64_t cutoffTransforms;
26
+
27
+ /* from dictionary_hash.h, for fast encoder */
28
+ const uint16_t* hash_table;
29
+
30
+ /* from static_dict_lut.h, for slow encoder */
31
+ const uint16_t* buckets;
32
+ const DictWord* dict_words;
33
+ } BrotliEncoderDictionary;
34
+
35
+ BROTLI_INTERNAL void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict);
36
+
37
+ #if defined(__cplusplus) || defined(c_plusplus)
38
+ } /* extern "C" */
39
+ #endif
40
+
41
+ #endif /* BROTLI_ENC_ENCODER_DICT_H_ */
@@ -11,8 +11,8 @@
11
11
  #include <string.h> /* memset */
12
12
 
13
13
  #include "../common/constants.h"
14
+ #include "../common/platform.h"
14
15
  #include <brotli/types.h>
15
- #include "./port.h"
16
16
 
17
17
  #if defined(__cplusplus) || defined(c_plusplus)
18
18
  extern "C" {
@@ -23,7 +23,7 @@ BROTLI_BOOL BrotliSetDepth(
23
23
  int stack[16];
24
24
  int level = 0;
25
25
  int p = p0;
26
- assert(max_depth <= 15);
26
+ BROTLI_DCHECK(max_depth <= 15);
27
27
  stack[0] = -1;
28
28
  while (BROTLI_TRUE) {
29
29
  if (pool[p].index_left_ >= 0) {
@@ -66,11 +66,11 @@ static BROTLI_INLINE BROTLI_BOOL SortHuffmanTree(
66
66
  we are not planning to use this with extremely long blocks.
67
67
 
68
68
  See http://en.wikipedia.org/wiki/Huffman_coding */
69
- void BrotliCreateHuffmanTree(const uint32_t *data,
69
+ void BrotliCreateHuffmanTree(const uint32_t* data,
70
70
  const size_t length,
71
71
  const int tree_limit,
72
72
  HuffmanTree* tree,
73
- uint8_t *depth) {
73
+ uint8_t* depth) {
74
74
  uint32_t count_limit;
75
75
  HuffmanTree sentinel;
76
76
  InitHuffmanTree(&sentinel, BROTLI_UINT32_MAX, -1, -1);
@@ -165,7 +165,7 @@ static void BrotliWriteHuffmanTreeRepetitions(
165
165
  size_t* tree_size,
166
166
  uint8_t* tree,
167
167
  uint8_t* extra_bits_data) {
168
- assert(repetitions > 0);
168
+ BROTLI_DCHECK(repetitions > 0);
169
169
  if (previous_value != value) {
170
170
  tree[*tree_size] = value;
171
171
  extra_bits_data[*tree_size] = 0;
@@ -371,8 +371,8 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
371
371
  }
372
372
 
373
373
  static void DecideOverRleUse(const uint8_t* depth, const size_t length,
374
- BROTLI_BOOL *use_rle_for_non_zero,
375
- BROTLI_BOOL *use_rle_for_zero) {
374
+ BROTLI_BOOL* use_rle_for_non_zero,
375
+ BROTLI_BOOL* use_rle_for_zero) {
376
376
  size_t total_reps_zero = 0;
377
377
  size_t total_reps_non_zero = 0;
378
378
  size_t count_reps_zero = 1;
@@ -454,26 +454,26 @@ void BrotliWriteHuffmanTree(const uint8_t* depth,
454
454
 
455
455
  static uint16_t BrotliReverseBits(size_t num_bits, uint16_t bits) {
456
456
  static const size_t kLut[16] = { /* Pre-reversed 4-bit values. */
457
- 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe,
458
- 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf
457
+ 0x00, 0x08, 0x04, 0x0C, 0x02, 0x0A, 0x06, 0x0E,
458
+ 0x01, 0x09, 0x05, 0x0D, 0x03, 0x0B, 0x07, 0x0F
459
459
  };
460
- size_t retval = kLut[bits & 0xf];
460
+ size_t retval = kLut[bits & 0x0F];
461
461
  size_t i;
462
462
  for (i = 4; i < num_bits; i += 4) {
463
463
  retval <<= 4;
464
464
  bits = (uint16_t)(bits >> 4);
465
- retval |= kLut[bits & 0xf];
465
+ retval |= kLut[bits & 0x0F];
466
466
  }
467
- retval >>= ((0 - num_bits) & 0x3);
467
+ retval >>= ((0 - num_bits) & 0x03);
468
468
  return (uint16_t)retval;
469
469
  }
470
470
 
471
471
  /* 0..15 are values for bits */
472
472
  #define MAX_HUFFMAN_BITS 16
473
473
 
474
- void BrotliConvertBitDepthsToSymbols(const uint8_t *depth,
474
+ void BrotliConvertBitDepthsToSymbols(const uint8_t* depth,
475
475
  size_t len,
476
- uint16_t *bits) {
476
+ uint16_t* bits) {
477
477
  /* In Brotli, all bit depths are [1..15]
478
478
  0 bit depth means that the symbol does not exist. */
479
479
  uint16_t bl_count[MAX_HUFFMAN_BITS] = { 0 };
@@ -9,8 +9,8 @@
9
9
  #ifndef BROTLI_ENC_ENTROPY_ENCODE_H_
10
10
  #define BROTLI_ENC_ENTROPY_ENCODE_H_
11
11
 
12
+ #include "../common/platform.h"
12
13
  #include <brotli/types.h>
13
- #include "./port.h"
14
14
 
15
15
  #if defined(__cplusplus) || defined(c_plusplus)
16
16
  extern "C" {
@@ -46,11 +46,11 @@ BROTLI_INTERNAL BROTLI_BOOL BrotliSetDepth(
46
46
  be at least 2 * length + 1 long.
47
47
 
48
48
  See http://en.wikipedia.org/wiki/Huffman_coding */
49
- BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t *data,
49
+ BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t* data,
50
50
  const size_t length,
51
51
  const int tree_limit,
52
52
  HuffmanTree* tree,
53
- uint8_t *depth);
53
+ uint8_t* depth);
54
54
 
55
55
  /* Change the population counts in a way that the consequent
56
56
  Huffman tree compression, especially its RLE-part will be more
@@ -72,9 +72,9 @@ BROTLI_INTERNAL void BrotliWriteHuffmanTree(const uint8_t* depth,
72
72
  uint8_t* extra_bits_data);
73
73
 
74
74
  /* Get the actual bit values for a tree of bit depths. */
75
- BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t *depth,
75
+ BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t* depth,
76
76
  size_t len,
77
- uint16_t *bits);
77
+ uint16_t* bits);
78
78
 
79
79
  /* Input size optimized Shell sort. */
80
80
  typedef BROTLI_BOOL (*HuffmanTreeComparator)(
@@ -10,7 +10,7 @@
10
10
  #define BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_
11
11
 
12
12
  #include "../common/constants.h"
13
- #include <brotli/port.h>
13
+ #include "../common/platform.h"
14
14
  #include <brotli/types.h>
15
15
  #include "./write_bits.h"
16
16
 
@@ -83,7 +83,7 @@ static const uint32_t kCodeLengthBits[18] = {
83
83
  static BROTLI_INLINE void StoreStaticCodeLengthCode(
84
84
  size_t* storage_ix, uint8_t* storage) {
85
85
  BrotliWriteBits(
86
- 40, BROTLI_MAKE_UINT64_T(0x0000ffU, 0x55555554U), storage_ix, storage);
86
+ 40, BROTLI_MAKE_UINT64_T(0x0000FFu, 0x55555554u), storage_ix, storage);
87
87
  }
88
88
 
89
89
  static const uint64_t kZeroRepsBits[BROTLI_NUM_COMMAND_SYMBOLS] = {
@@ -529,7 +529,7 @@ static const uint16_t kStaticDistanceCodeBits[64] = {
529
529
 
530
530
  static BROTLI_INLINE void StoreStaticDistanceHuffmanTree(
531
531
  size_t* storage_ix, uint8_t* storage) {
532
- BrotliWriteBits(28, 0x0369dc03U, storage_ix, storage);
532
+ BrotliWriteBits(28, 0x0369DC03u, storage_ix, storage);
533
533
  }
534
534
 
535
535
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -11,15 +11,17 @@
11
11
 
12
12
  #include <math.h>
13
13
 
14
+ #include "../common/platform.h"
14
15
  #include <brotli/types.h>
15
- #include <brotli/port.h>
16
16
 
17
17
  #if defined(__cplusplus) || defined(c_plusplus)
18
18
  extern "C" {
19
19
  #endif
20
20
 
21
21
  static BROTLI_INLINE uint32_t Log2FloorNonZero(size_t n) {
22
- #if BROTLI_MODERN_COMPILER || __has_builtin(__builtin_clz)
22
+ /* TODO: generalize and move to platform.h */
23
+ #if BROTLI_GNUC_HAS_BUILTIN(__builtin_clz, 3, 4, 0) || \
24
+ BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
23
25
  return 31u ^ (uint32_t)__builtin_clz((uint32_t)n);
24
26
  #else
25
27
  uint32_t result = 0;
@@ -9,8 +9,8 @@
9
9
  #ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_
10
10
  #define BROTLI_ENC_FIND_MATCH_LENGTH_H_
11
11
 
12
+ #include "../common/platform.h"
12
13
  #include <brotli/types.h>
13
- #include "./port.h"
14
14
 
15
15
  #if defined(__cplusplus) || defined(c_plusplus)
16
16
  extern "C" {
@@ -60,8 +60,8 @@ static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
60
60
  the first non-matching bit and use that to calculate the total
61
61
  length of the match. */
62
62
  while (s2_ptr <= s2_limit - 4 &&
63
- BROTLI_UNALIGNED_LOAD32(s2_ptr) ==
64
- BROTLI_UNALIGNED_LOAD32(s1 + matched)) {
63
+ BrotliUnalignedRead32(s2_ptr) ==
64
+ BrotliUnalignedRead32(s1 + matched)) {
65
65
  s2_ptr += 4;
66
66
  matched += 4;
67
67
  }
@@ -14,11 +14,12 @@
14
14
 
15
15
  #include "../common/constants.h"
16
16
  #include "../common/dictionary.h"
17
+ #include "../common/platform.h"
17
18
  #include <brotli/types.h>
19
+ #include "./encoder_dict.h"
18
20
  #include "./fast_log.h"
19
21
  #include "./find_match_length.h"
20
22
  #include "./memory.h"
21
- #include "./port.h"
22
23
  #include "./quality.h"
23
24
  #include "./static_dict.h"
24
25
 
@@ -35,8 +36,10 @@ extern "C" {
35
36
  * * HasherCommon structure
36
37
  * * private structured hasher data, depending on hasher type
37
38
  * * private dynamic hasher data, depending on hasher type and parameters
38
- */
39
- typedef uint8_t* HasherHandle;
39
+ *
40
+ * Using "define" instead of "typedef", because on MSVC __restrict does not work
41
+ * on typedef pointer types. */
42
+ #define HasherHandle uint8_t*
40
43
 
41
44
  typedef struct {
42
45
  BrotliHasherParams params;
@@ -73,13 +76,13 @@ typedef struct HasherSearchResult {
73
76
  * There is no effort to ensure that it is a prime, the oddity is enough
74
77
  for this use.
75
78
  * The number has been tuned heuristically against compression benchmarks. */
76
- static const uint32_t kHashMul32 = 0x1e35a7bd;
77
- static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(0x1e35a7bd, 0x1e35a7bd);
79
+ static const uint32_t kHashMul32 = 0x1E35A7BD;
80
+ static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(0x1E35A7BD, 0x1E35A7BD);
78
81
  static const uint64_t kHashMul64Long =
79
- BROTLI_MAKE_UINT64_T(0x1fe35a7bU, 0xd3579bd3U);
82
+ BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
80
83
 
81
84
  static BROTLI_INLINE uint32_t Hash14(const uint8_t* data) {
82
- uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
85
+ uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
83
86
  /* The higher bits contain more mixture from the multiplication,
84
87
  so we take our results from there. */
85
88
  return h >> (32 - 14);
@@ -146,34 +149,35 @@ static BROTLI_INLINE score_t BackwardReferencePenaltyUsingLastDistance(
146
149
  }
147
150
 
148
151
  static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
149
- const BrotliDictionary* dictionary, size_t item, const uint8_t* data,
150
- size_t max_length, size_t max_backward, HasherSearchResult* out) {
152
+ const BrotliEncoderDictionary* dictionary, size_t item, const uint8_t* data,
153
+ size_t max_length, size_t max_backward, size_t max_distance,
154
+ HasherSearchResult* out) {
151
155
  size_t len;
152
- size_t dist;
156
+ size_t word_idx;
153
157
  size_t offset;
154
158
  size_t matchlen;
155
159
  size_t backward;
156
160
  score_t score;
157
161
  len = item & 0x1F;
158
- dist = item >> 5;
159
- offset = dictionary->offsets_by_length[len] + len * dist;
162
+ word_idx = item >> 5;
163
+ offset = dictionary->words->offsets_by_length[len] + len * word_idx;
160
164
  if (len > max_length) {
161
165
  return BROTLI_FALSE;
162
166
  }
163
167
 
164
168
  matchlen =
165
- FindMatchLengthWithLimit(data, &dictionary->data[offset], len);
166
- if (matchlen + kCutoffTransformsCount <= len || matchlen == 0) {
169
+ FindMatchLengthWithLimit(data, &dictionary->words->data[offset], len);
170
+ if (matchlen + dictionary->cutoffTransformsCount <= len || matchlen == 0) {
167
171
  return BROTLI_FALSE;
168
172
  }
169
173
  {
170
174
  size_t cut = len - matchlen;
171
- size_t transform_id =
172
- (cut << 2) + (size_t)((kCutoffTransforms >> (cut * 6)) & 0x3F);
173
- backward = max_backward + dist + 1 +
174
- (transform_id << dictionary->size_bits_by_length[len]);
175
+ size_t transform_id = (cut << 2) +
176
+ (size_t)((dictionary->cutoffTransforms >> (cut * 6)) & 0x3F);
177
+ backward = max_backward + 1 + word_idx +
178
+ (transform_id << dictionary->words->size_bits_by_length[len]);
175
179
  }
176
- if (backward >= BROTLI_MAX_DISTANCE) {
180
+ if (backward > max_distance) {
177
181
  return BROTLI_FALSE;
178
182
  }
179
183
  score = BackwardReferenceScore(matchlen, backward);
@@ -188,9 +192,10 @@ static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
188
192
  }
189
193
 
190
194
  static BROTLI_INLINE void SearchInStaticDictionary(
191
- const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
195
+ const BrotliEncoderDictionary* dictionary,
192
196
  HasherHandle handle, const uint8_t* data, size_t max_length,
193
- size_t max_backward, HasherSearchResult* out, BROTLI_BOOL shallow) {
197
+ size_t max_backward, size_t max_distance,
198
+ HasherSearchResult* out, BROTLI_BOOL shallow) {
194
199
  size_t key;
195
200
  size_t i;
196
201
  HasherCommon* self = GetHasherCommon(handle);
@@ -199,11 +204,11 @@ static BROTLI_INLINE void SearchInStaticDictionary(
199
204
  }
200
205
  key = Hash14(data) << 1;
201
206
  for (i = 0; i < (shallow ? 1u : 2u); ++i, ++key) {
202
- size_t item = dictionary_hash[key];
207
+ size_t item = dictionary->hash_table[key];
203
208
  self->dict_num_lookups++;
204
209
  if (item != 0) {
205
210
  BROTLI_BOOL item_matches = TestStaticDictionaryItem(
206
- dictionary, item, data, max_length, max_backward, out);
211
+ dictionary, item, data, max_length, max_backward, max_distance, out);
207
212
  if (item_matches) {
208
213
  self->dict_num_matches++;
209
214
  }
@@ -338,11 +343,57 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
338
343
  #undef BUCKET_BITS
339
344
  #undef HASHER
340
345
 
346
+ /* fast large window hashers */
347
+
348
+ #define HASHER() HROLLING_FAST
349
+ #define CHUNKLEN 32
350
+ #define JUMP 4
351
+ #define NUMBUCKETS 16777216
352
+ #define MASK ((NUMBUCKETS * 64) - 1)
353
+ #include "./hash_rolling_inc.h" /* NOLINT(build/include) */
354
+ #undef JUMP
355
+ #undef HASHER
356
+
357
+
358
+ #define HASHER() HROLLING
359
+ #define JUMP 1
360
+ #include "./hash_rolling_inc.h" /* NOLINT(build/include) */
361
+ #undef MASK
362
+ #undef NUMBUCKETS
363
+ #undef JUMP
364
+ #undef CHUNKLEN
365
+ #undef HASHER
366
+
367
+ #define HASHER() H35
368
+ #define HASHER_A H3
369
+ #define HASHER_B HROLLING_FAST
370
+ #include "./hash_composite_inc.h" /* NOLINT(build/include) */
371
+ #undef HASHER_A
372
+ #undef HASHER_B
373
+ #undef HASHER
374
+
375
+ #define HASHER() H55
376
+ #define HASHER_A H54
377
+ #define HASHER_B HROLLING_FAST
378
+ #include "./hash_composite_inc.h" /* NOLINT(build/include) */
379
+ #undef HASHER_A
380
+ #undef HASHER_B
381
+ #undef HASHER
382
+
383
+ #define HASHER() H65
384
+ #define HASHER_A H6
385
+ #define HASHER_B HROLLING
386
+ #include "./hash_composite_inc.h" /* NOLINT(build/include) */
387
+ #undef HASHER_A
388
+ #undef HASHER_B
389
+ #undef HASHER
390
+
341
391
  #undef FN
342
392
  #undef CAT
343
393
  #undef EXPAND_CAT
344
394
 
345
- #define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)
395
+ #define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)\
396
+ H(35) H(55) H(65)
346
397
  #define FOR_ALL_HASHERS(H) FOR_GENERIC_HASHERS(H) H(10)
347
398
 
348
399
  static BROTLI_INLINE void DestroyHasher(
@@ -0,0 +1,133 @@
1
+ /* NOLINT(build/header_guard) */
2
+ /* Copyright 2018 Google Inc. All Rights Reserved.
3
+
4
+ Distributed under MIT license.
5
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
6
+ */
7
+
8
+ /* template parameters: FN, HASHER_A, HASHER_B */
9
+
10
+ /* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
11
+ and HASHER_B. */
12
+
13
+ #define HashComposite HASHER()
14
+
15
+ #define FN_A(X) EXPAND_CAT(X, HASHER_A)
16
+ #define FN_B(X) EXPAND_CAT(X, HASHER_B)
17
+
18
+ static BROTLI_INLINE size_t FN(HashTypeLength)(void) {
19
+ size_t a = FN_A(HashTypeLength)();
20
+ size_t b = FN_B(HashTypeLength)();
21
+ return a > b ? a : b;
22
+ }
23
+
24
+ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
25
+ size_t a = FN_A(StoreLookahead)();
26
+ size_t b = FN_B(StoreLookahead)();
27
+ return a > b ? a : b;
28
+ }
29
+
30
+ typedef struct HashComposite {
31
+ HasherHandle ha;
32
+ HasherHandle hb;
33
+ const BrotliEncoderParams* params;
34
+ } HashComposite;
35
+
36
+ static BROTLI_INLINE HashComposite* FN(Self)(HasherHandle handle) {
37
+ return (HashComposite*)&(GetHasherCommon(handle)[1]);
38
+ }
39
+
40
+ static void FN(Initialize)(
41
+ HasherHandle handle, const BrotliEncoderParams* params) {
42
+ HashComposite* self = FN(Self)(handle);
43
+ self->ha = 0;
44
+ self->hb = 0;
45
+ self->params = params;
46
+ /* TODO: Initialize of the hashers is defered to Prepare (and params
47
+ remembered here) because we don't get the one_shot and input_size params
48
+ here that are needed to know the memory size of them. Instead provide
49
+ those params to all hashers FN(Initialize) */
50
+ }
51
+
52
+ static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
53
+ size_t input_size, const uint8_t* data) {
54
+ HashComposite* self = FN(Self)(handle);
55
+ if (!self->ha) {
56
+ HasherCommon* common_a;
57
+ HasherCommon* common_b;
58
+
59
+ self->ha = handle + sizeof(HasherCommon) + sizeof(HashComposite);
60
+ common_a = (HasherCommon*)self->ha;
61
+ common_a->params = self->params->hasher;
62
+ common_a->is_prepared_ = BROTLI_FALSE;
63
+ common_a->dict_num_lookups = 0;
64
+ common_a->dict_num_matches = 0;
65
+ FN_A(Initialize)(self->ha, self->params);
66
+
67
+ self->hb = self->ha + sizeof(HasherCommon) + FN_A(HashMemAllocInBytes)(
68
+ self->params, one_shot, input_size);
69
+ common_b = (HasherCommon*)self->hb;
70
+ common_b->params = self->params->hasher;
71
+ common_b->is_prepared_ = BROTLI_FALSE;
72
+ common_b->dict_num_lookups = 0;
73
+ common_b->dict_num_matches = 0;
74
+ FN_B(Initialize)(self->hb, self->params);
75
+ }
76
+ FN_A(Prepare)(self->ha, one_shot, input_size, data);
77
+ FN_B(Prepare)(self->hb, one_shot, input_size, data);
78
+ }
79
+
80
+ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
81
+ const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
82
+ size_t input_size) {
83
+ return sizeof(HashComposite) + 2 * sizeof(HasherCommon) +
84
+ FN_A(HashMemAllocInBytes)(params, one_shot, input_size) +
85
+ FN_B(HashMemAllocInBytes)(params, one_shot, input_size);
86
+ }
87
+
88
+ static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
89
+ const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
90
+ HashComposite* self = FN(Self)(handle);
91
+ FN_A(Store)(self->ha, data, mask, ix);
92
+ FN_B(Store)(self->hb, data, mask, ix);
93
+ }
94
+
95
+ static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
96
+ const uint8_t* data, const size_t mask, const size_t ix_start,
97
+ const size_t ix_end) {
98
+ HashComposite* self = FN(Self)(handle);
99
+ FN_A(StoreRange)(self->ha, data, mask, ix_start, ix_end);
100
+ FN_B(StoreRange)(self->hb, data, mask, ix_start, ix_end);
101
+ }
102
+
103
+ static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
104
+ size_t num_bytes, size_t position, const uint8_t* ringbuffer,
105
+ size_t ring_buffer_mask) {
106
+ HashComposite* self = FN(Self)(handle);
107
+ FN_A(StitchToPreviousBlock)(self->ha, num_bytes, position, ringbuffer,
108
+ ring_buffer_mask);
109
+ FN_B(StitchToPreviousBlock)(self->hb, num_bytes, position, ringbuffer,
110
+ ring_buffer_mask);
111
+ }
112
+
113
+ static BROTLI_INLINE void FN(PrepareDistanceCache)(
114
+ HasherHandle handle, int* BROTLI_RESTRICT distance_cache) {
115
+ HashComposite* self = FN(Self)(handle);
116
+ FN_A(PrepareDistanceCache)(self->ha, distance_cache);
117
+ FN_B(PrepareDistanceCache)(self->hb, distance_cache);
118
+ }
119
+
120
+ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
121
+ const BrotliEncoderDictionary* dictionary,
122
+ const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
123
+ const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
124
+ const size_t max_length, const size_t max_backward, const size_t gap,
125
+ const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
126
+ HashComposite* self = FN(Self)(handle);
127
+ FN_A(FindLongestMatch)(self->ha, dictionary, data, ring_buffer_mask,
128
+ distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
129
+ FN_B(FindLongestMatch)(self->hb, dictionary, data, ring_buffer_mask,
130
+ distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
131
+ }
132
+
133
+ #undef HashComposite