brotli 0.2.0 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. checksums.yaml +5 -5
  2. data/Gemfile +1 -0
  3. data/Rakefile +6 -1
  4. data/brotli.gemspec +1 -1
  5. data/docs/Brotli.html +485 -0
  6. data/docs/Brotli/Error.html +124 -0
  7. data/docs/_index.html +122 -0
  8. data/docs/class_list.html +51 -0
  9. data/docs/css/common.css +1 -0
  10. data/docs/css/full_list.css +58 -0
  11. data/docs/css/style.css +496 -0
  12. data/docs/file.README.html +127 -0
  13. data/docs/file_list.html +56 -0
  14. data/docs/frames.html +17 -0
  15. data/docs/index.html +127 -0
  16. data/docs/js/app.js +292 -0
  17. data/docs/js/full_list.js +216 -0
  18. data/docs/js/jquery.js +4 -0
  19. data/docs/method_list.html +67 -0
  20. data/docs/top-level-namespace.html +110 -0
  21. data/ext/brotli/brotli.c +20 -0
  22. data/lib/brotli/version.rb +1 -1
  23. data/vendor/brotli/c/common/constants.h +13 -6
  24. data/vendor/brotli/c/{dec → common}/context.h +182 -172
  25. data/vendor/brotli/c/common/dictionary.bin +0 -0
  26. data/vendor/brotli/c/common/dictionary.bin.br +0 -0
  27. data/vendor/brotli/c/common/dictionary.c +1 -1
  28. data/vendor/brotli/c/common/dictionary.h +4 -4
  29. data/vendor/brotli/c/common/platform.h +509 -0
  30. data/vendor/brotli/c/common/transform.c +235 -0
  31. data/vendor/brotli/c/common/transform.h +80 -0
  32. data/vendor/brotli/c/common/version.h +8 -1
  33. data/vendor/brotli/c/dec/bit_reader.c +1 -1
  34. data/vendor/brotli/c/dec/bit_reader.h +35 -86
  35. data/vendor/brotli/c/dec/decode.c +322 -205
  36. data/vendor/brotli/c/dec/huffman.c +35 -37
  37. data/vendor/brotli/c/dec/huffman.h +13 -9
  38. data/vendor/brotli/c/dec/prefix.h +3 -4
  39. data/vendor/brotli/c/dec/state.c +26 -34
  40. data/vendor/brotli/c/dec/state.h +34 -23
  41. data/vendor/brotli/c/enc/backward_references.c +25 -15
  42. data/vendor/brotli/c/enc/backward_references.h +5 -6
  43. data/vendor/brotli/c/enc/backward_references_hq.c +94 -68
  44. data/vendor/brotli/c/enc/backward_references_hq.h +22 -25
  45. data/vendor/brotli/c/enc/backward_references_inc.h +10 -10
  46. data/vendor/brotli/c/enc/bit_cost.c +1 -1
  47. data/vendor/brotli/c/enc/bit_cost.h +5 -5
  48. data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
  49. data/vendor/brotli/c/enc/block_splitter.c +2 -3
  50. data/vendor/brotli/c/enc/block_splitter.h +1 -1
  51. data/vendor/brotli/c/enc/block_splitter_inc.h +11 -11
  52. data/vendor/brotli/c/enc/brotli_bit_stream.c +102 -101
  53. data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
  54. data/vendor/brotli/c/enc/cluster.c +1 -1
  55. data/vendor/brotli/c/enc/cluster.h +1 -1
  56. data/vendor/brotli/c/enc/command.h +40 -30
  57. data/vendor/brotli/c/enc/compress_fragment.c +21 -22
  58. data/vendor/brotli/c/enc/compress_fragment.h +1 -1
  59. data/vendor/brotli/c/enc/compress_fragment_two_pass.c +101 -68
  60. data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
  61. data/vendor/brotli/c/enc/dictionary_hash.c +1 -1
  62. data/vendor/brotli/c/enc/encode.c +262 -162
  63. data/vendor/brotli/c/enc/encoder_dict.c +32 -0
  64. data/vendor/brotli/c/enc/encoder_dict.h +41 -0
  65. data/vendor/brotli/c/enc/entropy_encode.c +14 -14
  66. data/vendor/brotli/c/enc/entropy_encode.h +5 -5
  67. data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
  68. data/vendor/brotli/c/enc/fast_log.h +4 -2
  69. data/vendor/brotli/c/enc/find_match_length.h +3 -3
  70. data/vendor/brotli/c/enc/hash.h +75 -24
  71. data/vendor/brotli/c/enc/hash_composite_inc.h +133 -0
  72. data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +9 -8
  73. data/vendor/brotli/c/enc/hash_longest_match64_inc.h +8 -8
  74. data/vendor/brotli/c/enc/hash_longest_match_inc.h +8 -8
  75. data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +10 -9
  76. data/vendor/brotli/c/enc/hash_rolling_inc.h +215 -0
  77. data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +9 -8
  78. data/vendor/brotli/c/enc/histogram.c +9 -6
  79. data/vendor/brotli/c/enc/histogram.h +6 -3
  80. data/vendor/brotli/c/enc/histogram_inc.h +1 -1
  81. data/vendor/brotli/c/enc/literal_cost.c +5 -5
  82. data/vendor/brotli/c/enc/literal_cost.h +2 -2
  83. data/vendor/brotli/c/enc/memory.c +5 -16
  84. data/vendor/brotli/c/enc/memory.h +40 -1
  85. data/vendor/brotli/c/enc/metablock.c +163 -25
  86. data/vendor/brotli/c/enc/metablock.h +13 -8
  87. data/vendor/brotli/c/enc/metablock_inc.h +1 -1
  88. data/vendor/brotli/c/enc/params.h +44 -0
  89. data/vendor/brotli/c/enc/prefix.h +3 -4
  90. data/vendor/brotli/c/enc/quality.h +29 -24
  91. data/vendor/brotli/c/enc/ringbuffer.h +15 -11
  92. data/vendor/brotli/c/enc/static_dict.c +49 -45
  93. data/vendor/brotli/c/enc/static_dict.h +4 -3
  94. data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
  95. data/vendor/brotli/c/enc/utf8_util.c +20 -20
  96. data/vendor/brotli/c/enc/utf8_util.h +1 -1
  97. data/vendor/brotli/c/enc/write_bits.h +16 -21
  98. data/vendor/brotli/c/include/brotli/decode.h +13 -8
  99. data/vendor/brotli/c/include/brotli/encode.h +33 -8
  100. data/vendor/brotli/c/include/brotli/port.h +211 -83
  101. data/vendor/brotli/c/include/brotli/types.h +0 -7
  102. metadata +33 -12
  103. data/vendor/brotli/c/dec/port.h +0 -168
  104. data/vendor/brotli/c/dec/transform.h +0 -300
  105. data/vendor/brotli/c/enc/context.h +0 -184
  106. data/vendor/brotli/c/enc/port.h +0 -184
@@ -0,0 +1,32 @@
1
+ /* Copyright 2017 Google Inc. All Rights Reserved.
2
+
3
+ Distributed under MIT license.
4
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
+ */
6
+
7
+ #include "./encoder_dict.h"
8
+
9
+ #include "../common/dictionary.h"
10
+ #include "../common/transform.h"
11
+ #include "./dictionary_hash.h"
12
+ #include "./hash.h"
13
+
14
+ #if defined(__cplusplus) || defined(c_plusplus)
15
+ extern "C" {
16
+ #endif
17
+
18
+ void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict) {
19
+ dict->words = BrotliGetDictionary();
20
+
21
+ dict->hash_table = kStaticDictionaryHash;
22
+ dict->buckets = kStaticDictionaryBuckets;
23
+ dict->dict_words = kStaticDictionaryWords;
24
+
25
+ dict->cutoffTransformsCount = kCutoffTransformsCount;
26
+ dict->cutoffTransforms = kCutoffTransforms;
27
+
28
+ }
29
+
30
+ #if defined(__cplusplus) || defined(c_plusplus)
31
+ } /* extern "C" */
32
+ #endif
@@ -0,0 +1,41 @@
1
+ /* Copyright 2017 Google Inc. All Rights Reserved.
2
+
3
+ Distributed under MIT license.
4
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
+ */
6
+
7
+ #ifndef BROTLI_ENC_ENCODER_DICT_H_
8
+ #define BROTLI_ENC_ENCODER_DICT_H_
9
+
10
+ #include "../common/dictionary.h"
11
+ #include "../common/platform.h"
12
+ #include <brotli/types.h>
13
+ #include "./static_dict_lut.h"
14
+
15
+ #if defined(__cplusplus) || defined(c_plusplus)
16
+ extern "C" {
17
+ #endif
18
+
19
+ /* Dictionary data (words and transforms) for 1 possible context */
20
+ typedef struct BrotliEncoderDictionary {
21
+ const BrotliDictionary* words;
22
+
23
+ /* cut off for fast encoder */
24
+ uint32_t cutoffTransformsCount;
25
+ uint64_t cutoffTransforms;
26
+
27
+ /* from dictionary_hash.h, for fast encoder */
28
+ const uint16_t* hash_table;
29
+
30
+ /* from static_dict_lut.h, for slow encoder */
31
+ const uint16_t* buckets;
32
+ const DictWord* dict_words;
33
+ } BrotliEncoderDictionary;
34
+
35
+ BROTLI_INTERNAL void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict);
36
+
37
+ #if defined(__cplusplus) || defined(c_plusplus)
38
+ } /* extern "C" */
39
+ #endif
40
+
41
+ #endif /* BROTLI_ENC_ENCODER_DICT_H_ */
@@ -11,8 +11,8 @@
11
11
  #include <string.h> /* memset */
12
12
 
13
13
  #include "../common/constants.h"
14
+ #include "../common/platform.h"
14
15
  #include <brotli/types.h>
15
- #include "./port.h"
16
16
 
17
17
  #if defined(__cplusplus) || defined(c_plusplus)
18
18
  extern "C" {
@@ -23,7 +23,7 @@ BROTLI_BOOL BrotliSetDepth(
23
23
  int stack[16];
24
24
  int level = 0;
25
25
  int p = p0;
26
- assert(max_depth <= 15);
26
+ BROTLI_DCHECK(max_depth <= 15);
27
27
  stack[0] = -1;
28
28
  while (BROTLI_TRUE) {
29
29
  if (pool[p].index_left_ >= 0) {
@@ -66,11 +66,11 @@ static BROTLI_INLINE BROTLI_BOOL SortHuffmanTree(
66
66
  we are not planning to use this with extremely long blocks.
67
67
 
68
68
  See http://en.wikipedia.org/wiki/Huffman_coding */
69
- void BrotliCreateHuffmanTree(const uint32_t *data,
69
+ void BrotliCreateHuffmanTree(const uint32_t* data,
70
70
  const size_t length,
71
71
  const int tree_limit,
72
72
  HuffmanTree* tree,
73
- uint8_t *depth) {
73
+ uint8_t* depth) {
74
74
  uint32_t count_limit;
75
75
  HuffmanTree sentinel;
76
76
  InitHuffmanTree(&sentinel, BROTLI_UINT32_MAX, -1, -1);
@@ -165,7 +165,7 @@ static void BrotliWriteHuffmanTreeRepetitions(
165
165
  size_t* tree_size,
166
166
  uint8_t* tree,
167
167
  uint8_t* extra_bits_data) {
168
- assert(repetitions > 0);
168
+ BROTLI_DCHECK(repetitions > 0);
169
169
  if (previous_value != value) {
170
170
  tree[*tree_size] = value;
171
171
  extra_bits_data[*tree_size] = 0;
@@ -371,8 +371,8 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
371
371
  }
372
372
 
373
373
  static void DecideOverRleUse(const uint8_t* depth, const size_t length,
374
- BROTLI_BOOL *use_rle_for_non_zero,
375
- BROTLI_BOOL *use_rle_for_zero) {
374
+ BROTLI_BOOL* use_rle_for_non_zero,
375
+ BROTLI_BOOL* use_rle_for_zero) {
376
376
  size_t total_reps_zero = 0;
377
377
  size_t total_reps_non_zero = 0;
378
378
  size_t count_reps_zero = 1;
@@ -454,26 +454,26 @@ void BrotliWriteHuffmanTree(const uint8_t* depth,
454
454
 
455
455
  static uint16_t BrotliReverseBits(size_t num_bits, uint16_t bits) {
456
456
  static const size_t kLut[16] = { /* Pre-reversed 4-bit values. */
457
- 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe,
458
- 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf
457
+ 0x00, 0x08, 0x04, 0x0C, 0x02, 0x0A, 0x06, 0x0E,
458
+ 0x01, 0x09, 0x05, 0x0D, 0x03, 0x0B, 0x07, 0x0F
459
459
  };
460
- size_t retval = kLut[bits & 0xf];
460
+ size_t retval = kLut[bits & 0x0F];
461
461
  size_t i;
462
462
  for (i = 4; i < num_bits; i += 4) {
463
463
  retval <<= 4;
464
464
  bits = (uint16_t)(bits >> 4);
465
- retval |= kLut[bits & 0xf];
465
+ retval |= kLut[bits & 0x0F];
466
466
  }
467
- retval >>= ((0 - num_bits) & 0x3);
467
+ retval >>= ((0 - num_bits) & 0x03);
468
468
  return (uint16_t)retval;
469
469
  }
470
470
 
471
471
  /* 0..15 are values for bits */
472
472
  #define MAX_HUFFMAN_BITS 16
473
473
 
474
- void BrotliConvertBitDepthsToSymbols(const uint8_t *depth,
474
+ void BrotliConvertBitDepthsToSymbols(const uint8_t* depth,
475
475
  size_t len,
476
- uint16_t *bits) {
476
+ uint16_t* bits) {
477
477
  /* In Brotli, all bit depths are [1..15]
478
478
  0 bit depth means that the symbol does not exist. */
479
479
  uint16_t bl_count[MAX_HUFFMAN_BITS] = { 0 };
@@ -9,8 +9,8 @@
9
9
  #ifndef BROTLI_ENC_ENTROPY_ENCODE_H_
10
10
  #define BROTLI_ENC_ENTROPY_ENCODE_H_
11
11
 
12
+ #include "../common/platform.h"
12
13
  #include <brotli/types.h>
13
- #include "./port.h"
14
14
 
15
15
  #if defined(__cplusplus) || defined(c_plusplus)
16
16
  extern "C" {
@@ -46,11 +46,11 @@ BROTLI_INTERNAL BROTLI_BOOL BrotliSetDepth(
46
46
  be at least 2 * length + 1 long.
47
47
 
48
48
  See http://en.wikipedia.org/wiki/Huffman_coding */
49
- BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t *data,
49
+ BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t* data,
50
50
  const size_t length,
51
51
  const int tree_limit,
52
52
  HuffmanTree* tree,
53
- uint8_t *depth);
53
+ uint8_t* depth);
54
54
 
55
55
  /* Change the population counts in a way that the consequent
56
56
  Huffman tree compression, especially its RLE-part will be more
@@ -72,9 +72,9 @@ BROTLI_INTERNAL void BrotliWriteHuffmanTree(const uint8_t* depth,
72
72
  uint8_t* extra_bits_data);
73
73
 
74
74
  /* Get the actual bit values for a tree of bit depths. */
75
- BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t *depth,
75
+ BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t* depth,
76
76
  size_t len,
77
- uint16_t *bits);
77
+ uint16_t* bits);
78
78
 
79
79
  /* Input size optimized Shell sort. */
80
80
  typedef BROTLI_BOOL (*HuffmanTreeComparator)(
@@ -10,7 +10,7 @@
10
10
  #define BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_
11
11
 
12
12
  #include "../common/constants.h"
13
- #include <brotli/port.h>
13
+ #include "../common/platform.h"
14
14
  #include <brotli/types.h>
15
15
  #include "./write_bits.h"
16
16
 
@@ -83,7 +83,7 @@ static const uint32_t kCodeLengthBits[18] = {
83
83
  static BROTLI_INLINE void StoreStaticCodeLengthCode(
84
84
  size_t* storage_ix, uint8_t* storage) {
85
85
  BrotliWriteBits(
86
- 40, BROTLI_MAKE_UINT64_T(0x0000ffU, 0x55555554U), storage_ix, storage);
86
+ 40, BROTLI_MAKE_UINT64_T(0x0000FFu, 0x55555554u), storage_ix, storage);
87
87
  }
88
88
 
89
89
  static const uint64_t kZeroRepsBits[BROTLI_NUM_COMMAND_SYMBOLS] = {
@@ -529,7 +529,7 @@ static const uint16_t kStaticDistanceCodeBits[64] = {
529
529
 
530
530
  static BROTLI_INLINE void StoreStaticDistanceHuffmanTree(
531
531
  size_t* storage_ix, uint8_t* storage) {
532
- BrotliWriteBits(28, 0x0369dc03U, storage_ix, storage);
532
+ BrotliWriteBits(28, 0x0369DC03u, storage_ix, storage);
533
533
  }
534
534
 
535
535
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -11,15 +11,17 @@
11
11
 
12
12
  #include <math.h>
13
13
 
14
+ #include "../common/platform.h"
14
15
  #include <brotli/types.h>
15
- #include <brotli/port.h>
16
16
 
17
17
  #if defined(__cplusplus) || defined(c_plusplus)
18
18
  extern "C" {
19
19
  #endif
20
20
 
21
21
  static BROTLI_INLINE uint32_t Log2FloorNonZero(size_t n) {
22
- #if BROTLI_MODERN_COMPILER || __has_builtin(__builtin_clz)
22
+ /* TODO: generalize and move to platform.h */
23
+ #if BROTLI_GNUC_HAS_BUILTIN(__builtin_clz, 3, 4, 0) || \
24
+ BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
23
25
  return 31u ^ (uint32_t)__builtin_clz((uint32_t)n);
24
26
  #else
25
27
  uint32_t result = 0;
@@ -9,8 +9,8 @@
9
9
  #ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_
10
10
  #define BROTLI_ENC_FIND_MATCH_LENGTH_H_
11
11
 
12
+ #include "../common/platform.h"
12
13
  #include <brotli/types.h>
13
- #include "./port.h"
14
14
 
15
15
  #if defined(__cplusplus) || defined(c_plusplus)
16
16
  extern "C" {
@@ -60,8 +60,8 @@ static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
60
60
  the first non-matching bit and use that to calculate the total
61
61
  length of the match. */
62
62
  while (s2_ptr <= s2_limit - 4 &&
63
- BROTLI_UNALIGNED_LOAD32(s2_ptr) ==
64
- BROTLI_UNALIGNED_LOAD32(s1 + matched)) {
63
+ BrotliUnalignedRead32(s2_ptr) ==
64
+ BrotliUnalignedRead32(s1 + matched)) {
65
65
  s2_ptr += 4;
66
66
  matched += 4;
67
67
  }
@@ -14,11 +14,12 @@
14
14
 
15
15
  #include "../common/constants.h"
16
16
  #include "../common/dictionary.h"
17
+ #include "../common/platform.h"
17
18
  #include <brotli/types.h>
19
+ #include "./encoder_dict.h"
18
20
  #include "./fast_log.h"
19
21
  #include "./find_match_length.h"
20
22
  #include "./memory.h"
21
- #include "./port.h"
22
23
  #include "./quality.h"
23
24
  #include "./static_dict.h"
24
25
 
@@ -35,8 +36,10 @@ extern "C" {
35
36
  * * HasherCommon structure
36
37
  * * private structured hasher data, depending on hasher type
37
38
  * * private dynamic hasher data, depending on hasher type and parameters
38
- */
39
- typedef uint8_t* HasherHandle;
39
+ *
40
+ * Using "define" instead of "typedef", because on MSVC __restrict does not work
41
+ * on typedef pointer types. */
42
+ #define HasherHandle uint8_t*
40
43
 
41
44
  typedef struct {
42
45
  BrotliHasherParams params;
@@ -73,13 +76,13 @@ typedef struct HasherSearchResult {
73
76
  * There is no effort to ensure that it is a prime, the oddity is enough
74
77
  for this use.
75
78
  * The number has been tuned heuristically against compression benchmarks. */
76
- static const uint32_t kHashMul32 = 0x1e35a7bd;
77
- static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(0x1e35a7bd, 0x1e35a7bd);
79
+ static const uint32_t kHashMul32 = 0x1E35A7BD;
80
+ static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(0x1E35A7BD, 0x1E35A7BD);
78
81
  static const uint64_t kHashMul64Long =
79
- BROTLI_MAKE_UINT64_T(0x1fe35a7bU, 0xd3579bd3U);
82
+ BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
80
83
 
81
84
  static BROTLI_INLINE uint32_t Hash14(const uint8_t* data) {
82
- uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
85
+ uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
83
86
  /* The higher bits contain more mixture from the multiplication,
84
87
  so we take our results from there. */
85
88
  return h >> (32 - 14);
@@ -146,34 +149,35 @@ static BROTLI_INLINE score_t BackwardReferencePenaltyUsingLastDistance(
146
149
  }
147
150
 
148
151
  static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
149
- const BrotliDictionary* dictionary, size_t item, const uint8_t* data,
150
- size_t max_length, size_t max_backward, HasherSearchResult* out) {
152
+ const BrotliEncoderDictionary* dictionary, size_t item, const uint8_t* data,
153
+ size_t max_length, size_t max_backward, size_t max_distance,
154
+ HasherSearchResult* out) {
151
155
  size_t len;
152
- size_t dist;
156
+ size_t word_idx;
153
157
  size_t offset;
154
158
  size_t matchlen;
155
159
  size_t backward;
156
160
  score_t score;
157
161
  len = item & 0x1F;
158
- dist = item >> 5;
159
- offset = dictionary->offsets_by_length[len] + len * dist;
162
+ word_idx = item >> 5;
163
+ offset = dictionary->words->offsets_by_length[len] + len * word_idx;
160
164
  if (len > max_length) {
161
165
  return BROTLI_FALSE;
162
166
  }
163
167
 
164
168
  matchlen =
165
- FindMatchLengthWithLimit(data, &dictionary->data[offset], len);
166
- if (matchlen + kCutoffTransformsCount <= len || matchlen == 0) {
169
+ FindMatchLengthWithLimit(data, &dictionary->words->data[offset], len);
170
+ if (matchlen + dictionary->cutoffTransformsCount <= len || matchlen == 0) {
167
171
  return BROTLI_FALSE;
168
172
  }
169
173
  {
170
174
  size_t cut = len - matchlen;
171
- size_t transform_id =
172
- (cut << 2) + (size_t)((kCutoffTransforms >> (cut * 6)) & 0x3F);
173
- backward = max_backward + dist + 1 +
174
- (transform_id << dictionary->size_bits_by_length[len]);
175
+ size_t transform_id = (cut << 2) +
176
+ (size_t)((dictionary->cutoffTransforms >> (cut * 6)) & 0x3F);
177
+ backward = max_backward + 1 + word_idx +
178
+ (transform_id << dictionary->words->size_bits_by_length[len]);
175
179
  }
176
- if (backward >= BROTLI_MAX_DISTANCE) {
180
+ if (backward > max_distance) {
177
181
  return BROTLI_FALSE;
178
182
  }
179
183
  score = BackwardReferenceScore(matchlen, backward);
@@ -188,9 +192,10 @@ static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
188
192
  }
189
193
 
190
194
  static BROTLI_INLINE void SearchInStaticDictionary(
191
- const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
195
+ const BrotliEncoderDictionary* dictionary,
192
196
  HasherHandle handle, const uint8_t* data, size_t max_length,
193
- size_t max_backward, HasherSearchResult* out, BROTLI_BOOL shallow) {
197
+ size_t max_backward, size_t max_distance,
198
+ HasherSearchResult* out, BROTLI_BOOL shallow) {
194
199
  size_t key;
195
200
  size_t i;
196
201
  HasherCommon* self = GetHasherCommon(handle);
@@ -199,11 +204,11 @@ static BROTLI_INLINE void SearchInStaticDictionary(
199
204
  }
200
205
  key = Hash14(data) << 1;
201
206
  for (i = 0; i < (shallow ? 1u : 2u); ++i, ++key) {
202
- size_t item = dictionary_hash[key];
207
+ size_t item = dictionary->hash_table[key];
203
208
  self->dict_num_lookups++;
204
209
  if (item != 0) {
205
210
  BROTLI_BOOL item_matches = TestStaticDictionaryItem(
206
- dictionary, item, data, max_length, max_backward, out);
211
+ dictionary, item, data, max_length, max_backward, max_distance, out);
207
212
  if (item_matches) {
208
213
  self->dict_num_matches++;
209
214
  }
@@ -338,11 +343,57 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
338
343
  #undef BUCKET_BITS
339
344
  #undef HASHER
340
345
 
346
+ /* fast large window hashers */
347
+
348
+ #define HASHER() HROLLING_FAST
349
+ #define CHUNKLEN 32
350
+ #define JUMP 4
351
+ #define NUMBUCKETS 16777216
352
+ #define MASK ((NUMBUCKETS * 64) - 1)
353
+ #include "./hash_rolling_inc.h" /* NOLINT(build/include) */
354
+ #undef JUMP
355
+ #undef HASHER
356
+
357
+
358
+ #define HASHER() HROLLING
359
+ #define JUMP 1
360
+ #include "./hash_rolling_inc.h" /* NOLINT(build/include) */
361
+ #undef MASK
362
+ #undef NUMBUCKETS
363
+ #undef JUMP
364
+ #undef CHUNKLEN
365
+ #undef HASHER
366
+
367
+ #define HASHER() H35
368
+ #define HASHER_A H3
369
+ #define HASHER_B HROLLING_FAST
370
+ #include "./hash_composite_inc.h" /* NOLINT(build/include) */
371
+ #undef HASHER_A
372
+ #undef HASHER_B
373
+ #undef HASHER
374
+
375
+ #define HASHER() H55
376
+ #define HASHER_A H54
377
+ #define HASHER_B HROLLING_FAST
378
+ #include "./hash_composite_inc.h" /* NOLINT(build/include) */
379
+ #undef HASHER_A
380
+ #undef HASHER_B
381
+ #undef HASHER
382
+
383
+ #define HASHER() H65
384
+ #define HASHER_A H6
385
+ #define HASHER_B HROLLING
386
+ #include "./hash_composite_inc.h" /* NOLINT(build/include) */
387
+ #undef HASHER_A
388
+ #undef HASHER_B
389
+ #undef HASHER
390
+
341
391
  #undef FN
342
392
  #undef CAT
343
393
  #undef EXPAND_CAT
344
394
 
345
- #define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)
395
+ #define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)\
396
+ H(35) H(55) H(65)
346
397
  #define FOR_ALL_HASHERS(H) FOR_GENERIC_HASHERS(H) H(10)
347
398
 
348
399
  static BROTLI_INLINE void DestroyHasher(
@@ -0,0 +1,133 @@
1
+ /* NOLINT(build/header_guard) */
2
+ /* Copyright 2018 Google Inc. All Rights Reserved.
3
+
4
+ Distributed under MIT license.
5
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
6
+ */
7
+
8
+ /* template parameters: FN, HASHER_A, HASHER_B */
9
+
10
+ /* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
11
+ and HASHER_B. */
12
+
13
+ #define HashComposite HASHER()
14
+
15
+ #define FN_A(X) EXPAND_CAT(X, HASHER_A)
16
+ #define FN_B(X) EXPAND_CAT(X, HASHER_B)
17
+
18
+ static BROTLI_INLINE size_t FN(HashTypeLength)(void) {
19
+ size_t a = FN_A(HashTypeLength)();
20
+ size_t b = FN_B(HashTypeLength)();
21
+ return a > b ? a : b;
22
+ }
23
+
24
+ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
25
+ size_t a = FN_A(StoreLookahead)();
26
+ size_t b = FN_B(StoreLookahead)();
27
+ return a > b ? a : b;
28
+ }
29
+
30
+ typedef struct HashComposite {
31
+ HasherHandle ha;
32
+ HasherHandle hb;
33
+ const BrotliEncoderParams* params;
34
+ } HashComposite;
35
+
36
+ static BROTLI_INLINE HashComposite* FN(Self)(HasherHandle handle) {
37
+ return (HashComposite*)&(GetHasherCommon(handle)[1]);
38
+ }
39
+
40
+ static void FN(Initialize)(
41
+ HasherHandle handle, const BrotliEncoderParams* params) {
42
+ HashComposite* self = FN(Self)(handle);
43
+ self->ha = 0;
44
+ self->hb = 0;
45
+ self->params = params;
46
+ /* TODO: Initialize of the hashers is defered to Prepare (and params
47
+ remembered here) because we don't get the one_shot and input_size params
48
+ here that are needed to know the memory size of them. Instead provide
49
+ those params to all hashers FN(Initialize) */
50
+ }
51
+
52
+ static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
53
+ size_t input_size, const uint8_t* data) {
54
+ HashComposite* self = FN(Self)(handle);
55
+ if (!self->ha) {
56
+ HasherCommon* common_a;
57
+ HasherCommon* common_b;
58
+
59
+ self->ha = handle + sizeof(HasherCommon) + sizeof(HashComposite);
60
+ common_a = (HasherCommon*)self->ha;
61
+ common_a->params = self->params->hasher;
62
+ common_a->is_prepared_ = BROTLI_FALSE;
63
+ common_a->dict_num_lookups = 0;
64
+ common_a->dict_num_matches = 0;
65
+ FN_A(Initialize)(self->ha, self->params);
66
+
67
+ self->hb = self->ha + sizeof(HasherCommon) + FN_A(HashMemAllocInBytes)(
68
+ self->params, one_shot, input_size);
69
+ common_b = (HasherCommon*)self->hb;
70
+ common_b->params = self->params->hasher;
71
+ common_b->is_prepared_ = BROTLI_FALSE;
72
+ common_b->dict_num_lookups = 0;
73
+ common_b->dict_num_matches = 0;
74
+ FN_B(Initialize)(self->hb, self->params);
75
+ }
76
+ FN_A(Prepare)(self->ha, one_shot, input_size, data);
77
+ FN_B(Prepare)(self->hb, one_shot, input_size, data);
78
+ }
79
+
80
+ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
81
+ const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
82
+ size_t input_size) {
83
+ return sizeof(HashComposite) + 2 * sizeof(HasherCommon) +
84
+ FN_A(HashMemAllocInBytes)(params, one_shot, input_size) +
85
+ FN_B(HashMemAllocInBytes)(params, one_shot, input_size);
86
+ }
87
+
88
+ static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
89
+ const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
90
+ HashComposite* self = FN(Self)(handle);
91
+ FN_A(Store)(self->ha, data, mask, ix);
92
+ FN_B(Store)(self->hb, data, mask, ix);
93
+ }
94
+
95
+ static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
96
+ const uint8_t* data, const size_t mask, const size_t ix_start,
97
+ const size_t ix_end) {
98
+ HashComposite* self = FN(Self)(handle);
99
+ FN_A(StoreRange)(self->ha, data, mask, ix_start, ix_end);
100
+ FN_B(StoreRange)(self->hb, data, mask, ix_start, ix_end);
101
+ }
102
+
103
+ static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
104
+ size_t num_bytes, size_t position, const uint8_t* ringbuffer,
105
+ size_t ring_buffer_mask) {
106
+ HashComposite* self = FN(Self)(handle);
107
+ FN_A(StitchToPreviousBlock)(self->ha, num_bytes, position, ringbuffer,
108
+ ring_buffer_mask);
109
+ FN_B(StitchToPreviousBlock)(self->hb, num_bytes, position, ringbuffer,
110
+ ring_buffer_mask);
111
+ }
112
+
113
+ static BROTLI_INLINE void FN(PrepareDistanceCache)(
114
+ HasherHandle handle, int* BROTLI_RESTRICT distance_cache) {
115
+ HashComposite* self = FN(Self)(handle);
116
+ FN_A(PrepareDistanceCache)(self->ha, distance_cache);
117
+ FN_B(PrepareDistanceCache)(self->hb, distance_cache);
118
+ }
119
+
120
+ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
121
+ const BrotliEncoderDictionary* dictionary,
122
+ const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
123
+ const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
124
+ const size_t max_length, const size_t max_backward, const size_t gap,
125
+ const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
126
+ HashComposite* self = FN(Self)(handle);
127
+ FN_A(FindLongestMatch)(self->ha, dictionary, data, ring_buffer_mask,
128
+ distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
129
+ FN_B(FindLongestMatch)(self->hb, dictionary, data, ring_buffer_mask,
130
+ distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
131
+ }
132
+
133
+ #undef HashComposite