brotli 0.2.3 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/main.yml +37 -0
  3. data/.github/workflows/publish.yml +24 -0
  4. data/.gitmodules +1 -1
  5. data/Gemfile +6 -3
  6. data/README.md +2 -2
  7. data/Rakefile +16 -9
  8. data/brotli.gemspec +7 -13
  9. data/ext/brotli/brotli.c +210 -31
  10. data/ext/brotli/buffer.c +1 -7
  11. data/ext/brotli/buffer.h +1 -1
  12. data/ext/brotli/extconf.rb +25 -17
  13. data/lib/brotli/version.rb +1 -1
  14. data/test/brotli_test.rb +107 -0
  15. data/test/brotli_writer_test.rb +36 -0
  16. data/test/test_helper.rb +8 -0
  17. data/vendor/brotli/c/common/constants.c +15 -0
  18. data/vendor/brotli/c/common/constants.h +137 -0
  19. data/vendor/brotli/c/common/context.c +156 -0
  20. data/vendor/brotli/c/common/context.h +4 -152
  21. data/vendor/brotli/c/common/dictionary.bin.br +0 -0
  22. data/vendor/brotli/c/common/dictionary.c +14 -3
  23. data/vendor/brotli/c/common/platform.c +23 -0
  24. data/vendor/brotli/c/common/platform.h +95 -122
  25. data/vendor/brotli/c/common/shared_dictionary.c +521 -0
  26. data/vendor/brotli/c/common/shared_dictionary_internal.h +75 -0
  27. data/vendor/brotli/c/common/transform.c +60 -4
  28. data/vendor/brotli/c/common/transform.h +5 -0
  29. data/vendor/brotli/c/common/version.h +31 -6
  30. data/vendor/brotli/c/dec/bit_reader.c +34 -4
  31. data/vendor/brotli/c/dec/bit_reader.h +221 -107
  32. data/vendor/brotli/c/dec/decode.c +772 -403
  33. data/vendor/brotli/c/dec/huffman.c +7 -4
  34. data/vendor/brotli/c/dec/huffman.h +8 -13
  35. data/vendor/brotli/c/dec/prefix.h +1 -18
  36. data/vendor/brotli/c/dec/state.c +40 -21
  37. data/vendor/brotli/c/dec/state.h +201 -59
  38. data/vendor/brotli/c/enc/backward_references.c +88 -25
  39. data/vendor/brotli/c/enc/backward_references.h +10 -8
  40. data/vendor/brotli/c/enc/backward_references_hq.c +194 -80
  41. data/vendor/brotli/c/enc/backward_references_hq.h +17 -13
  42. data/vendor/brotli/c/enc/backward_references_inc.h +52 -16
  43. data/vendor/brotli/c/enc/bit_cost.c +8 -7
  44. data/vendor/brotli/c/enc/bit_cost.h +5 -4
  45. data/vendor/brotli/c/enc/block_splitter.c +40 -17
  46. data/vendor/brotli/c/enc/block_splitter.h +5 -4
  47. data/vendor/brotli/c/enc/block_splitter_inc.h +99 -49
  48. data/vendor/brotli/c/enc/brotli_bit_stream.c +142 -137
  49. data/vendor/brotli/c/enc/brotli_bit_stream.h +11 -6
  50. data/vendor/brotli/c/enc/cluster.c +10 -9
  51. data/vendor/brotli/c/enc/cluster.h +7 -6
  52. data/vendor/brotli/c/enc/cluster_inc.h +30 -22
  53. data/vendor/brotli/c/enc/command.c +28 -0
  54. data/vendor/brotli/c/enc/command.h +17 -16
  55. data/vendor/brotli/c/enc/compound_dictionary.c +207 -0
  56. data/vendor/brotli/c/enc/compound_dictionary.h +74 -0
  57. data/vendor/brotli/c/enc/compress_fragment.c +93 -83
  58. data/vendor/brotli/c/enc/compress_fragment.h +32 -7
  59. data/vendor/brotli/c/enc/compress_fragment_two_pass.c +100 -88
  60. data/vendor/brotli/c/enc/compress_fragment_two_pass.h +21 -3
  61. data/vendor/brotli/c/enc/dictionary_hash.c +1829 -1101
  62. data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
  63. data/vendor/brotli/c/enc/encode.c +550 -416
  64. data/vendor/brotli/c/enc/encoder_dict.c +613 -5
  65. data/vendor/brotli/c/enc/encoder_dict.h +120 -4
  66. data/vendor/brotli/c/enc/entropy_encode.c +5 -2
  67. data/vendor/brotli/c/enc/entropy_encode.h +4 -3
  68. data/vendor/brotli/c/enc/entropy_encode_static.h +5 -2
  69. data/vendor/brotli/c/enc/fast_log.c +105 -0
  70. data/vendor/brotli/c/enc/fast_log.h +21 -101
  71. data/vendor/brotli/c/enc/find_match_length.h +17 -25
  72. data/vendor/brotli/c/enc/hash.h +350 -120
  73. data/vendor/brotli/c/enc/hash_composite_inc.h +71 -67
  74. data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +92 -51
  75. data/vendor/brotli/c/enc/hash_longest_match64_inc.h +79 -84
  76. data/vendor/brotli/c/enc/hash_longest_match_inc.h +53 -54
  77. data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +93 -62
  78. data/vendor/brotli/c/enc/hash_rolling_inc.h +25 -29
  79. data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +42 -40
  80. data/vendor/brotli/c/enc/histogram.c +4 -4
  81. data/vendor/brotli/c/enc/histogram.h +7 -6
  82. data/vendor/brotli/c/enc/literal_cost.c +20 -15
  83. data/vendor/brotli/c/enc/literal_cost.h +4 -2
  84. data/vendor/brotli/c/enc/memory.c +29 -5
  85. data/vendor/brotli/c/enc/memory.h +43 -14
  86. data/vendor/brotli/c/enc/metablock.c +95 -85
  87. data/vendor/brotli/c/enc/metablock.h +9 -8
  88. data/vendor/brotli/c/enc/metablock_inc.h +9 -7
  89. data/vendor/brotli/c/enc/params.h +7 -4
  90. data/vendor/brotli/c/enc/prefix.h +3 -2
  91. data/vendor/brotli/c/enc/quality.h +40 -3
  92. data/vendor/brotli/c/enc/ringbuffer.h +8 -4
  93. data/vendor/brotli/c/enc/state.h +104 -0
  94. data/vendor/brotli/c/enc/static_dict.c +60 -4
  95. data/vendor/brotli/c/enc/static_dict.h +3 -2
  96. data/vendor/brotli/c/enc/static_dict_lut.h +2 -0
  97. data/vendor/brotli/c/enc/utf8_util.c +2 -2
  98. data/vendor/brotli/c/enc/utf8_util.h +2 -1
  99. data/vendor/brotli/c/enc/write_bits.h +29 -26
  100. data/vendor/brotli/c/include/brotli/decode.h +67 -2
  101. data/vendor/brotli/c/include/brotli/encode.h +77 -3
  102. data/vendor/brotli/c/include/brotli/port.h +34 -3
  103. data/vendor/brotli/c/include/brotli/shared_dictionary.h +100 -0
  104. metadata +23 -97
  105. data/.travis.yml +0 -31
  106. data/docs/Brotli/Error.html +0 -124
  107. data/docs/Brotli.html +0 -485
  108. data/docs/_index.html +0 -122
  109. data/docs/class_list.html +0 -51
  110. data/docs/css/common.css +0 -1
  111. data/docs/css/full_list.css +0 -58
  112. data/docs/css/style.css +0 -496
  113. data/docs/file.README.html +0 -127
  114. data/docs/file_list.html +0 -56
  115. data/docs/frames.html +0 -17
  116. data/docs/index.html +0 -127
  117. data/docs/js/app.js +0 -292
  118. data/docs/js/full_list.js +0 -216
  119. data/docs/js/jquery.js +0 -4
  120. data/docs/method_list.html +0 -67
  121. data/docs/top-level-namespace.html +0 -110
  122. data/spec/brotli_spec.rb +0 -88
  123. data/spec/inflate_spec.rb +0 -75
  124. data/spec/spec_helper.rb +0 -4
@@ -5,15 +5,16 @@
5
5
  See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
6
6
  */
7
7
 
8
- /* template parameters: FN, BUCKET_BITS, BUCKET_SWEEP, HASH_LEN,
8
+ /* template parameters: FN, BUCKET_BITS, BUCKET_SWEEP_BITS, HASH_LEN,
9
9
  USE_DICTIONARY
10
10
  */
11
11
 
12
12
  #define HashLongestMatchQuickly HASHER()
13
13
 
14
14
  #define BUCKET_SIZE (1 << BUCKET_BITS)
15
-
16
- #define HASH_MAP_SIZE (4 << BUCKET_BITS)
15
+ #define BUCKET_MASK (BUCKET_SIZE - 1)
16
+ #define BUCKET_SWEEP (1 << BUCKET_SWEEP_BITS)
17
+ #define BUCKET_SWEEP_MASK ((BUCKET_SWEEP - 1) << 3)
17
18
 
18
19
  static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
19
20
  static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
@@ -32,87 +33,106 @@ static uint32_t FN(HashBytes)(const uint8_t* data) {
32
33
  /* A (forgetful) hash table to the data seen by the compressor, to
33
34
  help create backward references to previous data.
34
35
 
35
- This is a hash map of fixed size (BUCKET_SIZE). Starting from the
36
- given index, BUCKET_SWEEP buckets are used to store values of a key. */
36
+ This is a hash map of fixed size (BUCKET_SIZE). */
37
37
  typedef struct HashLongestMatchQuickly {
38
- uint32_t buckets_[BUCKET_SIZE + BUCKET_SWEEP];
39
- } HashLongestMatchQuickly;
38
+ /* Shortcuts. */
39
+ HasherCommon* common;
40
40
 
41
- static BROTLI_INLINE HashLongestMatchQuickly* FN(Self)(HasherHandle handle) {
42
- return (HashLongestMatchQuickly*)&(GetHasherCommon(handle)[1]);
43
- }
41
+ /* --- Dynamic size members --- */
42
+
43
+ uint32_t* buckets_; /* uint32_t[BUCKET_SIZE]; */
44
+ } HashLongestMatchQuickly;
44
45
 
45
46
  static void FN(Initialize)(
46
- HasherHandle handle, const BrotliEncoderParams* params) {
47
- BROTLI_UNUSED(handle);
47
+ HasherCommon* common, HashLongestMatchQuickly* BROTLI_RESTRICT self,
48
+ const BrotliEncoderParams* params) {
49
+ self->common = common;
50
+
48
51
  BROTLI_UNUSED(params);
52
+ self->buckets_ = (uint32_t*)common->extra[0];
49
53
  }
50
54
 
51
- static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
52
- size_t input_size, const uint8_t* data) {
53
- HashLongestMatchQuickly* self = FN(Self)(handle);
55
+ static void FN(Prepare)(
56
+ HashLongestMatchQuickly* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
57
+ size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
58
+ uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
54
59
  /* Partial preparation is 100 times slower (per socket). */
55
- size_t partial_prepare_threshold = HASH_MAP_SIZE >> 7;
60
+ size_t partial_prepare_threshold = BUCKET_SIZE >> 5;
56
61
  if (one_shot && input_size <= partial_prepare_threshold) {
57
62
  size_t i;
58
63
  for (i = 0; i < input_size; ++i) {
59
64
  const uint32_t key = FN(HashBytes)(&data[i]);
60
- memset(&self->buckets_[key], 0, BUCKET_SWEEP * sizeof(self->buckets_[0]));
65
+ if (BUCKET_SWEEP == 1) {
66
+ buckets[key] = 0;
67
+ } else {
68
+ uint32_t j;
69
+ for (j = 0; j < BUCKET_SWEEP; ++j) {
70
+ buckets[(key + (j << 3)) & BUCKET_MASK] = 0;
71
+ }
72
+ }
61
73
  }
62
74
  } else {
63
75
  /* It is not strictly necessary to fill this buffer here, but
64
76
  not filling will make the results of the compression stochastic
65
77
  (but correct). This is because random data would cause the
66
78
  system to find accidentally good backward references here and there. */
67
- memset(&self->buckets_[0], 0, sizeof(self->buckets_));
79
+ memset(buckets, 0, sizeof(uint32_t) * BUCKET_SIZE);
68
80
  }
69
81
  }
70
82
 
71
- static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
83
+ static BROTLI_INLINE void FN(HashMemAllocInBytes)(
72
84
  const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
73
- size_t input_size) {
85
+ size_t input_size, size_t* alloc_size) {
74
86
  BROTLI_UNUSED(params);
75
87
  BROTLI_UNUSED(one_shot);
76
88
  BROTLI_UNUSED(input_size);
77
- return sizeof(HashLongestMatchQuickly);
89
+ alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE;
78
90
  }
79
91
 
80
92
  /* Look at 5 bytes at &data[ix & mask].
81
93
  Compute a hash from these, and store the value somewhere within
82
94
  [ix .. ix+3]. */
83
- static BROTLI_INLINE void FN(Store)(HasherHandle handle,
84
- const uint8_t* data, const size_t mask, const size_t ix) {
95
+ static BROTLI_INLINE void FN(Store)(
96
+ HashLongestMatchQuickly* BROTLI_RESTRICT self,
97
+ const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
85
98
  const uint32_t key = FN(HashBytes)(&data[ix & mask]);
86
- /* Wiggle the value with the bucket sweep range. */
87
- const uint32_t off = (ix >> 3) % BUCKET_SWEEP;
88
- FN(Self)(handle)->buckets_[key + off] = (uint32_t)ix;
99
+ if (BUCKET_SWEEP == 1) {
100
+ self->buckets_[key] = (uint32_t)ix;
101
+ } else {
102
+ /* Wiggle the value with the bucket sweep range. */
103
+ const uint32_t off = ix & BUCKET_SWEEP_MASK;
104
+ self->buckets_[(key + off) & BUCKET_MASK] = (uint32_t)ix;
105
+ }
89
106
  }
90
107
 
91
- static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
92
- const uint8_t* data, const size_t mask, const size_t ix_start,
93
- const size_t ix_end) {
108
+ static BROTLI_INLINE void FN(StoreRange)(
109
+ HashLongestMatchQuickly* BROTLI_RESTRICT self,
110
+ const uint8_t* BROTLI_RESTRICT data, const size_t mask,
111
+ const size_t ix_start, const size_t ix_end) {
94
112
  size_t i;
95
113
  for (i = ix_start; i < ix_end; ++i) {
96
- FN(Store)(handle, data, mask, i);
114
+ FN(Store)(self, data, mask, i);
97
115
  }
98
116
  }
99
117
 
100
118
  static BROTLI_INLINE void FN(StitchToPreviousBlock)(
101
- HasherHandle handle, size_t num_bytes, size_t position,
119
+ HashLongestMatchQuickly* BROTLI_RESTRICT self,
120
+ size_t num_bytes, size_t position,
102
121
  const uint8_t* ringbuffer, size_t ringbuffer_mask) {
103
122
  if (num_bytes >= FN(HashTypeLength)() - 1 && position >= 3) {
104
123
  /* Prepare the hashes for three last bytes of the last write.
105
124
  These could not be calculated before, since they require knowledge
106
125
  of both the previous and the current block. */
107
- FN(Store)(handle, ringbuffer, ringbuffer_mask, position - 3);
108
- FN(Store)(handle, ringbuffer, ringbuffer_mask, position - 2);
109
- FN(Store)(handle, ringbuffer, ringbuffer_mask, position - 1);
126
+ FN(Store)(self, ringbuffer, ringbuffer_mask, position - 3);
127
+ FN(Store)(self, ringbuffer, ringbuffer_mask, position - 2);
128
+ FN(Store)(self, ringbuffer, ringbuffer_mask, position - 1);
110
129
  }
111
130
  }
112
131
 
113
132
  static BROTLI_INLINE void FN(PrepareDistanceCache)(
114
- HasherHandle handle, int* BROTLI_RESTRICT distance_cache) {
115
- BROTLI_UNUSED(handle);
133
+ HashLongestMatchQuickly* BROTLI_RESTRICT self,
134
+ int* BROTLI_RESTRICT distance_cache) {
135
+ BROTLI_UNUSED(self);
116
136
  BROTLI_UNUSED(distance_cache);
117
137
  }
118
138
 
@@ -125,17 +145,19 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
125
145
  Writes the best match into |out|.
126
146
  |out|->score is updated only if a better match is found. */
127
147
  static BROTLI_INLINE void FN(FindLongestMatch)(
128
- HasherHandle handle, const BrotliEncoderDictionary* dictionary,
148
+ HashLongestMatchQuickly* BROTLI_RESTRICT self,
149
+ const BrotliEncoderDictionary* dictionary,
129
150
  const uint8_t* BROTLI_RESTRICT data,
130
151
  const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
131
152
  const size_t cur_ix, const size_t max_length, const size_t max_backward,
132
- const size_t gap, const size_t max_distance,
153
+ const size_t dictionary_distance, const size_t max_distance,
133
154
  HasherSearchResult* BROTLI_RESTRICT out) {
134
- HashLongestMatchQuickly* self = FN(Self)(handle);
155
+ uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
135
156
  const size_t best_len_in = out->len;
136
157
  const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
137
- const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
138
158
  int compare_char = data[cur_ix_masked + best_len_in];
159
+ size_t key = FN(HashBytes)(&data[cur_ix_masked]);
160
+ size_t key_out;
139
161
  score_t min_score = out->score;
140
162
  score_t best_score = out->score;
141
163
  size_t best_len = best_len_in;
@@ -145,21 +167,21 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
145
167
  if (prev_ix < cur_ix) {
146
168
  prev_ix &= (uint32_t)ring_buffer_mask;
147
169
  if (compare_char == data[prev_ix + best_len]) {
148
- size_t len = FindMatchLengthWithLimit(&data[prev_ix],
149
- &data[cur_ix_masked],
150
- max_length);
170
+ const size_t len = FindMatchLengthWithLimit(
171
+ &data[prev_ix], &data[cur_ix_masked], max_length);
151
172
  if (len >= 4) {
152
173
  const score_t score = BackwardReferenceScoreUsingLastDistance(len);
153
174
  if (best_score < score) {
154
- best_score = score;
155
- best_len = len;
156
175
  out->len = len;
157
176
  out->distance = cached_backward;
158
- out->score = best_score;
159
- compare_char = data[cur_ix_masked + best_len];
177
+ out->score = score;
160
178
  if (BUCKET_SWEEP == 1) {
161
- self->buckets_[key] = (uint32_t)cur_ix;
179
+ buckets[key] = (uint32_t)cur_ix;
162
180
  return;
181
+ } else {
182
+ best_len = len;
183
+ best_score = score;
184
+ compare_char = data[cur_ix_masked + len];
163
185
  }
164
186
  }
165
187
  }
@@ -169,8 +191,8 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
169
191
  size_t backward;
170
192
  size_t len;
171
193
  /* Only one to look for, don't bother to prepare for a loop. */
172
- prev_ix = self->buckets_[key];
173
- self->buckets_[key] = (uint32_t)cur_ix;
194
+ prev_ix = buckets[key];
195
+ buckets[key] = (uint32_t)cur_ix;
174
196
  backward = cur_ix - prev_ix;
175
197
  prev_ix &= (uint32_t)ring_buffer_mask;
176
198
  if (compare_char != data[prev_ix + best_len_in]) {
@@ -192,12 +214,17 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
192
214
  }
193
215
  }
194
216
  } else {
195
- uint32_t* bucket = self->buckets_ + key;
196
- int i;
197
- prev_ix = *bucket++;
198
- for (i = 0; i < BUCKET_SWEEP; ++i, prev_ix = *bucket++) {
199
- const size_t backward = cur_ix - prev_ix;
217
+ size_t keys[BUCKET_SWEEP];
218
+ size_t i;
219
+ for (i = 0; i < BUCKET_SWEEP; ++i) {
220
+ keys[i] = (key + (i << 3)) & BUCKET_MASK;
221
+ }
222
+ key_out = keys[(cur_ix & BUCKET_SWEEP_MASK) >> 3];
223
+ for (i = 0; i < BUCKET_SWEEP; ++i) {
200
224
  size_t len;
225
+ size_t backward;
226
+ prev_ix = buckets[keys[i]];
227
+ backward = cur_ix - prev_ix;
201
228
  prev_ix &= (uint32_t)ring_buffer_mask;
202
229
  if (compare_char != data[prev_ix + best_len]) {
203
230
  continue;
@@ -211,25 +238,29 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
211
238
  if (len >= 4) {
212
239
  const score_t score = BackwardReferenceScore(len, backward);
213
240
  if (best_score < score) {
214
- best_score = score;
215
241
  best_len = len;
216
- out->len = best_len;
217
- out->distance = backward;
242
+ out->len = len;
243
+ compare_char = data[cur_ix_masked + len];
244
+ best_score = score;
218
245
  out->score = score;
219
- compare_char = data[cur_ix_masked + best_len];
246
+ out->distance = backward;
220
247
  }
221
248
  }
222
249
  }
223
250
  }
224
251
  if (USE_DICTIONARY && min_score == out->score) {
225
252
  SearchInStaticDictionary(dictionary,
226
- handle, &data[cur_ix_masked], max_length, max_backward + gap,
253
+ self->common, &data[cur_ix_masked], max_length, dictionary_distance,
227
254
  max_distance, out, BROTLI_TRUE);
228
255
  }
229
- self->buckets_[key + ((cur_ix >> 3) % BUCKET_SWEEP)] = (uint32_t)cur_ix;
256
+ if (BUCKET_SWEEP != 1) {
257
+ buckets[key_out] = (uint32_t)cur_ix;
258
+ }
230
259
  }
231
260
 
232
- #undef HASH_MAP_SIZE
261
+ #undef BUCKET_SWEEP_MASK
262
+ #undef BUCKET_SWEEP
263
+ #undef BUCKET_MASK
233
264
  #undef BUCKET_SIZE
234
265
 
235
266
  #undef HashLongestMatchQuickly
@@ -51,13 +51,9 @@ typedef struct HashRolling {
51
51
  uint32_t factor_remove;
52
52
  } HashRolling;
53
53
 
54
- static BROTLI_INLINE HashRolling* FN(Self)(HasherHandle handle) {
55
- return (HashRolling*)&(GetHasherCommon(handle)[1]);
56
- }
57
-
58
54
  static void FN(Initialize)(
59
- HasherHandle handle, const BrotliEncoderParams* params) {
60
- HashRolling* self = FN(Self)(handle);
55
+ HasherCommon* common, HashRolling* BROTLI_RESTRICT self,
56
+ const BrotliEncoderParams* params) {
61
57
  size_t i;
62
58
  self->state = 0;
63
59
  self->next_ix = 0;
@@ -71,7 +67,7 @@ static void FN(Initialize)(
71
67
  self->factor_remove *= self->factor;
72
68
  }
73
69
 
74
- self->table = (uint32_t*)((HasherHandle)self + sizeof(HashRolling));
70
+ self->table = (uint32_t*)common->extra[0];
75
71
  for (i = 0; i < NUMBUCKETS; i++) {
76
72
  self->table[i] = FN(kInvalidPos);
77
73
  }
@@ -79,9 +75,8 @@ static void FN(Initialize)(
79
75
  BROTLI_UNUSED(params);
80
76
  }
81
77
 
82
- static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
83
- size_t input_size, const uint8_t* data) {
84
- HashRolling* self = FN(Self)(handle);
78
+ static void FN(Prepare)(HashRolling* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
79
+ size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
85
80
  size_t i;
86
81
  /* Too small size, cannot use this hasher. */
87
82
  if (input_size < CHUNKLEN) return;
@@ -93,39 +88,39 @@ static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
93
88
  BROTLI_UNUSED(one_shot);
94
89
  }
95
90
 
96
- static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
91
+ static BROTLI_INLINE void FN(HashMemAllocInBytes)(
97
92
  const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
98
- size_t input_size) {
99
- return sizeof(HashRolling) + NUMBUCKETS * sizeof(uint32_t);
93
+ size_t input_size, size_t* alloc_size) {
100
94
  BROTLI_UNUSED(params);
101
95
  BROTLI_UNUSED(one_shot);
102
96
  BROTLI_UNUSED(input_size);
97
+ alloc_size[0] = NUMBUCKETS * sizeof(uint32_t);
103
98
  }
104
99
 
105
- static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
100
+ static BROTLI_INLINE void FN(Store)(HashRolling* BROTLI_RESTRICT self,
106
101
  const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
107
- BROTLI_UNUSED(handle);
102
+ BROTLI_UNUSED(self);
108
103
  BROTLI_UNUSED(data);
109
104
  BROTLI_UNUSED(mask);
110
105
  BROTLI_UNUSED(ix);
111
106
  }
112
107
 
113
- static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
114
- const uint8_t* data, const size_t mask, const size_t ix_start,
115
- const size_t ix_end) {
116
- BROTLI_UNUSED(handle);
108
+ static BROTLI_INLINE void FN(StoreRange)(HashRolling* BROTLI_RESTRICT self,
109
+ const uint8_t* BROTLI_RESTRICT data, const size_t mask,
110
+ const size_t ix_start, const size_t ix_end) {
111
+ BROTLI_UNUSED(self);
117
112
  BROTLI_UNUSED(data);
118
113
  BROTLI_UNUSED(mask);
119
114
  BROTLI_UNUSED(ix_start);
120
115
  BROTLI_UNUSED(ix_end);
121
116
  }
122
117
 
123
- static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
118
+ static BROTLI_INLINE void FN(StitchToPreviousBlock)(
119
+ HashRolling* BROTLI_RESTRICT self,
124
120
  size_t num_bytes, size_t position, const uint8_t* ringbuffer,
125
121
  size_t ring_buffer_mask) {
126
122
  /* In this case we must re-initialize the hasher from scratch from the
127
123
  current position. */
128
- HashRolling* self = FN(Self)(handle);
129
124
  size_t position_masked;
130
125
  size_t available = num_bytes;
131
126
  if ((position & (JUMP - 1)) != 0) {
@@ -139,28 +134,29 @@ static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
139
134
  available = ring_buffer_mask - position_masked;
140
135
  }
141
136
 
142
- FN(Prepare)(handle, BROTLI_FALSE, available,
137
+ FN(Prepare)(self, BROTLI_FALSE, available,
143
138
  ringbuffer + (position & ring_buffer_mask));
144
139
  self->next_ix = position;
145
140
  BROTLI_UNUSED(num_bytes);
146
141
  }
147
142
 
148
143
  static BROTLI_INLINE void FN(PrepareDistanceCache)(
149
- HasherHandle handle, int* BROTLI_RESTRICT distance_cache) {
150
- BROTLI_UNUSED(handle);
144
+ HashRolling* BROTLI_RESTRICT self,
145
+ int* BROTLI_RESTRICT distance_cache) {
146
+ BROTLI_UNUSED(self);
151
147
  BROTLI_UNUSED(distance_cache);
152
148
  }
153
149
 
154
- static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
150
+ static BROTLI_INLINE void FN(FindLongestMatch)(
151
+ HashRolling* BROTLI_RESTRICT self,
155
152
  const BrotliEncoderDictionary* dictionary,
156
153
  const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
157
154
  const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
158
155
  const size_t max_length, const size_t max_backward,
159
- const size_t gap, const size_t max_distance,
156
+ const size_t dictionary_distance, const size_t max_distance,
160
157
  HasherSearchResult* BROTLI_RESTRICT out) {
161
- HashRolling* self = FN(Self)(handle);
162
158
  const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
163
- size_t pos = self->next_ix;
159
+ size_t pos;
164
160
 
165
161
  if ((cur_ix & (JUMP - 1)) != 0) return;
166
162
 
@@ -209,7 +205,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
209
205
  backup-hasher, the main hasher already searches in it. */
210
206
  BROTLI_UNUSED(dictionary);
211
207
  BROTLI_UNUSED(distance_cache);
212
- BROTLI_UNUSED(gap);
208
+ BROTLI_UNUSED(dictionary_distance);
213
209
  BROTLI_UNUSED(max_distance);
214
210
  }
215
211
 
@@ -24,7 +24,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
24
24
  return MAX_TREE_COMP_LENGTH;
25
25
  }
26
26
 
27
- static uint32_t FN(HashBytes)(const uint8_t* data) {
27
+ static uint32_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data) {
28
28
  uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
29
29
  /* The higher bits contain more mixture from the multiplication,
30
30
  so we take our results from there. */
@@ -38,7 +38,7 @@ typedef struct HashToBinaryTree {
38
38
  /* Hash table that maps the 4-byte hashes of the sequence to the last
39
39
  position where this hash was found, which is the root of the binary
40
40
  tree of sequences that share this hash bucket. */
41
- uint32_t buckets_[BUCKET_SIZE];
41
+ uint32_t* buckets_; /* uint32_t[BUCKET_SIZE]; */
42
42
 
43
43
  /* A position used to mark a non-existent sequence, i.e. a tree is empty if
44
44
  its root is at invalid_pos_ and a node is a leaf if both its children
@@ -51,53 +51,52 @@ typedef struct HashToBinaryTree {
51
51
  corresponding to a hash is a sequence starting at buckets_[hash] and
52
52
  the left and right children of a sequence starting at pos are
53
53
  forest_[2 * pos] and forest_[2 * pos + 1]. */
54
- /* uint32_t forest[2 * num_nodes] */
54
+ uint32_t* forest_; /* uint32_t[2 * num_nodes] */
55
55
  } HashToBinaryTree;
56
56
 
57
- static BROTLI_INLINE HashToBinaryTree* FN(Self)(HasherHandle handle) {
58
- return (HashToBinaryTree*)&(GetHasherCommon(handle)[1]);
59
- }
60
-
61
- static BROTLI_INLINE uint32_t* FN(Forest)(HashToBinaryTree* self) {
62
- return (uint32_t*)(&self[1]);
63
- }
64
-
65
57
  static void FN(Initialize)(
66
- HasherHandle handle, const BrotliEncoderParams* params) {
67
- HashToBinaryTree* self = FN(Self)(handle);
58
+ HasherCommon* common, HashToBinaryTree* BROTLI_RESTRICT self,
59
+ const BrotliEncoderParams* params) {
60
+ self->buckets_ = (uint32_t*)common->extra[0];
61
+ self->forest_ = (uint32_t*)common->extra[1];
62
+
68
63
  self->window_mask_ = (1u << params->lgwin) - 1u;
69
64
  self->invalid_pos_ = (uint32_t)(0 - self->window_mask_);
70
65
  }
71
66
 
72
- static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
73
- size_t input_size, const uint8_t* data) {
74
- HashToBinaryTree* self = FN(Self)(handle);
67
+ static void FN(Prepare)
68
+ (HashToBinaryTree* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
69
+ size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
75
70
  uint32_t invalid_pos = self->invalid_pos_;
76
71
  uint32_t i;
72
+ uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
77
73
  BROTLI_UNUSED(data);
78
74
  BROTLI_UNUSED(one_shot);
79
75
  BROTLI_UNUSED(input_size);
80
76
  for (i = 0; i < BUCKET_SIZE; i++) {
81
- self->buckets_[i] = invalid_pos;
77
+ buckets[i] = invalid_pos;
82
78
  }
83
79
  }
84
80
 
85
- static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
81
+ static BROTLI_INLINE void FN(HashMemAllocInBytes)(
86
82
  const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
87
- size_t input_size) {
83
+ size_t input_size, size_t* alloc_size) {
88
84
  size_t num_nodes = (size_t)1 << params->lgwin;
89
85
  if (one_shot && input_size < num_nodes) {
90
86
  num_nodes = input_size;
91
87
  }
92
- return sizeof(HashToBinaryTree) + 2 * sizeof(uint32_t) * num_nodes;
88
+ alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE;
89
+ alloc_size[1] = 2 * sizeof(uint32_t) * num_nodes;
93
90
  }
94
91
 
95
- static BROTLI_INLINE size_t FN(LeftChildIndex)(HashToBinaryTree* self,
92
+ static BROTLI_INLINE size_t FN(LeftChildIndex)(
93
+ HashToBinaryTree* BROTLI_RESTRICT self,
96
94
  const size_t pos) {
97
95
  return 2 * (pos & self->window_mask_);
98
96
  }
99
97
 
100
- static BROTLI_INLINE size_t FN(RightChildIndex)(HashToBinaryTree* self,
98
+ static BROTLI_INLINE size_t FN(RightChildIndex)(
99
+ HashToBinaryTree* BROTLI_RESTRICT self,
101
100
  const size_t pos) {
102
101
  return 2 * (pos & self->window_mask_) + 1;
103
102
  }
@@ -113,7 +112,7 @@ static BROTLI_INLINE size_t FN(RightChildIndex)(HashToBinaryTree* self,
113
112
 
114
113
  This function must be called with increasing cur_ix positions. */
115
114
  static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
116
- HashToBinaryTree* self, const uint8_t* const BROTLI_RESTRICT data,
115
+ HashToBinaryTree* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
117
116
  const size_t cur_ix, const size_t ring_buffer_mask, const size_t max_length,
118
117
  const size_t max_backward, size_t* const BROTLI_RESTRICT best_len,
119
118
  BackwardMatch* BROTLI_RESTRICT matches) {
@@ -123,8 +122,9 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
123
122
  const BROTLI_BOOL should_reroot_tree =
124
123
  TO_BROTLI_BOOL(max_length >= MAX_TREE_COMP_LENGTH);
125
124
  const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
126
- uint32_t* forest = FN(Forest)(self);
127
- size_t prev_ix = self->buckets_[key];
125
+ uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
126
+ uint32_t* BROTLI_RESTRICT forest = self->forest_;
127
+ size_t prev_ix = buckets[key];
128
128
  /* The forest index of the rightmost node of the left subtree of the new
129
129
  root, updated as we traverse and re-root the tree of the hash bucket. */
130
130
  size_t node_left = FN(LeftChildIndex)(self, cur_ix);
@@ -139,7 +139,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
139
139
  size_t best_len_right = 0;
140
140
  size_t depth_remaining;
141
141
  if (should_reroot_tree) {
142
- self->buckets_[key] = (uint32_t)cur_ix;
142
+ buckets[key] = (uint32_t)cur_ix;
143
143
  }
144
144
  for (depth_remaining = MAX_TREE_SEARCH_DEPTH; ; --depth_remaining) {
145
145
  const size_t backward = cur_ix - prev_ix;
@@ -199,11 +199,13 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
199
199
  matches in matches[0] to matches[*num_matches - 1]. The matches will be
200
200
  sorted by strictly increasing length and (non-strictly) increasing
201
201
  distance. */
202
- static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
203
- const BrotliEncoderDictionary* dictionary, const uint8_t* data,
202
+ static BROTLI_INLINE size_t FN(FindAllMatches)(
203
+ HashToBinaryTree* BROTLI_RESTRICT self,
204
+ const BrotliEncoderDictionary* dictionary,
205
+ const uint8_t* BROTLI_RESTRICT data,
204
206
  const size_t ring_buffer_mask, const size_t cur_ix,
205
207
  const size_t max_length, const size_t max_backward,
206
- const size_t gap, const BrotliEncoderParams* params,
208
+ const size_t dictionary_distance, const BrotliEncoderParams* params,
207
209
  BackwardMatch* matches) {
208
210
  BackwardMatch* const orig_matches = matches;
209
211
  const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
@@ -236,7 +238,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
236
238
  }
237
239
  }
238
240
  if (best_len < max_length) {
239
- matches = FN(StoreAndFindMatches)(FN(Self)(handle), data, cur_ix,
241
+ matches = FN(StoreAndFindMatches)(self, data, cur_ix,
240
242
  ring_buffer_mask, max_length, max_backward, &best_len, matches);
241
243
  }
242
244
  for (i = 0; i <= BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN; ++i) {
@@ -252,7 +254,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
252
254
  for (l = minlen; l <= maxlen; ++l) {
253
255
  uint32_t dict_id = dict_matches[l];
254
256
  if (dict_id < kInvalidMatch) {
255
- size_t distance = max_backward + gap + (dict_id >> 5) + 1;
257
+ size_t distance = dictionary_distance + (dict_id >> 5) + 1;
256
258
  if (distance <= params->dist.max_distance) {
257
259
  InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
258
260
  }
@@ -266,18 +268,18 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
266
268
  /* Stores the hash of the next 4 bytes and re-roots the binary tree at the
267
269
  current sequence, without returning any matches.
268
270
  REQUIRES: ix + MAX_TREE_COMP_LENGTH <= end-of-current-block */
269
- static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t* data,
271
+ static BROTLI_INLINE void FN(Store)(HashToBinaryTree* BROTLI_RESTRICT self,
272
+ const uint8_t* BROTLI_RESTRICT data,
270
273
  const size_t mask, const size_t ix) {
271
- HashToBinaryTree* self = FN(Self)(handle);
272
274
  /* Maximum distance is window size - 16, see section 9.1. of the spec. */
273
275
  const size_t max_backward = self->window_mask_ - BROTLI_WINDOW_GAP + 1;
274
276
  FN(StoreAndFindMatches)(self, data, ix, mask, MAX_TREE_COMP_LENGTH,
275
277
  max_backward, NULL, NULL);
276
278
  }
277
279
 
278
- static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
279
- const uint8_t* data, const size_t mask, const size_t ix_start,
280
- const size_t ix_end) {
280
+ static BROTLI_INLINE void FN(StoreRange)(HashToBinaryTree* BROTLI_RESTRICT self,
281
+ const uint8_t* BROTLI_RESTRICT data, const size_t mask,
282
+ const size_t ix_start, const size_t ix_end) {
281
283
  size_t i = ix_start;
282
284
  size_t j = ix_start;
283
285
  if (ix_start + 63 <= ix_end) {
@@ -285,18 +287,18 @@ static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
285
287
  }
286
288
  if (ix_start + 512 <= i) {
287
289
  for (; j < i; j += 8) {
288
- FN(Store)(handle, data, mask, j);
290
+ FN(Store)(self, data, mask, j);
289
291
  }
290
292
  }
291
293
  for (; i < ix_end; ++i) {
292
- FN(Store)(handle, data, mask, i);
294
+ FN(Store)(self, data, mask, i);
293
295
  }
294
296
  }
295
297
 
296
- static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
298
+ static BROTLI_INLINE void FN(StitchToPreviousBlock)(
299
+ HashToBinaryTree* BROTLI_RESTRICT self,
297
300
  size_t num_bytes, size_t position, const uint8_t* ringbuffer,
298
301
  size_t ringbuffer_mask) {
299
- HashToBinaryTree* self = FN(Self)(handle);
300
302
  if (num_bytes >= FN(HashTypeLength)() - 1 &&
301
303
  position >= MAX_TREE_COMP_LENGTH) {
302
304
  /* Store the last `MAX_TREE_COMP_LENGTH - 1` positions in the hasher.
@@ -6,11 +6,11 @@
6
6
 
7
7
  /* Build per-context histograms of literals, commands and distance codes. */
8
8
 
9
- #include "./histogram.h"
9
+ #include "histogram.h"
10
10
 
11
11
  #include "../common/context.h"
12
- #include "./block_splitter.h"
13
- #include "./command.h"
12
+ #include "block_splitter.h"
13
+ #include "command.h"
14
14
 
15
15
  #if defined(__cplusplus) || defined(c_plusplus)
16
16
  extern "C" {
@@ -63,7 +63,7 @@ void BrotliBuildHistogramsWithContext(
63
63
  BlockSplitIteratorNext(&insert_and_copy_it);
64
64
  HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_],
65
65
  cmd->cmd_prefix_);
66
- /* TODO: unwrap iterator blocks. */
66
+ /* TODO(eustas): unwrap iterator blocks. */
67
67
  for (j = cmd->insert_len_; j != 0; --j) {
68
68
  size_t context;
69
69
  BlockSplitIteratorNext(&literal_it);