brotli 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile +1 -0
- data/Rakefile +6 -1
- data/brotli.gemspec +1 -1
- data/docs/Brotli.html +485 -0
- data/docs/Brotli/Error.html +124 -0
- data/docs/_index.html +122 -0
- data/docs/class_list.html +51 -0
- data/docs/css/common.css +1 -0
- data/docs/css/full_list.css +58 -0
- data/docs/css/style.css +496 -0
- data/docs/file.README.html +127 -0
- data/docs/file_list.html +56 -0
- data/docs/frames.html +17 -0
- data/docs/index.html +127 -0
- data/docs/js/app.js +292 -0
- data/docs/js/full_list.js +216 -0
- data/docs/js/jquery.js +4 -0
- data/docs/method_list.html +67 -0
- data/docs/top-level-namespace.html +110 -0
- data/ext/brotli/brotli.c +20 -0
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/c/common/constants.h +13 -6
- data/vendor/brotli/c/{dec → common}/context.h +182 -172
- data/vendor/brotli/c/common/dictionary.bin +0 -0
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +1 -1
- data/vendor/brotli/c/common/dictionary.h +4 -4
- data/vendor/brotli/c/common/platform.h +509 -0
- data/vendor/brotli/c/common/transform.c +235 -0
- data/vendor/brotli/c/common/transform.h +80 -0
- data/vendor/brotli/c/common/version.h +8 -1
- data/vendor/brotli/c/dec/bit_reader.c +1 -1
- data/vendor/brotli/c/dec/bit_reader.h +35 -86
- data/vendor/brotli/c/dec/decode.c +322 -205
- data/vendor/brotli/c/dec/huffman.c +35 -37
- data/vendor/brotli/c/dec/huffman.h +13 -9
- data/vendor/brotli/c/dec/prefix.h +3 -4
- data/vendor/brotli/c/dec/state.c +26 -34
- data/vendor/brotli/c/dec/state.h +34 -23
- data/vendor/brotli/c/enc/backward_references.c +25 -15
- data/vendor/brotli/c/enc/backward_references.h +5 -6
- data/vendor/brotli/c/enc/backward_references_hq.c +94 -68
- data/vendor/brotli/c/enc/backward_references_hq.h +22 -25
- data/vendor/brotli/c/enc/backward_references_inc.h +10 -10
- data/vendor/brotli/c/enc/bit_cost.c +1 -1
- data/vendor/brotli/c/enc/bit_cost.h +5 -5
- data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
- data/vendor/brotli/c/enc/block_splitter.c +2 -3
- data/vendor/brotli/c/enc/block_splitter.h +1 -1
- data/vendor/brotli/c/enc/block_splitter_inc.h +11 -11
- data/vendor/brotli/c/enc/brotli_bit_stream.c +102 -101
- data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
- data/vendor/brotli/c/enc/cluster.c +1 -1
- data/vendor/brotli/c/enc/cluster.h +1 -1
- data/vendor/brotli/c/enc/command.h +40 -30
- data/vendor/brotli/c/enc/compress_fragment.c +21 -22
- data/vendor/brotli/c/enc/compress_fragment.h +1 -1
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +101 -68
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1 -1
- data/vendor/brotli/c/enc/encode.c +262 -162
- data/vendor/brotli/c/enc/encoder_dict.c +32 -0
- data/vendor/brotli/c/enc/encoder_dict.h +41 -0
- data/vendor/brotli/c/enc/entropy_encode.c +14 -14
- data/vendor/brotli/c/enc/entropy_encode.h +5 -5
- data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
- data/vendor/brotli/c/enc/fast_log.h +4 -2
- data/vendor/brotli/c/enc/find_match_length.h +3 -3
- data/vendor/brotli/c/enc/hash.h +75 -24
- data/vendor/brotli/c/enc/hash_composite_inc.h +133 -0
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +9 -8
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +10 -9
- data/vendor/brotli/c/enc/hash_rolling_inc.h +215 -0
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +9 -8
- data/vendor/brotli/c/enc/histogram.c +9 -6
- data/vendor/brotli/c/enc/histogram.h +6 -3
- data/vendor/brotli/c/enc/histogram_inc.h +1 -1
- data/vendor/brotli/c/enc/literal_cost.c +5 -5
- data/vendor/brotli/c/enc/literal_cost.h +2 -2
- data/vendor/brotli/c/enc/memory.c +5 -16
- data/vendor/brotli/c/enc/memory.h +40 -1
- data/vendor/brotli/c/enc/metablock.c +163 -25
- data/vendor/brotli/c/enc/metablock.h +13 -8
- data/vendor/brotli/c/enc/metablock_inc.h +1 -1
- data/vendor/brotli/c/enc/params.h +44 -0
- data/vendor/brotli/c/enc/prefix.h +3 -4
- data/vendor/brotli/c/enc/quality.h +29 -24
- data/vendor/brotli/c/enc/ringbuffer.h +15 -11
- data/vendor/brotli/c/enc/static_dict.c +49 -45
- data/vendor/brotli/c/enc/static_dict.h +4 -3
- data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
- data/vendor/brotli/c/enc/utf8_util.c +20 -20
- data/vendor/brotli/c/enc/utf8_util.h +1 -1
- data/vendor/brotli/c/enc/write_bits.h +16 -21
- data/vendor/brotli/c/include/brotli/decode.h +13 -8
- data/vendor/brotli/c/include/brotli/encode.h +33 -8
- data/vendor/brotli/c/include/brotli/port.h +211 -83
- data/vendor/brotli/c/include/brotli/types.h +0 -7
- metadata +33 -12
- data/vendor/brotli/c/dec/port.h +0 -168
- data/vendor/brotli/c/dec/transform.h +0 -300
- data/vendor/brotli/c/enc/context.h +0 -184
- data/vendor/brotli/c/enc/port.h +0 -184
|
@@ -28,8 +28,8 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
|
|
|
28
28
|
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
|
|
29
29
|
|
|
30
30
|
/* HashBytes is the function that chooses the bucket to place the address in.*/
|
|
31
|
-
static BROTLI_INLINE size_t FN(HashBytes)(const uint8_t
|
|
32
|
-
const uint32_t h =
|
|
31
|
+
static BROTLI_INLINE size_t FN(HashBytes)(const uint8_t* data) {
|
|
32
|
+
const uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
|
33
33
|
/* The higher bits contain more mixture from the multiplication,
|
|
34
34
|
so we take our results from there. */
|
|
35
35
|
return h >> (32 - BUCKET_BITS);
|
|
@@ -115,7 +115,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
|
|
|
115
115
|
}
|
|
116
116
|
|
|
117
117
|
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
|
118
|
-
const uint8_t
|
|
118
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
|
119
119
|
const size_t ix_end) {
|
|
120
120
|
size_t i;
|
|
121
121
|
for (i = ix_start; i < ix_end; ++i) {
|
|
@@ -154,11 +154,12 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
|
154
154
|
Writes the best match into |out|.
|
|
155
155
|
|out|->score is updated only if a better match is found. */
|
|
156
156
|
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
157
|
-
const
|
|
157
|
+
const BrotliEncoderDictionary* dictionary,
|
|
158
158
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
|
159
159
|
const int* BROTLI_RESTRICT distance_cache,
|
|
160
160
|
const size_t cur_ix, const size_t max_length, const size_t max_backward,
|
|
161
|
-
const size_t gap,
|
|
161
|
+
const size_t gap, const size_t max_distance,
|
|
162
|
+
HasherSearchResult* BROTLI_RESTRICT out) {
|
|
162
163
|
HashForgetfulChain* self = FN(Self)(handle);
|
|
163
164
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
|
164
165
|
/* Don't accept a short copy from far away. */
|
|
@@ -240,9 +241,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
|
240
241
|
FN(Store)(handle, data, ring_buffer_mask, cur_ix);
|
|
241
242
|
}
|
|
242
243
|
if (out->score == min_score) {
|
|
243
|
-
SearchInStaticDictionary(dictionary,
|
|
244
|
-
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
|
245
|
-
BROTLI_FALSE);
|
|
244
|
+
SearchInStaticDictionary(dictionary,
|
|
245
|
+
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
|
246
|
+
max_distance, out, BROTLI_FALSE);
|
|
246
247
|
}
|
|
247
248
|
}
|
|
248
249
|
|
|
@@ -20,7 +20,7 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
|
|
|
20
20
|
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
|
|
21
21
|
|
|
22
22
|
/* HashBytes is the function that chooses the bucket to place the address in. */
|
|
23
|
-
static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t
|
|
23
|
+
static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t* data,
|
|
24
24
|
const uint64_t mask,
|
|
25
25
|
const int shift) {
|
|
26
26
|
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(data) & mask) * kHashMul64Long;
|
|
@@ -105,7 +105,7 @@ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
|
|
105
105
|
|
|
106
106
|
/* Look at 4 bytes at &data[ix & mask].
|
|
107
107
|
Compute a hash from these, and store the value of ix at that position. */
|
|
108
|
-
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t
|
|
108
|
+
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t* data,
|
|
109
109
|
const size_t mask, const size_t ix) {
|
|
110
110
|
HashLongestMatch* self = FN(Self)(handle);
|
|
111
111
|
uint16_t* num = FN(Num)(self);
|
|
@@ -119,7 +119,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t *data,
|
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
|
122
|
-
const uint8_t
|
|
122
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
|
123
123
|
const size_t ix_end) {
|
|
124
124
|
size_t i;
|
|
125
125
|
for (i = ix_start; i < ix_end; ++i) {
|
|
@@ -158,11 +158,11 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
|
158
158
|
Writes the best match into |out|.
|
|
159
159
|
|out|->score is updated only if a better match is found. */
|
|
160
160
|
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
161
|
-
const
|
|
161
|
+
const BrotliEncoderDictionary* dictionary,
|
|
162
162
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
|
163
163
|
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
|
164
164
|
const size_t max_length, const size_t max_backward, const size_t gap,
|
|
165
|
-
HasherSearchResult* BROTLI_RESTRICT out) {
|
|
165
|
+
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
|
|
166
166
|
HasherCommon* common = GetHasherCommon(handle);
|
|
167
167
|
HashLongestMatch* self = FN(Self)(handle);
|
|
168
168
|
uint16_t* num = FN(Num)(self);
|
|
@@ -257,9 +257,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
|
257
257
|
++num[key];
|
|
258
258
|
}
|
|
259
259
|
if (min_score == out->score) {
|
|
260
|
-
SearchInStaticDictionary(dictionary,
|
|
261
|
-
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
|
262
|
-
BROTLI_FALSE);
|
|
260
|
+
SearchInStaticDictionary(dictionary,
|
|
261
|
+
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
|
262
|
+
max_distance, out, BROTLI_FALSE);
|
|
263
263
|
}
|
|
264
264
|
}
|
|
265
265
|
|
|
@@ -20,8 +20,8 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
|
|
|
20
20
|
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
|
|
21
21
|
|
|
22
22
|
/* HashBytes is the function that chooses the bucket to place the address in. */
|
|
23
|
-
static uint32_t FN(HashBytes)(const uint8_t
|
|
24
|
-
uint32_t h =
|
|
23
|
+
static uint32_t FN(HashBytes)(const uint8_t* data, const int shift) {
|
|
24
|
+
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
|
25
25
|
/* The higher bits contain more mixture from the multiplication,
|
|
26
26
|
so we take our results from there. */
|
|
27
27
|
return (uint32_t)(h >> shift);
|
|
@@ -112,7 +112,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t* data,
|
|
|
112
112
|
}
|
|
113
113
|
|
|
114
114
|
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
|
115
|
-
const uint8_t
|
|
115
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
|
116
116
|
const size_t ix_end) {
|
|
117
117
|
size_t i;
|
|
118
118
|
for (i = ix_start; i < ix_end; ++i) {
|
|
@@ -151,11 +151,11 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
|
151
151
|
Writes the best match into |out|.
|
|
152
152
|
|out|->score is updated only if a better match is found. */
|
|
153
153
|
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
154
|
-
const
|
|
154
|
+
const BrotliEncoderDictionary* dictionary,
|
|
155
155
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
|
156
156
|
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
|
157
157
|
const size_t max_length, const size_t max_backward, const size_t gap,
|
|
158
|
-
HasherSearchResult* BROTLI_RESTRICT out) {
|
|
158
|
+
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
|
|
159
159
|
HasherCommon* common = GetHasherCommon(handle);
|
|
160
160
|
HashLongestMatch* self = FN(Self)(handle);
|
|
161
161
|
uint16_t* num = FN(Num)(self);
|
|
@@ -249,9 +249,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
|
249
249
|
++num[key];
|
|
250
250
|
}
|
|
251
251
|
if (min_score == out->score) {
|
|
252
|
-
SearchInStaticDictionary(dictionary,
|
|
253
|
-
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
|
254
|
-
BROTLI_FALSE);
|
|
252
|
+
SearchInStaticDictionary(dictionary,
|
|
253
|
+
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
|
254
|
+
max_distance, out, BROTLI_FALSE);
|
|
255
255
|
}
|
|
256
256
|
}
|
|
257
257
|
|
|
@@ -81,7 +81,7 @@ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
|
|
81
81
|
Compute a hash from these, and store the value somewhere within
|
|
82
82
|
[ix .. ix+3]. */
|
|
83
83
|
static BROTLI_INLINE void FN(Store)(HasherHandle handle,
|
|
84
|
-
const uint8_t
|
|
84
|
+
const uint8_t* data, const size_t mask, const size_t ix) {
|
|
85
85
|
const uint32_t key = FN(HashBytes)(&data[ix & mask]);
|
|
86
86
|
/* Wiggle the value with the bucket sweep range. */
|
|
87
87
|
const uint32_t off = (ix >> 3) % BUCKET_SWEEP;
|
|
@@ -89,7 +89,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle,
|
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
|
92
|
-
const uint8_t
|
|
92
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
|
93
93
|
const size_t ix_end) {
|
|
94
94
|
size_t i;
|
|
95
95
|
for (i = ix_start; i < ix_end; ++i) {
|
|
@@ -125,11 +125,12 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
|
125
125
|
Writes the best match into |out|.
|
|
126
126
|
|out|->score is updated only if a better match is found. */
|
|
127
127
|
static BROTLI_INLINE void FN(FindLongestMatch)(
|
|
128
|
-
HasherHandle handle, const
|
|
129
|
-
const
|
|
128
|
+
HasherHandle handle, const BrotliEncoderDictionary* dictionary,
|
|
129
|
+
const uint8_t* BROTLI_RESTRICT data,
|
|
130
130
|
const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
|
|
131
131
|
const size_t cur_ix, const size_t max_length, const size_t max_backward,
|
|
132
|
-
const size_t gap,
|
|
132
|
+
const size_t gap, const size_t max_distance,
|
|
133
|
+
HasherSearchResult* BROTLI_RESTRICT out) {
|
|
133
134
|
HashLongestMatchQuickly* self = FN(Self)(handle);
|
|
134
135
|
const size_t best_len_in = out->len;
|
|
135
136
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
|
@@ -191,7 +192,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
|
|
|
191
192
|
}
|
|
192
193
|
}
|
|
193
194
|
} else {
|
|
194
|
-
uint32_t
|
|
195
|
+
uint32_t* bucket = self->buckets_ + key;
|
|
195
196
|
int i;
|
|
196
197
|
prev_ix = *bucket++;
|
|
197
198
|
for (i = 0; i < BUCKET_SWEEP; ++i, prev_ix = *bucket++) {
|
|
@@ -221,9 +222,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
|
|
|
221
222
|
}
|
|
222
223
|
}
|
|
223
224
|
if (USE_DICTIONARY && min_score == out->score) {
|
|
224
|
-
SearchInStaticDictionary(dictionary,
|
|
225
|
-
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
|
226
|
-
BROTLI_TRUE);
|
|
225
|
+
SearchInStaticDictionary(dictionary,
|
|
226
|
+
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
|
227
|
+
max_distance, out, BROTLI_TRUE);
|
|
227
228
|
}
|
|
228
229
|
self->buckets_[key + ((cur_ix >> 3) % BUCKET_SWEEP)] = (uint32_t)cur_ix;
|
|
229
230
|
}
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
/* NOLINT(build/header_guard) */
|
|
2
|
+
/* Copyright 2018 Google Inc. All Rights Reserved.
|
|
3
|
+
|
|
4
|
+
Distributed under MIT license.
|
|
5
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/* template parameters: FN, JUMP, NUMBUCKETS, MASK, CHUNKLEN */
|
|
9
|
+
/* NUMBUCKETS / (MASK + 1) = probability of storing and using hash code. */
|
|
10
|
+
/* JUMP = skip bytes for speedup */
|
|
11
|
+
|
|
12
|
+
/* Rolling hash for long distance long string matches. Stores one position
|
|
13
|
+
per bucket, bucket key is computed over a long region. */
|
|
14
|
+
|
|
15
|
+
#define HashRolling HASHER()
|
|
16
|
+
|
|
17
|
+
static const uint32_t FN(kRollingHashMul32) = 69069;
|
|
18
|
+
static const uint32_t FN(kInvalidPos) = 0xffffffff;
|
|
19
|
+
|
|
20
|
+
/* This hasher uses a longer forward length, but returning a higher value here
|
|
21
|
+
will hurt compression by the main hasher when combined with a composite
|
|
22
|
+
hasher. The hasher tests for forward itself instead. */
|
|
23
|
+
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
|
|
24
|
+
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
|
|
25
|
+
|
|
26
|
+
/* Computes a code from a single byte. A lookup table of 256 values could be
|
|
27
|
+
used, but simply adding 1 works about as good. */
|
|
28
|
+
static uint32_t FN(HashByte)(uint8_t byte) {
|
|
29
|
+
return (uint32_t)byte + 1u;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
static uint32_t FN(HashRollingFunctionInitial)(uint32_t state, uint8_t add,
|
|
33
|
+
uint32_t factor) {
|
|
34
|
+
return (uint32_t)(factor * state + FN(HashByte)(add));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
static uint32_t FN(HashRollingFunction)(uint32_t state, uint8_t add,
|
|
38
|
+
uint8_t rem, uint32_t factor,
|
|
39
|
+
uint32_t factor_remove) {
|
|
40
|
+
return (uint32_t)(factor * state +
|
|
41
|
+
FN(HashByte)(add) - factor_remove * FN(HashByte)(rem));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
typedef struct HashRolling {
|
|
45
|
+
uint32_t state;
|
|
46
|
+
uint32_t* table;
|
|
47
|
+
size_t next_ix;
|
|
48
|
+
|
|
49
|
+
uint32_t chunk_len;
|
|
50
|
+
uint32_t factor;
|
|
51
|
+
uint32_t factor_remove;
|
|
52
|
+
} HashRolling;
|
|
53
|
+
|
|
54
|
+
static BROTLI_INLINE HashRolling* FN(Self)(HasherHandle handle) {
|
|
55
|
+
return (HashRolling*)&(GetHasherCommon(handle)[1]);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
static void FN(Initialize)(
|
|
59
|
+
HasherHandle handle, const BrotliEncoderParams* params) {
|
|
60
|
+
HashRolling* self = FN(Self)(handle);
|
|
61
|
+
size_t i;
|
|
62
|
+
self->state = 0;
|
|
63
|
+
self->next_ix = 0;
|
|
64
|
+
|
|
65
|
+
self->factor = FN(kRollingHashMul32);
|
|
66
|
+
|
|
67
|
+
/* Compute the factor of the oldest byte to remove: factor**steps modulo
|
|
68
|
+
0xffffffff (the multiplications rely on 32-bit overflow) */
|
|
69
|
+
self->factor_remove = 1;
|
|
70
|
+
for (i = 0; i < CHUNKLEN; i += JUMP) {
|
|
71
|
+
self->factor_remove *= self->factor;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
self->table = (uint32_t*)((HasherHandle)self + sizeof(HashRolling));
|
|
75
|
+
for (i = 0; i < NUMBUCKETS; i++) {
|
|
76
|
+
self->table[i] = FN(kInvalidPos);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
BROTLI_UNUSED(params);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
|
|
83
|
+
size_t input_size, const uint8_t* data) {
|
|
84
|
+
HashRolling* self = FN(Self)(handle);
|
|
85
|
+
size_t i;
|
|
86
|
+
/* Too small size, cannot use this hasher. */
|
|
87
|
+
if (input_size < CHUNKLEN) return;
|
|
88
|
+
self->state = 0;
|
|
89
|
+
for (i = 0; i < CHUNKLEN; i += JUMP) {
|
|
90
|
+
self->state = FN(HashRollingFunctionInitial)(
|
|
91
|
+
self->state, data[i], self->factor);
|
|
92
|
+
}
|
|
93
|
+
BROTLI_UNUSED(one_shot);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
|
97
|
+
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
|
98
|
+
size_t input_size) {
|
|
99
|
+
return sizeof(HashRolling) + NUMBUCKETS * sizeof(uint32_t);
|
|
100
|
+
BROTLI_UNUSED(params);
|
|
101
|
+
BROTLI_UNUSED(one_shot);
|
|
102
|
+
BROTLI_UNUSED(input_size);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
|
|
106
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
|
|
107
|
+
BROTLI_UNUSED(handle);
|
|
108
|
+
BROTLI_UNUSED(data);
|
|
109
|
+
BROTLI_UNUSED(mask);
|
|
110
|
+
BROTLI_UNUSED(ix);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
|
114
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
|
115
|
+
const size_t ix_end) {
|
|
116
|
+
BROTLI_UNUSED(handle);
|
|
117
|
+
BROTLI_UNUSED(data);
|
|
118
|
+
BROTLI_UNUSED(mask);
|
|
119
|
+
BROTLI_UNUSED(ix_start);
|
|
120
|
+
BROTLI_UNUSED(ix_end);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
|
|
124
|
+
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
|
125
|
+
size_t ring_buffer_mask) {
|
|
126
|
+
/* In this case we must re-initialize the hasher from scratch from the
|
|
127
|
+
current position. */
|
|
128
|
+
HashRolling* self = FN(Self)(handle);
|
|
129
|
+
size_t position_masked;
|
|
130
|
+
size_t available = num_bytes;
|
|
131
|
+
if ((position & (JUMP - 1)) != 0) {
|
|
132
|
+
size_t diff = JUMP - (position & (JUMP - 1));
|
|
133
|
+
available = (diff > available) ? 0 : (available - diff);
|
|
134
|
+
position += diff;
|
|
135
|
+
}
|
|
136
|
+
position_masked = position & ring_buffer_mask;
|
|
137
|
+
/* wrapping around ringbuffer not handled. */
|
|
138
|
+
if (available > ring_buffer_mask - position_masked) {
|
|
139
|
+
available = ring_buffer_mask - position_masked;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
FN(Prepare)(handle, BROTLI_FALSE, available,
|
|
143
|
+
ringbuffer + (position & ring_buffer_mask));
|
|
144
|
+
self->next_ix = position;
|
|
145
|
+
BROTLI_UNUSED(num_bytes);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
149
|
+
HasherHandle handle, int* BROTLI_RESTRICT distance_cache) {
|
|
150
|
+
BROTLI_UNUSED(handle);
|
|
151
|
+
BROTLI_UNUSED(distance_cache);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
155
|
+
const BrotliEncoderDictionary* dictionary,
|
|
156
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
|
157
|
+
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
|
158
|
+
const size_t max_length, const size_t max_backward, const size_t gap,
|
|
159
|
+
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
|
|
160
|
+
HashRolling* self = FN(Self)(handle);
|
|
161
|
+
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
|
162
|
+
size_t pos = self->next_ix;
|
|
163
|
+
|
|
164
|
+
if ((cur_ix & (JUMP - 1)) != 0) return;
|
|
165
|
+
|
|
166
|
+
/* Not enough lookahead */
|
|
167
|
+
if (max_length < CHUNKLEN) return;
|
|
168
|
+
|
|
169
|
+
for (pos = self->next_ix; pos <= cur_ix; pos += JUMP) {
|
|
170
|
+
uint32_t code = self->state & MASK;
|
|
171
|
+
|
|
172
|
+
uint8_t rem = data[pos & ring_buffer_mask];
|
|
173
|
+
uint8_t add = data[(pos + CHUNKLEN) & ring_buffer_mask];
|
|
174
|
+
size_t found_ix = FN(kInvalidPos);
|
|
175
|
+
|
|
176
|
+
self->state = FN(HashRollingFunction)(
|
|
177
|
+
self->state, add, rem, self->factor, self->factor_remove);
|
|
178
|
+
|
|
179
|
+
if (code < NUMBUCKETS) {
|
|
180
|
+
found_ix = self->table[code];
|
|
181
|
+
self->table[code] = (uint32_t)pos;
|
|
182
|
+
if (pos == cur_ix && found_ix != FN(kInvalidPos)) {
|
|
183
|
+
/* The cast to 32-bit makes backward distances up to 4GB work even
|
|
184
|
+
if cur_ix is above 4GB, despite using 32-bit values in the table. */
|
|
185
|
+
size_t backward = (uint32_t)(cur_ix - found_ix);
|
|
186
|
+
if (backward <= max_backward) {
|
|
187
|
+
const size_t found_ix_masked = found_ix & ring_buffer_mask;
|
|
188
|
+
const size_t len = FindMatchLengthWithLimit(&data[found_ix_masked],
|
|
189
|
+
&data[cur_ix_masked],
|
|
190
|
+
max_length);
|
|
191
|
+
if (len >= 4 && len > out->len) {
|
|
192
|
+
score_t score = BackwardReferenceScore(len, backward);
|
|
193
|
+
if (score > out->score) {
|
|
194
|
+
out->len = len;
|
|
195
|
+
out->distance = backward;
|
|
196
|
+
out->score = score;
|
|
197
|
+
out->len_code_delta = 0;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
self->next_ix = cur_ix + JUMP;
|
|
206
|
+
|
|
207
|
+
/* NOTE: this hasher does not search in the dictionary. It is used as
|
|
208
|
+
backup-hasher, the main hasher already searches in it. */
|
|
209
|
+
BROTLI_UNUSED(dictionary);
|
|
210
|
+
BROTLI_UNUSED(distance_cache);
|
|
211
|
+
BROTLI_UNUSED(gap);
|
|
212
|
+
BROTLI_UNUSED(max_distance);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
#undef HashRolling
|
|
@@ -24,8 +24,8 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
|
|
|
24
24
|
return MAX_TREE_COMP_LENGTH;
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
static uint32_t FN(HashBytes)(const uint8_t
|
|
28
|
-
uint32_t h =
|
|
27
|
+
static uint32_t FN(HashBytes)(const uint8_t* data) {
|
|
28
|
+
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
|
29
29
|
/* The higher bits contain more mixture from the multiplication,
|
|
30
30
|
so we take our results from there. */
|
|
31
31
|
return h >> (32 - BUCKET_BITS);
|
|
@@ -154,12 +154,13 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
|
154
154
|
{
|
|
155
155
|
const size_t cur_len = BROTLI_MIN(size_t, best_len_left, best_len_right);
|
|
156
156
|
size_t len;
|
|
157
|
-
|
|
157
|
+
BROTLI_DCHECK(cur_len <= MAX_TREE_COMP_LENGTH);
|
|
158
158
|
len = cur_len +
|
|
159
159
|
FindMatchLengthWithLimit(&data[cur_ix_masked + cur_len],
|
|
160
160
|
&data[prev_ix_masked + cur_len],
|
|
161
161
|
max_length - cur_len);
|
|
162
|
-
|
|
162
|
+
BROTLI_DCHECK(
|
|
163
|
+
0 == memcmp(&data[cur_ix_masked], &data[prev_ix_masked], len));
|
|
163
164
|
if (matches && len > *best_len) {
|
|
164
165
|
*best_len = len;
|
|
165
166
|
InitBackwardMatch(matches++, backward, len);
|
|
@@ -199,7 +200,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
|
199
200
|
sorted by strictly increasing length and (non-strictly) increasing
|
|
200
201
|
distance. */
|
|
201
202
|
static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
202
|
-
const
|
|
203
|
+
const BrotliEncoderDictionary* dictionary, const uint8_t* data,
|
|
203
204
|
const size_t ring_buffer_mask, const size_t cur_ix,
|
|
204
205
|
const size_t max_length, const size_t max_backward, const size_t gap,
|
|
205
206
|
const BrotliEncoderParams* params, BackwardMatch* matches) {
|
|
@@ -251,7 +252,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
|
251
252
|
uint32_t dict_id = dict_matches[l];
|
|
252
253
|
if (dict_id < kInvalidMatch) {
|
|
253
254
|
size_t distance = max_backward + gap + (dict_id >> 5) + 1;
|
|
254
|
-
if (distance
|
|
255
|
+
if (distance <= params->dist.max_distance) {
|
|
255
256
|
InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
|
|
256
257
|
}
|
|
257
258
|
}
|
|
@@ -264,7 +265,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
|
264
265
|
/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
|
|
265
266
|
current sequence, without returning any matches.
|
|
266
267
|
REQUIRES: ix + MAX_TREE_COMP_LENGTH <= end-of-current-block */
|
|
267
|
-
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t
|
|
268
|
+
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t* data,
|
|
268
269
|
const size_t mask, const size_t ix) {
|
|
269
270
|
HashToBinaryTree* self = FN(Self)(handle);
|
|
270
271
|
/* Maximum distance is window size - 16, see section 9.1. of the spec. */
|
|
@@ -274,7 +275,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t *data,
|
|
|
274
275
|
}
|
|
275
276
|
|
|
276
277
|
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
|
277
|
-
const uint8_t
|
|
278
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
|
278
279
|
const size_t ix_end) {
|
|
279
280
|
size_t i = ix_start;
|
|
280
281
|
size_t j = ix_start;
|