brotli 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Gemfile +1 -0
- data/Rakefile +6 -1
- data/brotli.gemspec +1 -1
- data/docs/Brotli.html +485 -0
- data/docs/Brotli/Error.html +124 -0
- data/docs/_index.html +122 -0
- data/docs/class_list.html +51 -0
- data/docs/css/common.css +1 -0
- data/docs/css/full_list.css +58 -0
- data/docs/css/style.css +496 -0
- data/docs/file.README.html +127 -0
- data/docs/file_list.html +56 -0
- data/docs/frames.html +17 -0
- data/docs/index.html +127 -0
- data/docs/js/app.js +292 -0
- data/docs/js/full_list.js +216 -0
- data/docs/js/jquery.js +4 -0
- data/docs/method_list.html +67 -0
- data/docs/top-level-namespace.html +110 -0
- data/ext/brotli/brotli.c +20 -0
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/c/common/constants.h +13 -6
- data/vendor/brotli/c/{dec → common}/context.h +182 -172
- data/vendor/brotli/c/common/dictionary.bin +0 -0
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +1 -1
- data/vendor/brotli/c/common/dictionary.h +4 -4
- data/vendor/brotli/c/common/platform.h +509 -0
- data/vendor/brotli/c/common/transform.c +235 -0
- data/vendor/brotli/c/common/transform.h +80 -0
- data/vendor/brotli/c/common/version.h +8 -1
- data/vendor/brotli/c/dec/bit_reader.c +1 -1
- data/vendor/brotli/c/dec/bit_reader.h +35 -86
- data/vendor/brotli/c/dec/decode.c +322 -205
- data/vendor/brotli/c/dec/huffman.c +35 -37
- data/vendor/brotli/c/dec/huffman.h +13 -9
- data/vendor/brotli/c/dec/prefix.h +3 -4
- data/vendor/brotli/c/dec/state.c +26 -34
- data/vendor/brotli/c/dec/state.h +34 -23
- data/vendor/brotli/c/enc/backward_references.c +25 -15
- data/vendor/brotli/c/enc/backward_references.h +5 -6
- data/vendor/brotli/c/enc/backward_references_hq.c +94 -68
- data/vendor/brotli/c/enc/backward_references_hq.h +22 -25
- data/vendor/brotli/c/enc/backward_references_inc.h +10 -10
- data/vendor/brotli/c/enc/bit_cost.c +1 -1
- data/vendor/brotli/c/enc/bit_cost.h +5 -5
- data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
- data/vendor/brotli/c/enc/block_splitter.c +2 -3
- data/vendor/brotli/c/enc/block_splitter.h +1 -1
- data/vendor/brotli/c/enc/block_splitter_inc.h +11 -11
- data/vendor/brotli/c/enc/brotli_bit_stream.c +102 -101
- data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
- data/vendor/brotli/c/enc/cluster.c +1 -1
- data/vendor/brotli/c/enc/cluster.h +1 -1
- data/vendor/brotli/c/enc/command.h +40 -30
- data/vendor/brotli/c/enc/compress_fragment.c +21 -22
- data/vendor/brotli/c/enc/compress_fragment.h +1 -1
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +101 -68
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1 -1
- data/vendor/brotli/c/enc/encode.c +262 -162
- data/vendor/brotli/c/enc/encoder_dict.c +32 -0
- data/vendor/brotli/c/enc/encoder_dict.h +41 -0
- data/vendor/brotli/c/enc/entropy_encode.c +14 -14
- data/vendor/brotli/c/enc/entropy_encode.h +5 -5
- data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
- data/vendor/brotli/c/enc/fast_log.h +4 -2
- data/vendor/brotli/c/enc/find_match_length.h +3 -3
- data/vendor/brotli/c/enc/hash.h +75 -24
- data/vendor/brotli/c/enc/hash_composite_inc.h +133 -0
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +9 -8
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +10 -9
- data/vendor/brotli/c/enc/hash_rolling_inc.h +215 -0
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +9 -8
- data/vendor/brotli/c/enc/histogram.c +9 -6
- data/vendor/brotli/c/enc/histogram.h +6 -3
- data/vendor/brotli/c/enc/histogram_inc.h +1 -1
- data/vendor/brotli/c/enc/literal_cost.c +5 -5
- data/vendor/brotli/c/enc/literal_cost.h +2 -2
- data/vendor/brotli/c/enc/memory.c +5 -16
- data/vendor/brotli/c/enc/memory.h +40 -1
- data/vendor/brotli/c/enc/metablock.c +163 -25
- data/vendor/brotli/c/enc/metablock.h +13 -8
- data/vendor/brotli/c/enc/metablock_inc.h +1 -1
- data/vendor/brotli/c/enc/params.h +44 -0
- data/vendor/brotli/c/enc/prefix.h +3 -4
- data/vendor/brotli/c/enc/quality.h +29 -24
- data/vendor/brotli/c/enc/ringbuffer.h +15 -11
- data/vendor/brotli/c/enc/static_dict.c +49 -45
- data/vendor/brotli/c/enc/static_dict.h +4 -3
- data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
- data/vendor/brotli/c/enc/utf8_util.c +20 -20
- data/vendor/brotli/c/enc/utf8_util.h +1 -1
- data/vendor/brotli/c/enc/write_bits.h +16 -21
- data/vendor/brotli/c/include/brotli/decode.h +13 -8
- data/vendor/brotli/c/include/brotli/encode.h +33 -8
- data/vendor/brotli/c/include/brotli/port.h +211 -83
- data/vendor/brotli/c/include/brotli/types.h +0 -7
- metadata +33 -12
- data/vendor/brotli/c/dec/port.h +0 -168
- data/vendor/brotli/c/dec/transform.h +0 -300
- data/vendor/brotli/c/enc/context.h +0 -184
- data/vendor/brotli/c/enc/port.h +0 -184
@@ -28,8 +28,8 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
|
|
28
28
|
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
|
29
29
|
|
30
30
|
/* HashBytes is the function that chooses the bucket to place the address in.*/
|
31
|
-
static BROTLI_INLINE size_t FN(HashBytes)(const uint8_t
|
32
|
-
const uint32_t h =
|
31
|
+
static BROTLI_INLINE size_t FN(HashBytes)(const uint8_t* data) {
|
32
|
+
const uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
33
33
|
/* The higher bits contain more mixture from the multiplication,
|
34
34
|
so we take our results from there. */
|
35
35
|
return h >> (32 - BUCKET_BITS);
|
@@ -115,7 +115,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
|
|
115
115
|
}
|
116
116
|
|
117
117
|
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
118
|
-
const uint8_t
|
118
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
119
119
|
const size_t ix_end) {
|
120
120
|
size_t i;
|
121
121
|
for (i = ix_start; i < ix_end; ++i) {
|
@@ -154,11 +154,12 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
154
154
|
Writes the best match into |out|.
|
155
155
|
|out|->score is updated only if a better match is found. */
|
156
156
|
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
157
|
-
const
|
157
|
+
const BrotliEncoderDictionary* dictionary,
|
158
158
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
159
159
|
const int* BROTLI_RESTRICT distance_cache,
|
160
160
|
const size_t cur_ix, const size_t max_length, const size_t max_backward,
|
161
|
-
const size_t gap,
|
161
|
+
const size_t gap, const size_t max_distance,
|
162
|
+
HasherSearchResult* BROTLI_RESTRICT out) {
|
162
163
|
HashForgetfulChain* self = FN(Self)(handle);
|
163
164
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
164
165
|
/* Don't accept a short copy from far away. */
|
@@ -240,9 +241,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
240
241
|
FN(Store)(handle, data, ring_buffer_mask, cur_ix);
|
241
242
|
}
|
242
243
|
if (out->score == min_score) {
|
243
|
-
SearchInStaticDictionary(dictionary,
|
244
|
-
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
245
|
-
BROTLI_FALSE);
|
244
|
+
SearchInStaticDictionary(dictionary,
|
245
|
+
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
246
|
+
max_distance, out, BROTLI_FALSE);
|
246
247
|
}
|
247
248
|
}
|
248
249
|
|
@@ -20,7 +20,7 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
|
|
20
20
|
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
|
21
21
|
|
22
22
|
/* HashBytes is the function that chooses the bucket to place the address in. */
|
23
|
-
static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t
|
23
|
+
static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t* data,
|
24
24
|
const uint64_t mask,
|
25
25
|
const int shift) {
|
26
26
|
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(data) & mask) * kHashMul64Long;
|
@@ -105,7 +105,7 @@ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
|
105
105
|
|
106
106
|
/* Look at 4 bytes at &data[ix & mask].
|
107
107
|
Compute a hash from these, and store the value of ix at that position. */
|
108
|
-
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t
|
108
|
+
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t* data,
|
109
109
|
const size_t mask, const size_t ix) {
|
110
110
|
HashLongestMatch* self = FN(Self)(handle);
|
111
111
|
uint16_t* num = FN(Num)(self);
|
@@ -119,7 +119,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t *data,
|
|
119
119
|
}
|
120
120
|
|
121
121
|
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
122
|
-
const uint8_t
|
122
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
123
123
|
const size_t ix_end) {
|
124
124
|
size_t i;
|
125
125
|
for (i = ix_start; i < ix_end; ++i) {
|
@@ -158,11 +158,11 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
158
158
|
Writes the best match into |out|.
|
159
159
|
|out|->score is updated only if a better match is found. */
|
160
160
|
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
161
|
-
const
|
161
|
+
const BrotliEncoderDictionary* dictionary,
|
162
162
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
163
163
|
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
164
164
|
const size_t max_length, const size_t max_backward, const size_t gap,
|
165
|
-
HasherSearchResult* BROTLI_RESTRICT out) {
|
165
|
+
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
|
166
166
|
HasherCommon* common = GetHasherCommon(handle);
|
167
167
|
HashLongestMatch* self = FN(Self)(handle);
|
168
168
|
uint16_t* num = FN(Num)(self);
|
@@ -257,9 +257,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
257
257
|
++num[key];
|
258
258
|
}
|
259
259
|
if (min_score == out->score) {
|
260
|
-
SearchInStaticDictionary(dictionary,
|
261
|
-
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
262
|
-
BROTLI_FALSE);
|
260
|
+
SearchInStaticDictionary(dictionary,
|
261
|
+
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
262
|
+
max_distance, out, BROTLI_FALSE);
|
263
263
|
}
|
264
264
|
}
|
265
265
|
|
@@ -20,8 +20,8 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
|
|
20
20
|
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
|
21
21
|
|
22
22
|
/* HashBytes is the function that chooses the bucket to place the address in. */
|
23
|
-
static uint32_t FN(HashBytes)(const uint8_t
|
24
|
-
uint32_t h =
|
23
|
+
static uint32_t FN(HashBytes)(const uint8_t* data, const int shift) {
|
24
|
+
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
25
25
|
/* The higher bits contain more mixture from the multiplication,
|
26
26
|
so we take our results from there. */
|
27
27
|
return (uint32_t)(h >> shift);
|
@@ -112,7 +112,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t* data,
|
|
112
112
|
}
|
113
113
|
|
114
114
|
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
115
|
-
const uint8_t
|
115
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
116
116
|
const size_t ix_end) {
|
117
117
|
size_t i;
|
118
118
|
for (i = ix_start; i < ix_end; ++i) {
|
@@ -151,11 +151,11 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
151
151
|
Writes the best match into |out|.
|
152
152
|
|out|->score is updated only if a better match is found. */
|
153
153
|
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
154
|
-
const
|
154
|
+
const BrotliEncoderDictionary* dictionary,
|
155
155
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
156
156
|
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
157
157
|
const size_t max_length, const size_t max_backward, const size_t gap,
|
158
|
-
HasherSearchResult* BROTLI_RESTRICT out) {
|
158
|
+
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
|
159
159
|
HasherCommon* common = GetHasherCommon(handle);
|
160
160
|
HashLongestMatch* self = FN(Self)(handle);
|
161
161
|
uint16_t* num = FN(Num)(self);
|
@@ -249,9 +249,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
249
249
|
++num[key];
|
250
250
|
}
|
251
251
|
if (min_score == out->score) {
|
252
|
-
SearchInStaticDictionary(dictionary,
|
253
|
-
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
254
|
-
BROTLI_FALSE);
|
252
|
+
SearchInStaticDictionary(dictionary,
|
253
|
+
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
254
|
+
max_distance, out, BROTLI_FALSE);
|
255
255
|
}
|
256
256
|
}
|
257
257
|
|
@@ -81,7 +81,7 @@ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
|
81
81
|
Compute a hash from these, and store the value somewhere within
|
82
82
|
[ix .. ix+3]. */
|
83
83
|
static BROTLI_INLINE void FN(Store)(HasherHandle handle,
|
84
|
-
const uint8_t
|
84
|
+
const uint8_t* data, const size_t mask, const size_t ix) {
|
85
85
|
const uint32_t key = FN(HashBytes)(&data[ix & mask]);
|
86
86
|
/* Wiggle the value with the bucket sweep range. */
|
87
87
|
const uint32_t off = (ix >> 3) % BUCKET_SWEEP;
|
@@ -89,7 +89,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle,
|
|
89
89
|
}
|
90
90
|
|
91
91
|
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
92
|
-
const uint8_t
|
92
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
93
93
|
const size_t ix_end) {
|
94
94
|
size_t i;
|
95
95
|
for (i = ix_start; i < ix_end; ++i) {
|
@@ -125,11 +125,12 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
125
125
|
Writes the best match into |out|.
|
126
126
|
|out|->score is updated only if a better match is found. */
|
127
127
|
static BROTLI_INLINE void FN(FindLongestMatch)(
|
128
|
-
HasherHandle handle, const
|
129
|
-
const
|
128
|
+
HasherHandle handle, const BrotliEncoderDictionary* dictionary,
|
129
|
+
const uint8_t* BROTLI_RESTRICT data,
|
130
130
|
const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
|
131
131
|
const size_t cur_ix, const size_t max_length, const size_t max_backward,
|
132
|
-
const size_t gap,
|
132
|
+
const size_t gap, const size_t max_distance,
|
133
|
+
HasherSearchResult* BROTLI_RESTRICT out) {
|
133
134
|
HashLongestMatchQuickly* self = FN(Self)(handle);
|
134
135
|
const size_t best_len_in = out->len;
|
135
136
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
@@ -191,7 +192,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
|
|
191
192
|
}
|
192
193
|
}
|
193
194
|
} else {
|
194
|
-
uint32_t
|
195
|
+
uint32_t* bucket = self->buckets_ + key;
|
195
196
|
int i;
|
196
197
|
prev_ix = *bucket++;
|
197
198
|
for (i = 0; i < BUCKET_SWEEP; ++i, prev_ix = *bucket++) {
|
@@ -221,9 +222,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
|
|
221
222
|
}
|
222
223
|
}
|
223
224
|
if (USE_DICTIONARY && min_score == out->score) {
|
224
|
-
SearchInStaticDictionary(dictionary,
|
225
|
-
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
226
|
-
BROTLI_TRUE);
|
225
|
+
SearchInStaticDictionary(dictionary,
|
226
|
+
handle, &data[cur_ix_masked], max_length, max_backward + gap,
|
227
|
+
max_distance, out, BROTLI_TRUE);
|
227
228
|
}
|
228
229
|
self->buckets_[key + ((cur_ix >> 3) % BUCKET_SWEEP)] = (uint32_t)cur_ix;
|
229
230
|
}
|
@@ -0,0 +1,215 @@
|
|
1
|
+
/* NOLINT(build/header_guard) */
|
2
|
+
/* Copyright 2018 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Distributed under MIT license.
|
5
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
6
|
+
*/
|
7
|
+
|
8
|
+
/* template parameters: FN, JUMP, NUMBUCKETS, MASK, CHUNKLEN */
|
9
|
+
/* NUMBUCKETS / (MASK + 1) = probability of storing and using hash code. */
|
10
|
+
/* JUMP = skip bytes for speedup */
|
11
|
+
|
12
|
+
/* Rolling hash for long distance long string matches. Stores one position
|
13
|
+
per bucket, bucket key is computed over a long region. */
|
14
|
+
|
15
|
+
#define HashRolling HASHER()
|
16
|
+
|
17
|
+
static const uint32_t FN(kRollingHashMul32) = 69069;
|
18
|
+
static const uint32_t FN(kInvalidPos) = 0xffffffff;
|
19
|
+
|
20
|
+
/* This hasher uses a longer forward length, but returning a higher value here
|
21
|
+
will hurt compression by the main hasher when combined with a composite
|
22
|
+
hasher. The hasher tests for forward itself instead. */
|
23
|
+
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
|
24
|
+
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
|
25
|
+
|
26
|
+
/* Computes a code from a single byte. A lookup table of 256 values could be
|
27
|
+
used, but simply adding 1 works about as good. */
|
28
|
+
static uint32_t FN(HashByte)(uint8_t byte) {
|
29
|
+
return (uint32_t)byte + 1u;
|
30
|
+
}
|
31
|
+
|
32
|
+
static uint32_t FN(HashRollingFunctionInitial)(uint32_t state, uint8_t add,
|
33
|
+
uint32_t factor) {
|
34
|
+
return (uint32_t)(factor * state + FN(HashByte)(add));
|
35
|
+
}
|
36
|
+
|
37
|
+
static uint32_t FN(HashRollingFunction)(uint32_t state, uint8_t add,
|
38
|
+
uint8_t rem, uint32_t factor,
|
39
|
+
uint32_t factor_remove) {
|
40
|
+
return (uint32_t)(factor * state +
|
41
|
+
FN(HashByte)(add) - factor_remove * FN(HashByte)(rem));
|
42
|
+
}
|
43
|
+
|
44
|
+
typedef struct HashRolling {
|
45
|
+
uint32_t state;
|
46
|
+
uint32_t* table;
|
47
|
+
size_t next_ix;
|
48
|
+
|
49
|
+
uint32_t chunk_len;
|
50
|
+
uint32_t factor;
|
51
|
+
uint32_t factor_remove;
|
52
|
+
} HashRolling;
|
53
|
+
|
54
|
+
static BROTLI_INLINE HashRolling* FN(Self)(HasherHandle handle) {
|
55
|
+
return (HashRolling*)&(GetHasherCommon(handle)[1]);
|
56
|
+
}
|
57
|
+
|
58
|
+
static void FN(Initialize)(
|
59
|
+
HasherHandle handle, const BrotliEncoderParams* params) {
|
60
|
+
HashRolling* self = FN(Self)(handle);
|
61
|
+
size_t i;
|
62
|
+
self->state = 0;
|
63
|
+
self->next_ix = 0;
|
64
|
+
|
65
|
+
self->factor = FN(kRollingHashMul32);
|
66
|
+
|
67
|
+
/* Compute the factor of the oldest byte to remove: factor**steps modulo
|
68
|
+
0xffffffff (the multiplications rely on 32-bit overflow) */
|
69
|
+
self->factor_remove = 1;
|
70
|
+
for (i = 0; i < CHUNKLEN; i += JUMP) {
|
71
|
+
self->factor_remove *= self->factor;
|
72
|
+
}
|
73
|
+
|
74
|
+
self->table = (uint32_t*)((HasherHandle)self + sizeof(HashRolling));
|
75
|
+
for (i = 0; i < NUMBUCKETS; i++) {
|
76
|
+
self->table[i] = FN(kInvalidPos);
|
77
|
+
}
|
78
|
+
|
79
|
+
BROTLI_UNUSED(params);
|
80
|
+
}
|
81
|
+
|
82
|
+
static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
|
83
|
+
size_t input_size, const uint8_t* data) {
|
84
|
+
HashRolling* self = FN(Self)(handle);
|
85
|
+
size_t i;
|
86
|
+
/* Too small size, cannot use this hasher. */
|
87
|
+
if (input_size < CHUNKLEN) return;
|
88
|
+
self->state = 0;
|
89
|
+
for (i = 0; i < CHUNKLEN; i += JUMP) {
|
90
|
+
self->state = FN(HashRollingFunctionInitial)(
|
91
|
+
self->state, data[i], self->factor);
|
92
|
+
}
|
93
|
+
BROTLI_UNUSED(one_shot);
|
94
|
+
}
|
95
|
+
|
96
|
+
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
97
|
+
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
98
|
+
size_t input_size) {
|
99
|
+
return sizeof(HashRolling) + NUMBUCKETS * sizeof(uint32_t);
|
100
|
+
BROTLI_UNUSED(params);
|
101
|
+
BROTLI_UNUSED(one_shot);
|
102
|
+
BROTLI_UNUSED(input_size);
|
103
|
+
}
|
104
|
+
|
105
|
+
static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
|
106
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
|
107
|
+
BROTLI_UNUSED(handle);
|
108
|
+
BROTLI_UNUSED(data);
|
109
|
+
BROTLI_UNUSED(mask);
|
110
|
+
BROTLI_UNUSED(ix);
|
111
|
+
}
|
112
|
+
|
113
|
+
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
114
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
115
|
+
const size_t ix_end) {
|
116
|
+
BROTLI_UNUSED(handle);
|
117
|
+
BROTLI_UNUSED(data);
|
118
|
+
BROTLI_UNUSED(mask);
|
119
|
+
BROTLI_UNUSED(ix_start);
|
120
|
+
BROTLI_UNUSED(ix_end);
|
121
|
+
}
|
122
|
+
|
123
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
|
124
|
+
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
125
|
+
size_t ring_buffer_mask) {
|
126
|
+
/* In this case we must re-initialize the hasher from scratch from the
|
127
|
+
current position. */
|
128
|
+
HashRolling* self = FN(Self)(handle);
|
129
|
+
size_t position_masked;
|
130
|
+
size_t available = num_bytes;
|
131
|
+
if ((position & (JUMP - 1)) != 0) {
|
132
|
+
size_t diff = JUMP - (position & (JUMP - 1));
|
133
|
+
available = (diff > available) ? 0 : (available - diff);
|
134
|
+
position += diff;
|
135
|
+
}
|
136
|
+
position_masked = position & ring_buffer_mask;
|
137
|
+
/* wrapping around ringbuffer not handled. */
|
138
|
+
if (available > ring_buffer_mask - position_masked) {
|
139
|
+
available = ring_buffer_mask - position_masked;
|
140
|
+
}
|
141
|
+
|
142
|
+
FN(Prepare)(handle, BROTLI_FALSE, available,
|
143
|
+
ringbuffer + (position & ring_buffer_mask));
|
144
|
+
self->next_ix = position;
|
145
|
+
BROTLI_UNUSED(num_bytes);
|
146
|
+
}
|
147
|
+
|
148
|
+
static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
149
|
+
HasherHandle handle, int* BROTLI_RESTRICT distance_cache) {
|
150
|
+
BROTLI_UNUSED(handle);
|
151
|
+
BROTLI_UNUSED(distance_cache);
|
152
|
+
}
|
153
|
+
|
154
|
+
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
155
|
+
const BrotliEncoderDictionary* dictionary,
|
156
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
157
|
+
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
158
|
+
const size_t max_length, const size_t max_backward, const size_t gap,
|
159
|
+
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
|
160
|
+
HashRolling* self = FN(Self)(handle);
|
161
|
+
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
162
|
+
size_t pos = self->next_ix;
|
163
|
+
|
164
|
+
if ((cur_ix & (JUMP - 1)) != 0) return;
|
165
|
+
|
166
|
+
/* Not enough lookahead */
|
167
|
+
if (max_length < CHUNKLEN) return;
|
168
|
+
|
169
|
+
for (pos = self->next_ix; pos <= cur_ix; pos += JUMP) {
|
170
|
+
uint32_t code = self->state & MASK;
|
171
|
+
|
172
|
+
uint8_t rem = data[pos & ring_buffer_mask];
|
173
|
+
uint8_t add = data[(pos + CHUNKLEN) & ring_buffer_mask];
|
174
|
+
size_t found_ix = FN(kInvalidPos);
|
175
|
+
|
176
|
+
self->state = FN(HashRollingFunction)(
|
177
|
+
self->state, add, rem, self->factor, self->factor_remove);
|
178
|
+
|
179
|
+
if (code < NUMBUCKETS) {
|
180
|
+
found_ix = self->table[code];
|
181
|
+
self->table[code] = (uint32_t)pos;
|
182
|
+
if (pos == cur_ix && found_ix != FN(kInvalidPos)) {
|
183
|
+
/* The cast to 32-bit makes backward distances up to 4GB work even
|
184
|
+
if cur_ix is above 4GB, despite using 32-bit values in the table. */
|
185
|
+
size_t backward = (uint32_t)(cur_ix - found_ix);
|
186
|
+
if (backward <= max_backward) {
|
187
|
+
const size_t found_ix_masked = found_ix & ring_buffer_mask;
|
188
|
+
const size_t len = FindMatchLengthWithLimit(&data[found_ix_masked],
|
189
|
+
&data[cur_ix_masked],
|
190
|
+
max_length);
|
191
|
+
if (len >= 4 && len > out->len) {
|
192
|
+
score_t score = BackwardReferenceScore(len, backward);
|
193
|
+
if (score > out->score) {
|
194
|
+
out->len = len;
|
195
|
+
out->distance = backward;
|
196
|
+
out->score = score;
|
197
|
+
out->len_code_delta = 0;
|
198
|
+
}
|
199
|
+
}
|
200
|
+
}
|
201
|
+
}
|
202
|
+
}
|
203
|
+
}
|
204
|
+
|
205
|
+
self->next_ix = cur_ix + JUMP;
|
206
|
+
|
207
|
+
/* NOTE: this hasher does not search in the dictionary. It is used as
|
208
|
+
backup-hasher, the main hasher already searches in it. */
|
209
|
+
BROTLI_UNUSED(dictionary);
|
210
|
+
BROTLI_UNUSED(distance_cache);
|
211
|
+
BROTLI_UNUSED(gap);
|
212
|
+
BROTLI_UNUSED(max_distance);
|
213
|
+
}
|
214
|
+
|
215
|
+
#undef HashRolling
|
@@ -24,8 +24,8 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
|
|
24
24
|
return MAX_TREE_COMP_LENGTH;
|
25
25
|
}
|
26
26
|
|
27
|
-
static uint32_t FN(HashBytes)(const uint8_t
|
28
|
-
uint32_t h =
|
27
|
+
static uint32_t FN(HashBytes)(const uint8_t* data) {
|
28
|
+
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
29
29
|
/* The higher bits contain more mixture from the multiplication,
|
30
30
|
so we take our results from there. */
|
31
31
|
return h >> (32 - BUCKET_BITS);
|
@@ -154,12 +154,13 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
154
154
|
{
|
155
155
|
const size_t cur_len = BROTLI_MIN(size_t, best_len_left, best_len_right);
|
156
156
|
size_t len;
|
157
|
-
|
157
|
+
BROTLI_DCHECK(cur_len <= MAX_TREE_COMP_LENGTH);
|
158
158
|
len = cur_len +
|
159
159
|
FindMatchLengthWithLimit(&data[cur_ix_masked + cur_len],
|
160
160
|
&data[prev_ix_masked + cur_len],
|
161
161
|
max_length - cur_len);
|
162
|
-
|
162
|
+
BROTLI_DCHECK(
|
163
|
+
0 == memcmp(&data[cur_ix_masked], &data[prev_ix_masked], len));
|
163
164
|
if (matches && len > *best_len) {
|
164
165
|
*best_len = len;
|
165
166
|
InitBackwardMatch(matches++, backward, len);
|
@@ -199,7 +200,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
199
200
|
sorted by strictly increasing length and (non-strictly) increasing
|
200
201
|
distance. */
|
201
202
|
static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
202
|
-
const
|
203
|
+
const BrotliEncoderDictionary* dictionary, const uint8_t* data,
|
203
204
|
const size_t ring_buffer_mask, const size_t cur_ix,
|
204
205
|
const size_t max_length, const size_t max_backward, const size_t gap,
|
205
206
|
const BrotliEncoderParams* params, BackwardMatch* matches) {
|
@@ -251,7 +252,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
251
252
|
uint32_t dict_id = dict_matches[l];
|
252
253
|
if (dict_id < kInvalidMatch) {
|
253
254
|
size_t distance = max_backward + gap + (dict_id >> 5) + 1;
|
254
|
-
if (distance
|
255
|
+
if (distance <= params->dist.max_distance) {
|
255
256
|
InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
|
256
257
|
}
|
257
258
|
}
|
@@ -264,7 +265,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
264
265
|
/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
|
265
266
|
current sequence, without returning any matches.
|
266
267
|
REQUIRES: ix + MAX_TREE_COMP_LENGTH <= end-of-current-block */
|
267
|
-
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t
|
268
|
+
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t* data,
|
268
269
|
const size_t mask, const size_t ix) {
|
269
270
|
HashToBinaryTree* self = FN(Self)(handle);
|
270
271
|
/* Maximum distance is window size - 16, see section 9.1. of the spec. */
|
@@ -274,7 +275,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t *data,
|
|
274
275
|
}
|
275
276
|
|
276
277
|
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
277
|
-
const uint8_t
|
278
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
278
279
|
const size_t ix_end) {
|
279
280
|
size_t i = ix_start;
|
280
281
|
size_t j = ix_start;
|