brotli 0.2.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +37 -0
- data/.github/workflows/publish.yml +24 -0
- data/.gitmodules +1 -1
- data/Gemfile +6 -3
- data/README.md +2 -2
- data/Rakefile +16 -9
- data/brotli.gemspec +7 -13
- data/ext/brotli/brotli.c +210 -31
- data/ext/brotli/buffer.c +1 -7
- data/ext/brotli/buffer.h +1 -1
- data/ext/brotli/extconf.rb +25 -17
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +107 -0
- data/test/brotli_writer_test.rb +36 -0
- data/test/test_helper.rb +8 -0
- data/vendor/brotli/c/common/constants.c +15 -0
- data/vendor/brotli/c/common/constants.h +137 -0
- data/vendor/brotli/c/common/context.c +156 -0
- data/vendor/brotli/c/common/context.h +4 -152
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +14 -3
- data/vendor/brotli/c/common/platform.c +23 -0
- data/vendor/brotli/c/common/platform.h +95 -122
- data/vendor/brotli/c/common/shared_dictionary.c +521 -0
- data/vendor/brotli/c/common/shared_dictionary_internal.h +75 -0
- data/vendor/brotli/c/common/transform.c +60 -4
- data/vendor/brotli/c/common/transform.h +5 -0
- data/vendor/brotli/c/common/version.h +31 -6
- data/vendor/brotli/c/dec/bit_reader.c +34 -4
- data/vendor/brotli/c/dec/bit_reader.h +221 -107
- data/vendor/brotli/c/dec/decode.c +772 -403
- data/vendor/brotli/c/dec/huffman.c +7 -4
- data/vendor/brotli/c/dec/huffman.h +8 -13
- data/vendor/brotli/c/dec/prefix.h +1 -18
- data/vendor/brotli/c/dec/state.c +40 -21
- data/vendor/brotli/c/dec/state.h +201 -59
- data/vendor/brotli/c/enc/backward_references.c +88 -25
- data/vendor/brotli/c/enc/backward_references.h +10 -8
- data/vendor/brotli/c/enc/backward_references_hq.c +194 -80
- data/vendor/brotli/c/enc/backward_references_hq.h +17 -13
- data/vendor/brotli/c/enc/backward_references_inc.h +52 -16
- data/vendor/brotli/c/enc/bit_cost.c +8 -7
- data/vendor/brotli/c/enc/bit_cost.h +5 -4
- data/vendor/brotli/c/enc/block_splitter.c +40 -17
- data/vendor/brotli/c/enc/block_splitter.h +5 -4
- data/vendor/brotli/c/enc/block_splitter_inc.h +99 -49
- data/vendor/brotli/c/enc/brotli_bit_stream.c +142 -137
- data/vendor/brotli/c/enc/brotli_bit_stream.h +11 -6
- data/vendor/brotli/c/enc/cluster.c +10 -9
- data/vendor/brotli/c/enc/cluster.h +7 -6
- data/vendor/brotli/c/enc/cluster_inc.h +30 -22
- data/vendor/brotli/c/enc/command.c +28 -0
- data/vendor/brotli/c/enc/command.h +17 -16
- data/vendor/brotli/c/enc/compound_dictionary.c +207 -0
- data/vendor/brotli/c/enc/compound_dictionary.h +74 -0
- data/vendor/brotli/c/enc/compress_fragment.c +93 -83
- data/vendor/brotli/c/enc/compress_fragment.h +32 -7
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +100 -88
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +21 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +1829 -1101
- data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
- data/vendor/brotli/c/enc/encode.c +550 -416
- data/vendor/brotli/c/enc/encoder_dict.c +613 -5
- data/vendor/brotli/c/enc/encoder_dict.h +120 -4
- data/vendor/brotli/c/enc/entropy_encode.c +5 -2
- data/vendor/brotli/c/enc/entropy_encode.h +4 -3
- data/vendor/brotli/c/enc/entropy_encode_static.h +5 -2
- data/vendor/brotli/c/enc/fast_log.c +105 -0
- data/vendor/brotli/c/enc/fast_log.h +21 -101
- data/vendor/brotli/c/enc/find_match_length.h +17 -25
- data/vendor/brotli/c/enc/hash.h +350 -120
- data/vendor/brotli/c/enc/hash_composite_inc.h +71 -67
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +92 -51
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +79 -84
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +53 -54
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +93 -62
- data/vendor/brotli/c/enc/hash_rolling_inc.h +25 -29
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +42 -40
- data/vendor/brotli/c/enc/histogram.c +4 -4
- data/vendor/brotli/c/enc/histogram.h +7 -6
- data/vendor/brotli/c/enc/literal_cost.c +20 -15
- data/vendor/brotli/c/enc/literal_cost.h +4 -2
- data/vendor/brotli/c/enc/memory.c +29 -5
- data/vendor/brotli/c/enc/memory.h +43 -14
- data/vendor/brotli/c/enc/metablock.c +95 -85
- data/vendor/brotli/c/enc/metablock.h +9 -8
- data/vendor/brotli/c/enc/metablock_inc.h +9 -7
- data/vendor/brotli/c/enc/params.h +7 -4
- data/vendor/brotli/c/enc/prefix.h +3 -2
- data/vendor/brotli/c/enc/quality.h +40 -3
- data/vendor/brotli/c/enc/ringbuffer.h +8 -4
- data/vendor/brotli/c/enc/state.h +104 -0
- data/vendor/brotli/c/enc/static_dict.c +60 -4
- data/vendor/brotli/c/enc/static_dict.h +3 -2
- data/vendor/brotli/c/enc/static_dict_lut.h +2 -0
- data/vendor/brotli/c/enc/utf8_util.c +2 -2
- data/vendor/brotli/c/enc/utf8_util.h +2 -1
- data/vendor/brotli/c/enc/write_bits.h +29 -26
- data/vendor/brotli/c/include/brotli/decode.h +67 -2
- data/vendor/brotli/c/include/brotli/encode.h +77 -3
- data/vendor/brotli/c/include/brotli/port.h +34 -3
- data/vendor/brotli/c/include/brotli/shared_dictionary.h +100 -0
- metadata +23 -97
- data/.travis.yml +0 -31
- data/docs/Brotli/Error.html +0 -124
- data/docs/Brotli.html +0 -485
- data/docs/_index.html +0 -122
- data/docs/class_list.html +0 -51
- data/docs/css/common.css +0 -1
- data/docs/css/full_list.css +0 -58
- data/docs/css/style.css +0 -496
- data/docs/file.README.html +0 -127
- data/docs/file_list.html +0 -56
- data/docs/frames.html +0 -17
- data/docs/index.html +0 -127
- data/docs/js/app.js +0 -292
- data/docs/js/full_list.js +0 -216
- data/docs/js/jquery.js +0 -4
- data/docs/method_list.html +0 -67
- data/docs/top-level-namespace.html +0 -110
- data/spec/brotli_spec.rb +0 -88
- data/spec/inflate_spec.rb +0 -75
- data/spec/spec_helper.rb +0 -4
@@ -5,15 +5,16 @@
|
|
5
5
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
6
6
|
*/
|
7
7
|
|
8
|
-
/* template parameters: FN, BUCKET_BITS,
|
8
|
+
/* template parameters: FN, BUCKET_BITS, BUCKET_SWEEP_BITS, HASH_LEN,
|
9
9
|
USE_DICTIONARY
|
10
10
|
*/
|
11
11
|
|
12
12
|
#define HashLongestMatchQuickly HASHER()
|
13
13
|
|
14
14
|
#define BUCKET_SIZE (1 << BUCKET_BITS)
|
15
|
-
|
16
|
-
#define
|
15
|
+
#define BUCKET_MASK (BUCKET_SIZE - 1)
|
16
|
+
#define BUCKET_SWEEP (1 << BUCKET_SWEEP_BITS)
|
17
|
+
#define BUCKET_SWEEP_MASK ((BUCKET_SWEEP - 1) << 3)
|
17
18
|
|
18
19
|
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
|
19
20
|
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
|
@@ -32,87 +33,106 @@ static uint32_t FN(HashBytes)(const uint8_t* data) {
|
|
32
33
|
/* A (forgetful) hash table to the data seen by the compressor, to
|
33
34
|
help create backward references to previous data.
|
34
35
|
|
35
|
-
This is a hash map of fixed size (BUCKET_SIZE).
|
36
|
-
given index, BUCKET_SWEEP buckets are used to store values of a key. */
|
36
|
+
This is a hash map of fixed size (BUCKET_SIZE). */
|
37
37
|
typedef struct HashLongestMatchQuickly {
|
38
|
-
|
39
|
-
|
38
|
+
/* Shortcuts. */
|
39
|
+
HasherCommon* common;
|
40
40
|
|
41
|
-
|
42
|
-
|
43
|
-
|
41
|
+
/* --- Dynamic size members --- */
|
42
|
+
|
43
|
+
uint32_t* buckets_; /* uint32_t[BUCKET_SIZE]; */
|
44
|
+
} HashLongestMatchQuickly;
|
44
45
|
|
45
46
|
static void FN(Initialize)(
|
46
|
-
|
47
|
-
|
47
|
+
HasherCommon* common, HashLongestMatchQuickly* BROTLI_RESTRICT self,
|
48
|
+
const BrotliEncoderParams* params) {
|
49
|
+
self->common = common;
|
50
|
+
|
48
51
|
BROTLI_UNUSED(params);
|
52
|
+
self->buckets_ = (uint32_t*)common->extra[0];
|
49
53
|
}
|
50
54
|
|
51
|
-
static void FN(Prepare)(
|
52
|
-
|
53
|
-
|
55
|
+
static void FN(Prepare)(
|
56
|
+
HashLongestMatchQuickly* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
|
57
|
+
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
|
58
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
54
59
|
/* Partial preparation is 100 times slower (per socket). */
|
55
|
-
size_t partial_prepare_threshold =
|
60
|
+
size_t partial_prepare_threshold = BUCKET_SIZE >> 5;
|
56
61
|
if (one_shot && input_size <= partial_prepare_threshold) {
|
57
62
|
size_t i;
|
58
63
|
for (i = 0; i < input_size; ++i) {
|
59
64
|
const uint32_t key = FN(HashBytes)(&data[i]);
|
60
|
-
|
65
|
+
if (BUCKET_SWEEP == 1) {
|
66
|
+
buckets[key] = 0;
|
67
|
+
} else {
|
68
|
+
uint32_t j;
|
69
|
+
for (j = 0; j < BUCKET_SWEEP; ++j) {
|
70
|
+
buckets[(key + (j << 3)) & BUCKET_MASK] = 0;
|
71
|
+
}
|
72
|
+
}
|
61
73
|
}
|
62
74
|
} else {
|
63
75
|
/* It is not strictly necessary to fill this buffer here, but
|
64
76
|
not filling will make the results of the compression stochastic
|
65
77
|
(but correct). This is because random data would cause the
|
66
78
|
system to find accidentally good backward references here and there. */
|
67
|
-
memset(
|
79
|
+
memset(buckets, 0, sizeof(uint32_t) * BUCKET_SIZE);
|
68
80
|
}
|
69
81
|
}
|
70
82
|
|
71
|
-
static BROTLI_INLINE
|
83
|
+
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
|
72
84
|
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
73
|
-
size_t input_size) {
|
85
|
+
size_t input_size, size_t* alloc_size) {
|
74
86
|
BROTLI_UNUSED(params);
|
75
87
|
BROTLI_UNUSED(one_shot);
|
76
88
|
BROTLI_UNUSED(input_size);
|
77
|
-
|
89
|
+
alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE;
|
78
90
|
}
|
79
91
|
|
80
92
|
/* Look at 5 bytes at &data[ix & mask].
|
81
93
|
Compute a hash from these, and store the value somewhere within
|
82
94
|
[ix .. ix+3]. */
|
83
|
-
static BROTLI_INLINE void FN(Store)(
|
84
|
-
|
95
|
+
static BROTLI_INLINE void FN(Store)(
|
96
|
+
HashLongestMatchQuickly* BROTLI_RESTRICT self,
|
97
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
|
85
98
|
const uint32_t key = FN(HashBytes)(&data[ix & mask]);
|
86
|
-
|
87
|
-
|
88
|
-
|
99
|
+
if (BUCKET_SWEEP == 1) {
|
100
|
+
self->buckets_[key] = (uint32_t)ix;
|
101
|
+
} else {
|
102
|
+
/* Wiggle the value with the bucket sweep range. */
|
103
|
+
const uint32_t off = ix & BUCKET_SWEEP_MASK;
|
104
|
+
self->buckets_[(key + off) & BUCKET_MASK] = (uint32_t)ix;
|
105
|
+
}
|
89
106
|
}
|
90
107
|
|
91
|
-
static BROTLI_INLINE void FN(StoreRange)(
|
92
|
-
|
93
|
-
const size_t
|
108
|
+
static BROTLI_INLINE void FN(StoreRange)(
|
109
|
+
HashLongestMatchQuickly* BROTLI_RESTRICT self,
|
110
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
|
111
|
+
const size_t ix_start, const size_t ix_end) {
|
94
112
|
size_t i;
|
95
113
|
for (i = ix_start; i < ix_end; ++i) {
|
96
|
-
FN(Store)(
|
114
|
+
FN(Store)(self, data, mask, i);
|
97
115
|
}
|
98
116
|
}
|
99
117
|
|
100
118
|
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
101
|
-
|
119
|
+
HashLongestMatchQuickly* BROTLI_RESTRICT self,
|
120
|
+
size_t num_bytes, size_t position,
|
102
121
|
const uint8_t* ringbuffer, size_t ringbuffer_mask) {
|
103
122
|
if (num_bytes >= FN(HashTypeLength)() - 1 && position >= 3) {
|
104
123
|
/* Prepare the hashes for three last bytes of the last write.
|
105
124
|
These could not be calculated before, since they require knowledge
|
106
125
|
of both the previous and the current block. */
|
107
|
-
FN(Store)(
|
108
|
-
FN(Store)(
|
109
|
-
FN(Store)(
|
126
|
+
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 3);
|
127
|
+
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 2);
|
128
|
+
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 1);
|
110
129
|
}
|
111
130
|
}
|
112
131
|
|
113
132
|
static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
114
|
-
|
115
|
-
|
133
|
+
HashLongestMatchQuickly* BROTLI_RESTRICT self,
|
134
|
+
int* BROTLI_RESTRICT distance_cache) {
|
135
|
+
BROTLI_UNUSED(self);
|
116
136
|
BROTLI_UNUSED(distance_cache);
|
117
137
|
}
|
118
138
|
|
@@ -125,17 +145,19 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
125
145
|
Writes the best match into |out|.
|
126
146
|
|out|->score is updated only if a better match is found. */
|
127
147
|
static BROTLI_INLINE void FN(FindLongestMatch)(
|
128
|
-
|
148
|
+
HashLongestMatchQuickly* BROTLI_RESTRICT self,
|
149
|
+
const BrotliEncoderDictionary* dictionary,
|
129
150
|
const uint8_t* BROTLI_RESTRICT data,
|
130
151
|
const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
|
131
152
|
const size_t cur_ix, const size_t max_length, const size_t max_backward,
|
132
|
-
const size_t
|
153
|
+
const size_t dictionary_distance, const size_t max_distance,
|
133
154
|
HasherSearchResult* BROTLI_RESTRICT out) {
|
134
|
-
|
155
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
135
156
|
const size_t best_len_in = out->len;
|
136
157
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
137
|
-
const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
|
138
158
|
int compare_char = data[cur_ix_masked + best_len_in];
|
159
|
+
size_t key = FN(HashBytes)(&data[cur_ix_masked]);
|
160
|
+
size_t key_out;
|
139
161
|
score_t min_score = out->score;
|
140
162
|
score_t best_score = out->score;
|
141
163
|
size_t best_len = best_len_in;
|
@@ -145,21 +167,21 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
|
|
145
167
|
if (prev_ix < cur_ix) {
|
146
168
|
prev_ix &= (uint32_t)ring_buffer_mask;
|
147
169
|
if (compare_char == data[prev_ix + best_len]) {
|
148
|
-
size_t len = FindMatchLengthWithLimit(
|
149
|
-
|
150
|
-
max_length);
|
170
|
+
const size_t len = FindMatchLengthWithLimit(
|
171
|
+
&data[prev_ix], &data[cur_ix_masked], max_length);
|
151
172
|
if (len >= 4) {
|
152
173
|
const score_t score = BackwardReferenceScoreUsingLastDistance(len);
|
153
174
|
if (best_score < score) {
|
154
|
-
best_score = score;
|
155
|
-
best_len = len;
|
156
175
|
out->len = len;
|
157
176
|
out->distance = cached_backward;
|
158
|
-
out->score =
|
159
|
-
compare_char = data[cur_ix_masked + best_len];
|
177
|
+
out->score = score;
|
160
178
|
if (BUCKET_SWEEP == 1) {
|
161
|
-
|
179
|
+
buckets[key] = (uint32_t)cur_ix;
|
162
180
|
return;
|
181
|
+
} else {
|
182
|
+
best_len = len;
|
183
|
+
best_score = score;
|
184
|
+
compare_char = data[cur_ix_masked + len];
|
163
185
|
}
|
164
186
|
}
|
165
187
|
}
|
@@ -169,8 +191,8 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
|
|
169
191
|
size_t backward;
|
170
192
|
size_t len;
|
171
193
|
/* Only one to look for, don't bother to prepare for a loop. */
|
172
|
-
prev_ix =
|
173
|
-
|
194
|
+
prev_ix = buckets[key];
|
195
|
+
buckets[key] = (uint32_t)cur_ix;
|
174
196
|
backward = cur_ix - prev_ix;
|
175
197
|
prev_ix &= (uint32_t)ring_buffer_mask;
|
176
198
|
if (compare_char != data[prev_ix + best_len_in]) {
|
@@ -192,12 +214,17 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
|
|
192
214
|
}
|
193
215
|
}
|
194
216
|
} else {
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
217
|
+
size_t keys[BUCKET_SWEEP];
|
218
|
+
size_t i;
|
219
|
+
for (i = 0; i < BUCKET_SWEEP; ++i) {
|
220
|
+
keys[i] = (key + (i << 3)) & BUCKET_MASK;
|
221
|
+
}
|
222
|
+
key_out = keys[(cur_ix & BUCKET_SWEEP_MASK) >> 3];
|
223
|
+
for (i = 0; i < BUCKET_SWEEP; ++i) {
|
200
224
|
size_t len;
|
225
|
+
size_t backward;
|
226
|
+
prev_ix = buckets[keys[i]];
|
227
|
+
backward = cur_ix - prev_ix;
|
201
228
|
prev_ix &= (uint32_t)ring_buffer_mask;
|
202
229
|
if (compare_char != data[prev_ix + best_len]) {
|
203
230
|
continue;
|
@@ -211,25 +238,29 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
|
|
211
238
|
if (len >= 4) {
|
212
239
|
const score_t score = BackwardReferenceScore(len, backward);
|
213
240
|
if (best_score < score) {
|
214
|
-
best_score = score;
|
215
241
|
best_len = len;
|
216
|
-
out->len =
|
217
|
-
|
242
|
+
out->len = len;
|
243
|
+
compare_char = data[cur_ix_masked + len];
|
244
|
+
best_score = score;
|
218
245
|
out->score = score;
|
219
|
-
|
246
|
+
out->distance = backward;
|
220
247
|
}
|
221
248
|
}
|
222
249
|
}
|
223
250
|
}
|
224
251
|
if (USE_DICTIONARY && min_score == out->score) {
|
225
252
|
SearchInStaticDictionary(dictionary,
|
226
|
-
|
253
|
+
self->common, &data[cur_ix_masked], max_length, dictionary_distance,
|
227
254
|
max_distance, out, BROTLI_TRUE);
|
228
255
|
}
|
229
|
-
|
256
|
+
if (BUCKET_SWEEP != 1) {
|
257
|
+
buckets[key_out] = (uint32_t)cur_ix;
|
258
|
+
}
|
230
259
|
}
|
231
260
|
|
232
|
-
#undef
|
261
|
+
#undef BUCKET_SWEEP_MASK
|
262
|
+
#undef BUCKET_SWEEP
|
263
|
+
#undef BUCKET_MASK
|
233
264
|
#undef BUCKET_SIZE
|
234
265
|
|
235
266
|
#undef HashLongestMatchQuickly
|
@@ -51,13 +51,9 @@ typedef struct HashRolling {
|
|
51
51
|
uint32_t factor_remove;
|
52
52
|
} HashRolling;
|
53
53
|
|
54
|
-
static BROTLI_INLINE HashRolling* FN(Self)(HasherHandle handle) {
|
55
|
-
return (HashRolling*)&(GetHasherCommon(handle)[1]);
|
56
|
-
}
|
57
|
-
|
58
54
|
static void FN(Initialize)(
|
59
|
-
|
60
|
-
|
55
|
+
HasherCommon* common, HashRolling* BROTLI_RESTRICT self,
|
56
|
+
const BrotliEncoderParams* params) {
|
61
57
|
size_t i;
|
62
58
|
self->state = 0;
|
63
59
|
self->next_ix = 0;
|
@@ -71,7 +67,7 @@ static void FN(Initialize)(
|
|
71
67
|
self->factor_remove *= self->factor;
|
72
68
|
}
|
73
69
|
|
74
|
-
self->table = (uint32_t*)
|
70
|
+
self->table = (uint32_t*)common->extra[0];
|
75
71
|
for (i = 0; i < NUMBUCKETS; i++) {
|
76
72
|
self->table[i] = FN(kInvalidPos);
|
77
73
|
}
|
@@ -79,9 +75,8 @@ static void FN(Initialize)(
|
|
79
75
|
BROTLI_UNUSED(params);
|
80
76
|
}
|
81
77
|
|
82
|
-
static void FN(Prepare)(
|
83
|
-
size_t input_size, const uint8_t* data) {
|
84
|
-
HashRolling* self = FN(Self)(handle);
|
78
|
+
static void FN(Prepare)(HashRolling* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
|
79
|
+
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
|
85
80
|
size_t i;
|
86
81
|
/* Too small size, cannot use this hasher. */
|
87
82
|
if (input_size < CHUNKLEN) return;
|
@@ -93,39 +88,39 @@ static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
|
|
93
88
|
BROTLI_UNUSED(one_shot);
|
94
89
|
}
|
95
90
|
|
96
|
-
static BROTLI_INLINE
|
91
|
+
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
|
97
92
|
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
98
|
-
size_t input_size) {
|
99
|
-
return sizeof(HashRolling) + NUMBUCKETS * sizeof(uint32_t);
|
93
|
+
size_t input_size, size_t* alloc_size) {
|
100
94
|
BROTLI_UNUSED(params);
|
101
95
|
BROTLI_UNUSED(one_shot);
|
102
96
|
BROTLI_UNUSED(input_size);
|
97
|
+
alloc_size[0] = NUMBUCKETS * sizeof(uint32_t);
|
103
98
|
}
|
104
99
|
|
105
|
-
static BROTLI_INLINE void FN(Store)(
|
100
|
+
static BROTLI_INLINE void FN(Store)(HashRolling* BROTLI_RESTRICT self,
|
106
101
|
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
|
107
|
-
BROTLI_UNUSED(
|
102
|
+
BROTLI_UNUSED(self);
|
108
103
|
BROTLI_UNUSED(data);
|
109
104
|
BROTLI_UNUSED(mask);
|
110
105
|
BROTLI_UNUSED(ix);
|
111
106
|
}
|
112
107
|
|
113
|
-
static BROTLI_INLINE void FN(StoreRange)(
|
114
|
-
const uint8_t* data, const size_t mask,
|
115
|
-
const size_t ix_end) {
|
116
|
-
BROTLI_UNUSED(
|
108
|
+
static BROTLI_INLINE void FN(StoreRange)(HashRolling* BROTLI_RESTRICT self,
|
109
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
|
110
|
+
const size_t ix_start, const size_t ix_end) {
|
111
|
+
BROTLI_UNUSED(self);
|
117
112
|
BROTLI_UNUSED(data);
|
118
113
|
BROTLI_UNUSED(mask);
|
119
114
|
BROTLI_UNUSED(ix_start);
|
120
115
|
BROTLI_UNUSED(ix_end);
|
121
116
|
}
|
122
117
|
|
123
|
-
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
118
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
119
|
+
HashRolling* BROTLI_RESTRICT self,
|
124
120
|
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
125
121
|
size_t ring_buffer_mask) {
|
126
122
|
/* In this case we must re-initialize the hasher from scratch from the
|
127
123
|
current position. */
|
128
|
-
HashRolling* self = FN(Self)(handle);
|
129
124
|
size_t position_masked;
|
130
125
|
size_t available = num_bytes;
|
131
126
|
if ((position & (JUMP - 1)) != 0) {
|
@@ -139,28 +134,29 @@ static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
|
|
139
134
|
available = ring_buffer_mask - position_masked;
|
140
135
|
}
|
141
136
|
|
142
|
-
FN(Prepare)(
|
137
|
+
FN(Prepare)(self, BROTLI_FALSE, available,
|
143
138
|
ringbuffer + (position & ring_buffer_mask));
|
144
139
|
self->next_ix = position;
|
145
140
|
BROTLI_UNUSED(num_bytes);
|
146
141
|
}
|
147
142
|
|
148
143
|
static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
149
|
-
|
150
|
-
|
144
|
+
HashRolling* BROTLI_RESTRICT self,
|
145
|
+
int* BROTLI_RESTRICT distance_cache) {
|
146
|
+
BROTLI_UNUSED(self);
|
151
147
|
BROTLI_UNUSED(distance_cache);
|
152
148
|
}
|
153
149
|
|
154
|
-
static BROTLI_INLINE void FN(FindLongestMatch)(
|
150
|
+
static BROTLI_INLINE void FN(FindLongestMatch)(
|
151
|
+
HashRolling* BROTLI_RESTRICT self,
|
155
152
|
const BrotliEncoderDictionary* dictionary,
|
156
153
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
157
154
|
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
158
155
|
const size_t max_length, const size_t max_backward,
|
159
|
-
const size_t
|
156
|
+
const size_t dictionary_distance, const size_t max_distance,
|
160
157
|
HasherSearchResult* BROTLI_RESTRICT out) {
|
161
|
-
HashRolling* self = FN(Self)(handle);
|
162
158
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
163
|
-
size_t pos
|
159
|
+
size_t pos;
|
164
160
|
|
165
161
|
if ((cur_ix & (JUMP - 1)) != 0) return;
|
166
162
|
|
@@ -209,7 +205,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
209
205
|
backup-hasher, the main hasher already searches in it. */
|
210
206
|
BROTLI_UNUSED(dictionary);
|
211
207
|
BROTLI_UNUSED(distance_cache);
|
212
|
-
BROTLI_UNUSED(
|
208
|
+
BROTLI_UNUSED(dictionary_distance);
|
213
209
|
BROTLI_UNUSED(max_distance);
|
214
210
|
}
|
215
211
|
|
@@ -24,7 +24,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
|
|
24
24
|
return MAX_TREE_COMP_LENGTH;
|
25
25
|
}
|
26
26
|
|
27
|
-
static uint32_t FN(HashBytes)(const uint8_t* data) {
|
27
|
+
static uint32_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data) {
|
28
28
|
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
29
29
|
/* The higher bits contain more mixture from the multiplication,
|
30
30
|
so we take our results from there. */
|
@@ -38,7 +38,7 @@ typedef struct HashToBinaryTree {
|
|
38
38
|
/* Hash table that maps the 4-byte hashes of the sequence to the last
|
39
39
|
position where this hash was found, which is the root of the binary
|
40
40
|
tree of sequences that share this hash bucket. */
|
41
|
-
uint32_t buckets_[BUCKET_SIZE];
|
41
|
+
uint32_t* buckets_; /* uint32_t[BUCKET_SIZE]; */
|
42
42
|
|
43
43
|
/* A position used to mark a non-existent sequence, i.e. a tree is empty if
|
44
44
|
its root is at invalid_pos_ and a node is a leaf if both its children
|
@@ -51,53 +51,52 @@ typedef struct HashToBinaryTree {
|
|
51
51
|
corresponding to a hash is a sequence starting at buckets_[hash] and
|
52
52
|
the left and right children of a sequence starting at pos are
|
53
53
|
forest_[2 * pos] and forest_[2 * pos + 1]. */
|
54
|
-
/* uint32_t
|
54
|
+
uint32_t* forest_; /* uint32_t[2 * num_nodes] */
|
55
55
|
} HashToBinaryTree;
|
56
56
|
|
57
|
-
static BROTLI_INLINE HashToBinaryTree* FN(Self)(HasherHandle handle) {
|
58
|
-
return (HashToBinaryTree*)&(GetHasherCommon(handle)[1]);
|
59
|
-
}
|
60
|
-
|
61
|
-
static BROTLI_INLINE uint32_t* FN(Forest)(HashToBinaryTree* self) {
|
62
|
-
return (uint32_t*)(&self[1]);
|
63
|
-
}
|
64
|
-
|
65
57
|
static void FN(Initialize)(
|
66
|
-
|
67
|
-
|
58
|
+
HasherCommon* common, HashToBinaryTree* BROTLI_RESTRICT self,
|
59
|
+
const BrotliEncoderParams* params) {
|
60
|
+
self->buckets_ = (uint32_t*)common->extra[0];
|
61
|
+
self->forest_ = (uint32_t*)common->extra[1];
|
62
|
+
|
68
63
|
self->window_mask_ = (1u << params->lgwin) - 1u;
|
69
64
|
self->invalid_pos_ = (uint32_t)(0 - self->window_mask_);
|
70
65
|
}
|
71
66
|
|
72
|
-
static void FN(Prepare)
|
73
|
-
|
74
|
-
|
67
|
+
static void FN(Prepare)
|
68
|
+
(HashToBinaryTree* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
|
69
|
+
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
|
75
70
|
uint32_t invalid_pos = self->invalid_pos_;
|
76
71
|
uint32_t i;
|
72
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
77
73
|
BROTLI_UNUSED(data);
|
78
74
|
BROTLI_UNUSED(one_shot);
|
79
75
|
BROTLI_UNUSED(input_size);
|
80
76
|
for (i = 0; i < BUCKET_SIZE; i++) {
|
81
|
-
|
77
|
+
buckets[i] = invalid_pos;
|
82
78
|
}
|
83
79
|
}
|
84
80
|
|
85
|
-
static BROTLI_INLINE
|
81
|
+
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
|
86
82
|
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
87
|
-
size_t input_size) {
|
83
|
+
size_t input_size, size_t* alloc_size) {
|
88
84
|
size_t num_nodes = (size_t)1 << params->lgwin;
|
89
85
|
if (one_shot && input_size < num_nodes) {
|
90
86
|
num_nodes = input_size;
|
91
87
|
}
|
92
|
-
|
88
|
+
alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE;
|
89
|
+
alloc_size[1] = 2 * sizeof(uint32_t) * num_nodes;
|
93
90
|
}
|
94
91
|
|
95
|
-
static BROTLI_INLINE size_t FN(LeftChildIndex)(
|
92
|
+
static BROTLI_INLINE size_t FN(LeftChildIndex)(
|
93
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
96
94
|
const size_t pos) {
|
97
95
|
return 2 * (pos & self->window_mask_);
|
98
96
|
}
|
99
97
|
|
100
|
-
static BROTLI_INLINE size_t FN(RightChildIndex)(
|
98
|
+
static BROTLI_INLINE size_t FN(RightChildIndex)(
|
99
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
101
100
|
const size_t pos) {
|
102
101
|
return 2 * (pos & self->window_mask_) + 1;
|
103
102
|
}
|
@@ -113,7 +112,7 @@ static BROTLI_INLINE size_t FN(RightChildIndex)(HashToBinaryTree* self,
|
|
113
112
|
|
114
113
|
This function must be called with increasing cur_ix positions. */
|
115
114
|
static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
116
|
-
HashToBinaryTree* self, const uint8_t*
|
115
|
+
HashToBinaryTree* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
|
117
116
|
const size_t cur_ix, const size_t ring_buffer_mask, const size_t max_length,
|
118
117
|
const size_t max_backward, size_t* const BROTLI_RESTRICT best_len,
|
119
118
|
BackwardMatch* BROTLI_RESTRICT matches) {
|
@@ -123,8 +122,9 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
123
122
|
const BROTLI_BOOL should_reroot_tree =
|
124
123
|
TO_BROTLI_BOOL(max_length >= MAX_TREE_COMP_LENGTH);
|
125
124
|
const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
|
126
|
-
uint32_t*
|
127
|
-
|
125
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
126
|
+
uint32_t* BROTLI_RESTRICT forest = self->forest_;
|
127
|
+
size_t prev_ix = buckets[key];
|
128
128
|
/* The forest index of the rightmost node of the left subtree of the new
|
129
129
|
root, updated as we traverse and re-root the tree of the hash bucket. */
|
130
130
|
size_t node_left = FN(LeftChildIndex)(self, cur_ix);
|
@@ -139,7 +139,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
139
139
|
size_t best_len_right = 0;
|
140
140
|
size_t depth_remaining;
|
141
141
|
if (should_reroot_tree) {
|
142
|
-
|
142
|
+
buckets[key] = (uint32_t)cur_ix;
|
143
143
|
}
|
144
144
|
for (depth_remaining = MAX_TREE_SEARCH_DEPTH; ; --depth_remaining) {
|
145
145
|
const size_t backward = cur_ix - prev_ix;
|
@@ -199,11 +199,13 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
199
199
|
matches in matches[0] to matches[*num_matches - 1]. The matches will be
|
200
200
|
sorted by strictly increasing length and (non-strictly) increasing
|
201
201
|
distance. */
|
202
|
-
static BROTLI_INLINE size_t FN(FindAllMatches)(
|
203
|
-
|
202
|
+
static BROTLI_INLINE size_t FN(FindAllMatches)(
|
203
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
204
|
+
const BrotliEncoderDictionary* dictionary,
|
205
|
+
const uint8_t* BROTLI_RESTRICT data,
|
204
206
|
const size_t ring_buffer_mask, const size_t cur_ix,
|
205
207
|
const size_t max_length, const size_t max_backward,
|
206
|
-
const size_t
|
208
|
+
const size_t dictionary_distance, const BrotliEncoderParams* params,
|
207
209
|
BackwardMatch* matches) {
|
208
210
|
BackwardMatch* const orig_matches = matches;
|
209
211
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
@@ -236,7 +238,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
236
238
|
}
|
237
239
|
}
|
238
240
|
if (best_len < max_length) {
|
239
|
-
matches = FN(StoreAndFindMatches)(
|
241
|
+
matches = FN(StoreAndFindMatches)(self, data, cur_ix,
|
240
242
|
ring_buffer_mask, max_length, max_backward, &best_len, matches);
|
241
243
|
}
|
242
244
|
for (i = 0; i <= BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN; ++i) {
|
@@ -252,7 +254,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
252
254
|
for (l = minlen; l <= maxlen; ++l) {
|
253
255
|
uint32_t dict_id = dict_matches[l];
|
254
256
|
if (dict_id < kInvalidMatch) {
|
255
|
-
size_t distance =
|
257
|
+
size_t distance = dictionary_distance + (dict_id >> 5) + 1;
|
256
258
|
if (distance <= params->dist.max_distance) {
|
257
259
|
InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
|
258
260
|
}
|
@@ -266,18 +268,18 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
266
268
|
/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
|
267
269
|
current sequence, without returning any matches.
|
268
270
|
REQUIRES: ix + MAX_TREE_COMP_LENGTH <= end-of-current-block */
|
269
|
-
static BROTLI_INLINE void FN(Store)(
|
271
|
+
static BROTLI_INLINE void FN(Store)(HashToBinaryTree* BROTLI_RESTRICT self,
|
272
|
+
const uint8_t* BROTLI_RESTRICT data,
|
270
273
|
const size_t mask, const size_t ix) {
|
271
|
-
HashToBinaryTree* self = FN(Self)(handle);
|
272
274
|
/* Maximum distance is window size - 16, see section 9.1. of the spec. */
|
273
275
|
const size_t max_backward = self->window_mask_ - BROTLI_WINDOW_GAP + 1;
|
274
276
|
FN(StoreAndFindMatches)(self, data, ix, mask, MAX_TREE_COMP_LENGTH,
|
275
277
|
max_backward, NULL, NULL);
|
276
278
|
}
|
277
279
|
|
278
|
-
static BROTLI_INLINE void FN(StoreRange)(
|
279
|
-
const uint8_t* data, const size_t mask,
|
280
|
-
const size_t ix_end) {
|
280
|
+
static BROTLI_INLINE void FN(StoreRange)(HashToBinaryTree* BROTLI_RESTRICT self,
|
281
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
|
282
|
+
const size_t ix_start, const size_t ix_end) {
|
281
283
|
size_t i = ix_start;
|
282
284
|
size_t j = ix_start;
|
283
285
|
if (ix_start + 63 <= ix_end) {
|
@@ -285,18 +287,18 @@ static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
|
285
287
|
}
|
286
288
|
if (ix_start + 512 <= i) {
|
287
289
|
for (; j < i; j += 8) {
|
288
|
-
FN(Store)(
|
290
|
+
FN(Store)(self, data, mask, j);
|
289
291
|
}
|
290
292
|
}
|
291
293
|
for (; i < ix_end; ++i) {
|
292
|
-
FN(Store)(
|
294
|
+
FN(Store)(self, data, mask, i);
|
293
295
|
}
|
294
296
|
}
|
295
297
|
|
296
|
-
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
298
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
299
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
297
300
|
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
298
301
|
size_t ringbuffer_mask) {
|
299
|
-
HashToBinaryTree* self = FN(Self)(handle);
|
300
302
|
if (num_bytes >= FN(HashTypeLength)() - 1 &&
|
301
303
|
position >= MAX_TREE_COMP_LENGTH) {
|
302
304
|
/* Store the last `MAX_TREE_COMP_LENGTH - 1` positions in the hasher.
|
@@ -6,11 +6,11 @@
|
|
6
6
|
|
7
7
|
/* Build per-context histograms of literals, commands and distance codes. */
|
8
8
|
|
9
|
-
#include "
|
9
|
+
#include "histogram.h"
|
10
10
|
|
11
11
|
#include "../common/context.h"
|
12
|
-
#include "
|
13
|
-
#include "
|
12
|
+
#include "block_splitter.h"
|
13
|
+
#include "command.h"
|
14
14
|
|
15
15
|
#if defined(__cplusplus) || defined(c_plusplus)
|
16
16
|
extern "C" {
|
@@ -63,7 +63,7 @@ void BrotliBuildHistogramsWithContext(
|
|
63
63
|
BlockSplitIteratorNext(&insert_and_copy_it);
|
64
64
|
HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_],
|
65
65
|
cmd->cmd_prefix_);
|
66
|
-
/* TODO: unwrap iterator blocks. */
|
66
|
+
/* TODO(eustas): unwrap iterator blocks. */
|
67
67
|
for (j = cmd->insert_len_; j != 0; --j) {
|
68
68
|
size_t context;
|
69
69
|
BlockSplitIteratorNext(&literal_it);
|