brotli 0.2.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +37 -0
- data/.github/workflows/publish.yml +24 -0
- data/.gitmodules +1 -1
- data/Gemfile +6 -3
- data/README.md +2 -2
- data/Rakefile +16 -9
- data/brotli.gemspec +7 -13
- data/ext/brotli/brotli.c +210 -31
- data/ext/brotli/buffer.c +1 -7
- data/ext/brotli/buffer.h +1 -1
- data/ext/brotli/extconf.rb +25 -17
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +107 -0
- data/test/brotli_writer_test.rb +36 -0
- data/test/test_helper.rb +8 -0
- data/vendor/brotli/c/common/constants.c +15 -0
- data/vendor/brotli/c/common/constants.h +137 -0
- data/vendor/brotli/c/common/context.c +156 -0
- data/vendor/brotli/c/common/context.h +4 -152
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +14 -3
- data/vendor/brotli/c/common/platform.c +23 -0
- data/vendor/brotli/c/common/platform.h +95 -122
- data/vendor/brotli/c/common/shared_dictionary.c +521 -0
- data/vendor/brotli/c/common/shared_dictionary_internal.h +75 -0
- data/vendor/brotli/c/common/transform.c +60 -4
- data/vendor/brotli/c/common/transform.h +5 -0
- data/vendor/brotli/c/common/version.h +31 -6
- data/vendor/brotli/c/dec/bit_reader.c +34 -4
- data/vendor/brotli/c/dec/bit_reader.h +221 -107
- data/vendor/brotli/c/dec/decode.c +772 -403
- data/vendor/brotli/c/dec/huffman.c +7 -4
- data/vendor/brotli/c/dec/huffman.h +8 -13
- data/vendor/brotli/c/dec/prefix.h +1 -18
- data/vendor/brotli/c/dec/state.c +40 -21
- data/vendor/brotli/c/dec/state.h +201 -59
- data/vendor/brotli/c/enc/backward_references.c +88 -25
- data/vendor/brotli/c/enc/backward_references.h +10 -8
- data/vendor/brotli/c/enc/backward_references_hq.c +194 -80
- data/vendor/brotli/c/enc/backward_references_hq.h +17 -13
- data/vendor/brotli/c/enc/backward_references_inc.h +52 -16
- data/vendor/brotli/c/enc/bit_cost.c +8 -7
- data/vendor/brotli/c/enc/bit_cost.h +5 -4
- data/vendor/brotli/c/enc/block_splitter.c +40 -17
- data/vendor/brotli/c/enc/block_splitter.h +5 -4
- data/vendor/brotli/c/enc/block_splitter_inc.h +99 -49
- data/vendor/brotli/c/enc/brotli_bit_stream.c +142 -137
- data/vendor/brotli/c/enc/brotli_bit_stream.h +11 -6
- data/vendor/brotli/c/enc/cluster.c +10 -9
- data/vendor/brotli/c/enc/cluster.h +7 -6
- data/vendor/brotli/c/enc/cluster_inc.h +30 -22
- data/vendor/brotli/c/enc/command.c +28 -0
- data/vendor/brotli/c/enc/command.h +17 -16
- data/vendor/brotli/c/enc/compound_dictionary.c +207 -0
- data/vendor/brotli/c/enc/compound_dictionary.h +74 -0
- data/vendor/brotli/c/enc/compress_fragment.c +93 -83
- data/vendor/brotli/c/enc/compress_fragment.h +32 -7
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +100 -88
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +21 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +1829 -1101
- data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
- data/vendor/brotli/c/enc/encode.c +550 -416
- data/vendor/brotli/c/enc/encoder_dict.c +613 -5
- data/vendor/brotli/c/enc/encoder_dict.h +120 -4
- data/vendor/brotli/c/enc/entropy_encode.c +5 -2
- data/vendor/brotli/c/enc/entropy_encode.h +4 -3
- data/vendor/brotli/c/enc/entropy_encode_static.h +5 -2
- data/vendor/brotli/c/enc/fast_log.c +105 -0
- data/vendor/brotli/c/enc/fast_log.h +21 -101
- data/vendor/brotli/c/enc/find_match_length.h +17 -25
- data/vendor/brotli/c/enc/hash.h +350 -120
- data/vendor/brotli/c/enc/hash_composite_inc.h +71 -67
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +92 -51
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +79 -84
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +53 -54
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +93 -62
- data/vendor/brotli/c/enc/hash_rolling_inc.h +25 -29
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +42 -40
- data/vendor/brotli/c/enc/histogram.c +4 -4
- data/vendor/brotli/c/enc/histogram.h +7 -6
- data/vendor/brotli/c/enc/literal_cost.c +20 -15
- data/vendor/brotli/c/enc/literal_cost.h +4 -2
- data/vendor/brotli/c/enc/memory.c +29 -5
- data/vendor/brotli/c/enc/memory.h +43 -14
- data/vendor/brotli/c/enc/metablock.c +95 -85
- data/vendor/brotli/c/enc/metablock.h +9 -8
- data/vendor/brotli/c/enc/metablock_inc.h +9 -7
- data/vendor/brotli/c/enc/params.h +7 -4
- data/vendor/brotli/c/enc/prefix.h +3 -2
- data/vendor/brotli/c/enc/quality.h +40 -3
- data/vendor/brotli/c/enc/ringbuffer.h +8 -4
- data/vendor/brotli/c/enc/state.h +104 -0
- data/vendor/brotli/c/enc/static_dict.c +60 -4
- data/vendor/brotli/c/enc/static_dict.h +3 -2
- data/vendor/brotli/c/enc/static_dict_lut.h +2 -0
- data/vendor/brotli/c/enc/utf8_util.c +2 -2
- data/vendor/brotli/c/enc/utf8_util.h +2 -1
- data/vendor/brotli/c/enc/write_bits.h +29 -26
- data/vendor/brotli/c/include/brotli/decode.h +67 -2
- data/vendor/brotli/c/include/brotli/encode.h +77 -3
- data/vendor/brotli/c/include/brotli/port.h +34 -3
- data/vendor/brotli/c/include/brotli/shared_dictionary.h +100 -0
- metadata +23 -97
- data/.travis.yml +0 -31
- data/docs/Brotli/Error.html +0 -124
- data/docs/Brotli.html +0 -485
- data/docs/_index.html +0 -122
- data/docs/class_list.html +0 -51
- data/docs/css/common.css +0 -1
- data/docs/css/full_list.css +0 -58
- data/docs/css/style.css +0 -496
- data/docs/file.README.html +0 -127
- data/docs/file_list.html +0 -56
- data/docs/frames.html +0 -17
- data/docs/index.html +0 -127
- data/docs/js/app.js +0 -292
- data/docs/js/full_list.js +0 -216
- data/docs/js/jquery.js +0 -4
- data/docs/method_list.html +0 -67
- data/docs/top-level-namespace.html +0 -110
- data/spec/brotli_spec.rb +0 -88
- data/spec/inflate_spec.rb +0 -75
- data/spec/spec_helper.rb +0 -4
@@ -20,13 +20,12 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
|
|
20
20
|
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
|
21
21
|
|
22
22
|
/* HashBytes is the function that chooses the bucket to place the address in. */
|
23
|
-
static BROTLI_INLINE
|
24
|
-
|
25
|
-
|
26
|
-
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(data) & mask) * kHashMul64Long;
|
23
|
+
static BROTLI_INLINE size_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data,
|
24
|
+
uint64_t hash_mul) {
|
25
|
+
const uint64_t h = BROTLI_UNALIGNED_LOAD64LE(data) * hash_mul;
|
27
26
|
/* The higher bits contain more mixture from the multiplication,
|
28
27
|
so we take our results from there. */
|
29
|
-
return (
|
28
|
+
return (size_t)(h >> (64 - 15));
|
30
29
|
}
|
31
30
|
|
32
31
|
typedef struct HashLongestMatch {
|
@@ -35,57 +34,54 @@ typedef struct HashLongestMatch {
|
|
35
34
|
/* Only block_size_ newest backward references are kept,
|
36
35
|
and the older are forgotten. */
|
37
36
|
size_t block_size_;
|
38
|
-
/*
|
39
|
-
|
40
|
-
/* Mask for selecting the next 4-8 bytes of input */
|
41
|
-
uint64_t hash_mask_;
|
37
|
+
/* Hash multiplier tuned to match length. */
|
38
|
+
uint64_t hash_mul_;
|
42
39
|
/* Mask for accessing entries in a block (in a ring-buffer manner). */
|
43
40
|
uint32_t block_mask_;
|
44
41
|
|
42
|
+
int block_bits_;
|
43
|
+
int num_last_distances_to_check_;
|
44
|
+
|
45
|
+
/* Shortcuts. */
|
46
|
+
HasherCommon* common_;
|
47
|
+
|
45
48
|
/* --- Dynamic size members --- */
|
46
49
|
|
47
50
|
/* Number of entries in a particular bucket. */
|
48
|
-
/* uint16_t
|
51
|
+
uint16_t* num_; /* uint16_t[bucket_size]; */
|
49
52
|
|
50
53
|
/* Buckets containing block_size_ of backward references. */
|
51
|
-
/* uint32_t
|
54
|
+
uint32_t* buckets_; /* uint32_t[bucket_size * block_size]; */
|
52
55
|
} HashLongestMatch;
|
53
56
|
|
54
|
-
static BROTLI_INLINE HashLongestMatch* FN(Self)(HasherHandle handle) {
|
55
|
-
return (HashLongestMatch*)&(GetHasherCommon(handle)[1]);
|
56
|
-
}
|
57
|
-
|
58
|
-
static BROTLI_INLINE uint16_t* FN(Num)(HashLongestMatch* self) {
|
59
|
-
return (uint16_t*)(&self[1]);
|
60
|
-
}
|
61
|
-
|
62
|
-
static BROTLI_INLINE uint32_t* FN(Buckets)(HashLongestMatch* self) {
|
63
|
-
return (uint32_t*)(&FN(Num)(self)[self->bucket_size_]);
|
64
|
-
}
|
65
|
-
|
66
57
|
static void FN(Initialize)(
|
67
|
-
|
68
|
-
|
69
|
-
|
58
|
+
HasherCommon* common, HashLongestMatch* BROTLI_RESTRICT self,
|
59
|
+
const BrotliEncoderParams* params) {
|
60
|
+
self->common_ = common;
|
61
|
+
|
70
62
|
BROTLI_UNUSED(params);
|
71
|
-
self->
|
72
|
-
|
63
|
+
self->hash_mul_ = kHashMul64 << (64 - 5 * 8);
|
64
|
+
BROTLI_DCHECK(common->params.bucket_bits == 15);
|
73
65
|
self->bucket_size_ = (size_t)1 << common->params.bucket_bits;
|
66
|
+
self->block_bits_ = common->params.block_bits;
|
74
67
|
self->block_size_ = (size_t)1 << common->params.block_bits;
|
75
68
|
self->block_mask_ = (uint32_t)(self->block_size_ - 1);
|
69
|
+
self->num_last_distances_to_check_ =
|
70
|
+
common->params.num_last_distances_to_check;
|
71
|
+
self->num_ = (uint16_t*)common->extra[0];
|
72
|
+
self->buckets_ = (uint32_t*)common->extra[1];
|
76
73
|
}
|
77
74
|
|
78
|
-
static void FN(Prepare)(
|
79
|
-
|
80
|
-
|
81
|
-
uint16_t* num =
|
75
|
+
static void FN(Prepare)(
|
76
|
+
HashLongestMatch* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
|
77
|
+
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
|
78
|
+
uint16_t* BROTLI_RESTRICT num = self->num_;
|
82
79
|
/* Partial preparation is 100 times slower (per socket). */
|
83
80
|
size_t partial_prepare_threshold = self->bucket_size_ >> 6;
|
84
81
|
if (one_shot && input_size <= partial_prepare_threshold) {
|
85
82
|
size_t i;
|
86
83
|
for (i = 0; i < input_size; ++i) {
|
87
|
-
const
|
88
|
-
self->hash_shift_);
|
84
|
+
const size_t key = FN(HashBytes)(&data[i], self->hash_mul_);
|
89
85
|
num[key] = 0;
|
90
86
|
}
|
91
87
|
} else {
|
@@ -93,57 +89,58 @@ static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
|
|
93
89
|
}
|
94
90
|
}
|
95
91
|
|
96
|
-
static BROTLI_INLINE
|
92
|
+
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
|
97
93
|
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
98
|
-
size_t input_size) {
|
94
|
+
size_t input_size, size_t* alloc_size) {
|
99
95
|
size_t bucket_size = (size_t)1 << params->hasher.bucket_bits;
|
100
96
|
size_t block_size = (size_t)1 << params->hasher.block_bits;
|
101
97
|
BROTLI_UNUSED(one_shot);
|
102
98
|
BROTLI_UNUSED(input_size);
|
103
|
-
|
99
|
+
alloc_size[0] = sizeof(uint16_t) * bucket_size;
|
100
|
+
alloc_size[1] = sizeof(uint32_t) * bucket_size * block_size;
|
104
101
|
}
|
105
102
|
|
106
103
|
/* Look at 4 bytes at &data[ix & mask].
|
107
104
|
Compute a hash from these, and store the value of ix at that position. */
|
108
|
-
static BROTLI_INLINE void FN(Store)(
|
105
|
+
static BROTLI_INLINE void FN(Store)(
|
106
|
+
HashLongestMatch* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
|
109
107
|
const size_t mask, const size_t ix) {
|
110
|
-
|
111
|
-
|
112
|
-
const
|
113
|
-
self->hash_shift_);
|
108
|
+
uint16_t* BROTLI_RESTRICT num = self->num_;
|
109
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
110
|
+
const size_t key = FN(HashBytes)(&data[ix & mask], self->hash_mul_);
|
114
111
|
const size_t minor_ix = num[key] & self->block_mask_;
|
115
|
-
const size_t offset =
|
116
|
-
minor_ix + (key << GetHasherCommon(handle)->params.block_bits);
|
117
|
-
FN(Buckets)(self)[offset] = (uint32_t)ix;
|
112
|
+
const size_t offset = minor_ix + (key << self->block_bits_);
|
118
113
|
++num[key];
|
114
|
+
buckets[offset] = (uint32_t)ix;
|
119
115
|
}
|
120
116
|
|
121
|
-
static BROTLI_INLINE void FN(StoreRange)(
|
122
|
-
const uint8_t* data, const size_t mask,
|
123
|
-
const size_t ix_end) {
|
117
|
+
static BROTLI_INLINE void FN(StoreRange)(HashLongestMatch* BROTLI_RESTRICT self,
|
118
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
|
119
|
+
const size_t ix_start, const size_t ix_end) {
|
124
120
|
size_t i;
|
125
121
|
for (i = ix_start; i < ix_end; ++i) {
|
126
|
-
FN(Store)(
|
122
|
+
FN(Store)(self, data, mask, i);
|
127
123
|
}
|
128
124
|
}
|
129
125
|
|
130
|
-
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
126
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
127
|
+
HashLongestMatch* BROTLI_RESTRICT self,
|
131
128
|
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
132
129
|
size_t ringbuffer_mask) {
|
133
130
|
if (num_bytes >= FN(HashTypeLength)() - 1 && position >= 3) {
|
134
131
|
/* Prepare the hashes for three last bytes of the last write.
|
135
132
|
These could not be calculated before, since they require knowledge
|
136
133
|
of both the previous and the current block. */
|
137
|
-
FN(Store)(
|
138
|
-
FN(Store)(
|
139
|
-
FN(Store)(
|
134
|
+
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 3);
|
135
|
+
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 2);
|
136
|
+
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 1);
|
140
137
|
}
|
141
138
|
}
|
142
139
|
|
143
140
|
static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
144
|
-
|
145
|
-
|
146
|
-
|
141
|
+
HashLongestMatch* BROTLI_RESTRICT self,
|
142
|
+
int* BROTLI_RESTRICT distance_cache) {
|
143
|
+
PrepareDistanceCache(distance_cache, self->num_last_distances_to_check_);
|
147
144
|
}
|
148
145
|
|
149
146
|
/* Find a longest backward match of &data[cur_ix] up to the length of
|
@@ -157,17 +154,16 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
157
154
|
Does not look for matches further away than max_backward.
|
158
155
|
Writes the best match into |out|.
|
159
156
|
|out|->score is updated only if a better match is found. */
|
160
|
-
static BROTLI_INLINE void FN(FindLongestMatch)(
|
157
|
+
static BROTLI_INLINE void FN(FindLongestMatch)(
|
158
|
+
HashLongestMatch* BROTLI_RESTRICT self,
|
161
159
|
const BrotliEncoderDictionary* dictionary,
|
162
160
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
163
161
|
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
164
162
|
const size_t max_length, const size_t max_backward,
|
165
|
-
const size_t
|
163
|
+
const size_t dictionary_distance, const size_t max_distance,
|
166
164
|
HasherSearchResult* BROTLI_RESTRICT out) {
|
167
|
-
|
168
|
-
|
169
|
-
uint16_t* num = FN(Num)(self);
|
170
|
-
uint32_t* buckets = FN(Buckets)(self);
|
165
|
+
uint16_t* BROTLI_RESTRICT num = self->num_;
|
166
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
171
167
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
172
168
|
/* Don't accept a short copy from far away. */
|
173
169
|
score_t min_score = out->score;
|
@@ -177,7 +173,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
177
173
|
out->len = 0;
|
178
174
|
out->len_code_delta = 0;
|
179
175
|
/* Try last distance first. */
|
180
|
-
for (i = 0; i < (size_t)
|
176
|
+
for (i = 0; i < (size_t)self->num_last_distances_to_check_; ++i) {
|
181
177
|
const size_t backward = (size_t)distance_cache[i];
|
182
178
|
size_t prev_ix = (size_t)(cur_ix - backward);
|
183
179
|
if (prev_ix >= cur_ix) {
|
@@ -216,15 +212,17 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
216
212
|
}
|
217
213
|
}
|
218
214
|
{
|
219
|
-
const
|
220
|
-
|
221
|
-
uint32_t* BROTLI_RESTRICT bucket =
|
222
|
-
&buckets[key << common->params.block_bits];
|
215
|
+
const size_t key = FN(HashBytes)(&data[cur_ix_masked], self->hash_mul_);
|
216
|
+
uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
|
223
217
|
const size_t down =
|
224
218
|
(num[key] > self->block_size_) ?
|
225
219
|
(num[key] - self->block_size_) : 0u;
|
226
|
-
|
220
|
+
const uint32_t first4 = BrotliUnalignedRead32(data + cur_ix_masked);
|
221
|
+
const size_t max_length_m4 = max_length - 4;
|
222
|
+
i = num[key];
|
223
|
+
for (; i > down;) {
|
227
224
|
size_t prev_ix = bucket[--i & self->block_mask_];
|
225
|
+
uint32_t current4;
|
228
226
|
const size_t backward = cur_ix - prev_ix;
|
229
227
|
if (BROTLI_PREDICT_FALSE(backward > max_backward)) {
|
230
228
|
break;
|
@@ -235,22 +233,19 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
235
233
|
data[cur_ix_masked + best_len] != data[prev_ix + best_len]) {
|
236
234
|
continue;
|
237
235
|
}
|
236
|
+
current4 = BrotliUnalignedRead32(data + prev_ix);
|
237
|
+
if (first4 != current4) continue;
|
238
238
|
{
|
239
|
-
const size_t len = FindMatchLengthWithLimit(&data[prev_ix],
|
240
|
-
&data[cur_ix_masked],
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
best_len = len;
|
250
|
-
out->len = best_len;
|
251
|
-
out->distance = backward;
|
252
|
-
out->score = best_score;
|
253
|
-
}
|
239
|
+
const size_t len = FindMatchLengthWithLimit(&data[prev_ix + 4],
|
240
|
+
&data[cur_ix_masked + 4],
|
241
|
+
max_length_m4) + 4;
|
242
|
+
const score_t score = BackwardReferenceScore(len, backward);
|
243
|
+
if (best_score < score) {
|
244
|
+
best_score = score;
|
245
|
+
best_len = len;
|
246
|
+
out->len = best_len;
|
247
|
+
out->distance = backward;
|
248
|
+
out->score = best_score;
|
254
249
|
}
|
255
250
|
}
|
256
251
|
}
|
@@ -259,7 +254,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
259
254
|
}
|
260
255
|
if (min_score == out->score) {
|
261
256
|
SearchInStaticDictionary(dictionary,
|
262
|
-
|
257
|
+
self->common_, &data[cur_ix_masked], max_length, dictionary_distance,
|
263
258
|
max_distance, out, BROTLI_FALSE);
|
264
259
|
}
|
265
260
|
}
|
@@ -20,7 +20,8 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
|
|
20
20
|
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
|
21
21
|
|
22
22
|
/* HashBytes is the function that chooses the bucket to place the address in. */
|
23
|
-
static uint32_t FN(HashBytes)(
|
23
|
+
static uint32_t FN(HashBytes)(
|
24
|
+
const uint8_t* BROTLI_RESTRICT data, const int shift) {
|
24
25
|
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
25
26
|
/* The higher bits contain more mixture from the multiplication,
|
26
27
|
so we take our results from there. */
|
@@ -38,42 +39,42 @@ typedef struct HashLongestMatch {
|
|
38
39
|
/* Mask for accessing entries in a block (in a ring-buffer manner). */
|
39
40
|
uint32_t block_mask_;
|
40
41
|
|
42
|
+
int block_bits_;
|
43
|
+
int num_last_distances_to_check_;
|
44
|
+
|
45
|
+
/* Shortcuts. */
|
46
|
+
HasherCommon* common_;
|
47
|
+
|
41
48
|
/* --- Dynamic size members --- */
|
42
49
|
|
43
50
|
/* Number of entries in a particular bucket. */
|
44
|
-
/* uint16_t
|
51
|
+
uint16_t* num_; /* uint16_t[bucket_size]; */
|
45
52
|
|
46
53
|
/* Buckets containing block_size_ of backward references. */
|
47
|
-
/* uint32_t
|
54
|
+
uint32_t* buckets_; /* uint32_t[bucket_size * block_size]; */
|
48
55
|
} HashLongestMatch;
|
49
56
|
|
50
|
-
static BROTLI_INLINE HashLongestMatch* FN(Self)(HasherHandle handle) {
|
51
|
-
return (HashLongestMatch*)&(GetHasherCommon(handle)[1]);
|
52
|
-
}
|
53
|
-
|
54
|
-
static BROTLI_INLINE uint16_t* FN(Num)(HashLongestMatch* self) {
|
55
|
-
return (uint16_t*)(&self[1]);
|
56
|
-
}
|
57
|
-
|
58
|
-
static BROTLI_INLINE uint32_t* FN(Buckets)(HashLongestMatch* self) {
|
59
|
-
return (uint32_t*)(&FN(Num)(self)[self->bucket_size_]);
|
60
|
-
}
|
61
|
-
|
62
57
|
static void FN(Initialize)(
|
63
|
-
|
64
|
-
|
65
|
-
|
58
|
+
HasherCommon* common, HashLongestMatch* BROTLI_RESTRICT self,
|
59
|
+
const BrotliEncoderParams* params) {
|
60
|
+
self->common_ = common;
|
61
|
+
|
66
62
|
BROTLI_UNUSED(params);
|
67
63
|
self->hash_shift_ = 32 - common->params.bucket_bits;
|
68
64
|
self->bucket_size_ = (size_t)1 << common->params.bucket_bits;
|
69
65
|
self->block_size_ = (size_t)1 << common->params.block_bits;
|
70
66
|
self->block_mask_ = (uint32_t)(self->block_size_ - 1);
|
67
|
+
self->num_ = (uint16_t*)common->extra[0];
|
68
|
+
self->buckets_ = (uint32_t*)common->extra[1];
|
69
|
+
self->block_bits_ = common->params.block_bits;
|
70
|
+
self->num_last_distances_to_check_ =
|
71
|
+
common->params.num_last_distances_to_check;
|
71
72
|
}
|
72
73
|
|
73
|
-
static void FN(Prepare)(
|
74
|
-
|
75
|
-
|
76
|
-
uint16_t* num =
|
74
|
+
static void FN(Prepare)(
|
75
|
+
HashLongestMatch* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
|
76
|
+
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
|
77
|
+
uint16_t* BROTLI_RESTRICT num = self->num_;
|
77
78
|
/* Partial preparation is 100 times slower (per socket). */
|
78
79
|
size_t partial_prepare_threshold = self->bucket_size_ >> 6;
|
79
80
|
if (one_shot && input_size <= partial_prepare_threshold) {
|
@@ -87,56 +88,56 @@ static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
|
|
87
88
|
}
|
88
89
|
}
|
89
90
|
|
90
|
-
static BROTLI_INLINE
|
91
|
+
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
|
91
92
|
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
92
|
-
size_t input_size) {
|
93
|
+
size_t input_size, size_t* alloc_size) {
|
93
94
|
size_t bucket_size = (size_t)1 << params->hasher.bucket_bits;
|
94
95
|
size_t block_size = (size_t)1 << params->hasher.block_bits;
|
95
96
|
BROTLI_UNUSED(one_shot);
|
96
97
|
BROTLI_UNUSED(input_size);
|
97
|
-
|
98
|
+
alloc_size[0] = sizeof(uint16_t) * bucket_size;
|
99
|
+
alloc_size[1] = sizeof(uint32_t) * bucket_size * block_size;
|
98
100
|
}
|
99
101
|
|
100
102
|
/* Look at 4 bytes at &data[ix & mask].
|
101
103
|
Compute a hash from these, and store the value of ix at that position. */
|
102
|
-
static BROTLI_INLINE void FN(Store)(
|
104
|
+
static BROTLI_INLINE void FN(Store)(
|
105
|
+
HashLongestMatch* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
|
103
106
|
const size_t mask, const size_t ix) {
|
104
|
-
HashLongestMatch* self = FN(Self)(handle);
|
105
|
-
uint16_t* num = FN(Num)(self);
|
106
107
|
const uint32_t key = FN(HashBytes)(&data[ix & mask], self->hash_shift_);
|
107
|
-
const size_t minor_ix =
|
108
|
-
const size_t offset =
|
109
|
-
|
110
|
-
|
111
|
-
++num[key];
|
108
|
+
const size_t minor_ix = self->num_[key] & self->block_mask_;
|
109
|
+
const size_t offset = minor_ix + (key << self->block_bits_);
|
110
|
+
self->buckets_[offset] = (uint32_t)ix;
|
111
|
+
++self->num_[key];
|
112
112
|
}
|
113
113
|
|
114
|
-
static BROTLI_INLINE void FN(StoreRange)(
|
115
|
-
const uint8_t* data, const size_t mask,
|
116
|
-
const size_t ix_end) {
|
114
|
+
static BROTLI_INLINE void FN(StoreRange)(HashLongestMatch* BROTLI_RESTRICT self,
|
115
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
|
116
|
+
const size_t ix_start, const size_t ix_end) {
|
117
117
|
size_t i;
|
118
118
|
for (i = ix_start; i < ix_end; ++i) {
|
119
|
-
FN(Store)(
|
119
|
+
FN(Store)(self, data, mask, i);
|
120
120
|
}
|
121
121
|
}
|
122
122
|
|
123
|
-
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
123
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
124
|
+
HashLongestMatch* BROTLI_RESTRICT self,
|
124
125
|
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
125
126
|
size_t ringbuffer_mask) {
|
126
127
|
if (num_bytes >= FN(HashTypeLength)() - 1 && position >= 3) {
|
127
128
|
/* Prepare the hashes for three last bytes of the last write.
|
128
129
|
These could not be calculated before, since they require knowledge
|
129
130
|
of both the previous and the current block. */
|
130
|
-
FN(Store)(
|
131
|
-
FN(Store)(
|
132
|
-
FN(Store)(
|
131
|
+
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 3);
|
132
|
+
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 2);
|
133
|
+
FN(Store)(self, ringbuffer, ringbuffer_mask, position - 1);
|
133
134
|
}
|
134
135
|
}
|
135
136
|
|
136
137
|
static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
137
|
-
|
138
|
-
|
139
|
-
|
138
|
+
HashLongestMatch* BROTLI_RESTRICT self,
|
139
|
+
int* BROTLI_RESTRICT distance_cache) {
|
140
|
+
PrepareDistanceCache(distance_cache, self->num_last_distances_to_check_);
|
140
141
|
}
|
141
142
|
|
142
143
|
/* Find a longest backward match of &data[cur_ix] up to the length of
|
@@ -150,17 +151,16 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
150
151
|
Does not look for matches further away than max_backward.
|
151
152
|
Writes the best match into |out|.
|
152
153
|
|out|->score is updated only if a better match is found. */
|
153
|
-
static BROTLI_INLINE void FN(FindLongestMatch)(
|
154
|
+
static BROTLI_INLINE void FN(FindLongestMatch)(
|
155
|
+
HashLongestMatch* BROTLI_RESTRICT self,
|
154
156
|
const BrotliEncoderDictionary* dictionary,
|
155
157
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
156
158
|
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
157
159
|
const size_t max_length, const size_t max_backward,
|
158
|
-
const size_t
|
160
|
+
const size_t dictionary_distance, const size_t max_distance,
|
159
161
|
HasherSearchResult* BROTLI_RESTRICT out) {
|
160
|
-
|
161
|
-
|
162
|
-
uint16_t* num = FN(Num)(self);
|
163
|
-
uint32_t* buckets = FN(Buckets)(self);
|
162
|
+
uint16_t* BROTLI_RESTRICT num = self->num_;
|
163
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
164
164
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
165
165
|
/* Don't accept a short copy from far away. */
|
166
166
|
score_t min_score = out->score;
|
@@ -170,7 +170,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
170
170
|
out->len = 0;
|
171
171
|
out->len_code_delta = 0;
|
172
172
|
/* Try last distance first. */
|
173
|
-
for (i = 0; i < (size_t)
|
173
|
+
for (i = 0; i < (size_t)self->num_last_distances_to_check_; ++i) {
|
174
174
|
const size_t backward = (size_t)distance_cache[i];
|
175
175
|
size_t prev_ix = (size_t)(cur_ix - backward);
|
176
176
|
if (prev_ix >= cur_ix) {
|
@@ -211,8 +211,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
211
211
|
{
|
212
212
|
const uint32_t key =
|
213
213
|
FN(HashBytes)(&data[cur_ix_masked], self->hash_shift_);
|
214
|
-
uint32_t* BROTLI_RESTRICT bucket =
|
215
|
-
&buckets[key << common->params.block_bits];
|
214
|
+
uint32_t* BROTLI_RESTRICT bucket = &buckets[key << self->block_bits_];
|
216
215
|
const size_t down =
|
217
216
|
(num[key] > self->block_size_) ? (num[key] - self->block_size_) : 0u;
|
218
217
|
for (i = num[key]; i > down;) {
|
@@ -251,7 +250,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
251
250
|
}
|
252
251
|
if (min_score == out->score) {
|
253
252
|
SearchInStaticDictionary(dictionary,
|
254
|
-
|
253
|
+
self->common_, &data[cur_ix_masked], max_length, dictionary_distance,
|
255
254
|
max_distance, out, BROTLI_FALSE);
|
256
255
|
}
|
257
256
|
}
|