brotli 0.2.3 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +34 -0
- data/.github/workflows/publish.yml +34 -0
- data/Gemfile +6 -3
- data/Rakefile +16 -9
- data/brotli.gemspec +7 -13
- data/ext/brotli/brotli.c +209 -31
- data/ext/brotli/buffer.c +1 -7
- data/ext/brotli/buffer.h +1 -1
- data/ext/brotli/extconf.rb +20 -18
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +104 -0
- data/test/brotli_writer_test.rb +36 -0
- data/test/test_helper.rb +8 -0
- data/vendor/brotli/c/common/constants.c +15 -0
- data/vendor/brotli/c/common/constants.h +136 -0
- data/vendor/brotli/c/common/context.c +156 -0
- data/vendor/brotli/c/common/context.h +4 -152
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +10 -1
- data/vendor/brotli/c/common/platform.c +22 -0
- data/vendor/brotli/c/common/platform.h +43 -17
- data/vendor/brotli/c/common/transform.c +59 -3
- data/vendor/brotli/c/common/transform.h +5 -0
- data/vendor/brotli/c/common/version.h +2 -2
- data/vendor/brotli/c/dec/bit_reader.c +28 -0
- data/vendor/brotli/c/dec/bit_reader.h +58 -16
- data/vendor/brotli/c/dec/decode.c +353 -251
- data/vendor/brotli/c/dec/huffman.h +6 -12
- data/vendor/brotli/c/dec/prefix.h +0 -18
- data/vendor/brotli/c/dec/state.c +9 -14
- data/vendor/brotli/c/dec/state.h +144 -37
- data/vendor/brotli/c/enc/backward_references.c +8 -7
- data/vendor/brotli/c/enc/backward_references.h +5 -4
- data/vendor/brotli/c/enc/backward_references_hq.c +51 -33
- data/vendor/brotli/c/enc/backward_references_hq.h +11 -8
- data/vendor/brotli/c/enc/backward_references_inc.h +24 -14
- data/vendor/brotli/c/enc/block_splitter.c +3 -3
- data/vendor/brotli/c/enc/block_splitter_inc.h +15 -6
- data/vendor/brotli/c/enc/brotli_bit_stream.c +13 -30
- data/vendor/brotli/c/enc/cluster_inc.h +6 -3
- data/vendor/brotli/c/enc/command.c +28 -0
- data/vendor/brotli/c/enc/command.h +12 -12
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1826 -1100
- data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
- data/vendor/brotli/c/enc/encode.c +104 -39
- data/vendor/brotli/c/enc/encoder_dict.c +3 -2
- data/vendor/brotli/c/enc/encoder_dict.h +3 -1
- data/vendor/brotli/c/enc/entropy_encode.c +2 -0
- data/vendor/brotli/c/enc/entropy_encode.h +2 -2
- data/vendor/brotli/c/enc/fast_log.c +105 -0
- data/vendor/brotli/c/enc/fast_log.h +19 -100
- data/vendor/brotli/c/enc/find_match_length.h +2 -3
- data/vendor/brotli/c/enc/hash.h +80 -90
- data/vendor/brotli/c/enc/hash_composite_inc.h +52 -63
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +88 -49
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +50 -50
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +53 -50
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +91 -60
- data/vendor/brotli/c/enc/hash_rolling_inc.h +23 -27
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +39 -38
- data/vendor/brotli/c/enc/memory.h +24 -12
- data/vendor/brotli/c/enc/metablock.c +23 -27
- data/vendor/brotli/c/enc/metablock_inc.h +1 -1
- data/vendor/brotli/c/enc/params.h +3 -1
- data/vendor/brotli/c/enc/ringbuffer.h +4 -1
- data/vendor/brotli/c/enc/utf8_util.c +1 -1
- data/vendor/brotli/c/enc/write_bits.h +27 -25
- data/vendor/brotli/c/include/brotli/encode.h +22 -1
- data/vendor/brotli/c/include/brotli/port.h +14 -0
- metadata +17 -97
- data/.travis.yml +0 -31
- data/docs/Brotli.html +0 -485
- data/docs/Brotli/Error.html +0 -124
- data/docs/_index.html +0 -122
- data/docs/class_list.html +0 -51
- data/docs/css/common.css +0 -1
- data/docs/css/full_list.css +0 -58
- data/docs/css/style.css +0 -496
- data/docs/file.README.html +0 -127
- data/docs/file_list.html +0 -56
- data/docs/frames.html +0 -17
- data/docs/index.html +0 -127
- data/docs/js/app.js +0 -292
- data/docs/js/full_list.js +0 -216
- data/docs/js/jquery.js +0 -4
- data/docs/method_list.html +0 -67
- data/docs/top-level-namespace.html +0 -110
- data/spec/brotli_spec.rb +0 -88
- data/spec/inflate_spec.rb +0 -75
- data/spec/spec_helper.rb +0 -4
@@ -51,13 +51,9 @@ typedef struct HashRolling {
|
|
51
51
|
uint32_t factor_remove;
|
52
52
|
} HashRolling;
|
53
53
|
|
54
|
-
static BROTLI_INLINE HashRolling* FN(Self)(HasherHandle handle) {
|
55
|
-
return (HashRolling*)&(GetHasherCommon(handle)[1]);
|
56
|
-
}
|
57
|
-
|
58
54
|
static void FN(Initialize)(
|
59
|
-
|
60
|
-
|
55
|
+
HasherCommon* common, HashRolling* BROTLI_RESTRICT self,
|
56
|
+
const BrotliEncoderParams* params) {
|
61
57
|
size_t i;
|
62
58
|
self->state = 0;
|
63
59
|
self->next_ix = 0;
|
@@ -71,7 +67,7 @@ static void FN(Initialize)(
|
|
71
67
|
self->factor_remove *= self->factor;
|
72
68
|
}
|
73
69
|
|
74
|
-
self->table = (uint32_t*)
|
70
|
+
self->table = (uint32_t*)common->extra;
|
75
71
|
for (i = 0; i < NUMBUCKETS; i++) {
|
76
72
|
self->table[i] = FN(kInvalidPos);
|
77
73
|
}
|
@@ -79,9 +75,8 @@ static void FN(Initialize)(
|
|
79
75
|
BROTLI_UNUSED(params);
|
80
76
|
}
|
81
77
|
|
82
|
-
static void FN(Prepare)(
|
83
|
-
size_t input_size, const uint8_t* data) {
|
84
|
-
HashRolling* self = FN(Self)(handle);
|
78
|
+
static void FN(Prepare)(HashRolling* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
|
79
|
+
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
|
85
80
|
size_t i;
|
86
81
|
/* Too small size, cannot use this hasher. */
|
87
82
|
if (input_size < CHUNKLEN) return;
|
@@ -96,36 +91,36 @@ static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
|
|
96
91
|
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
97
92
|
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
98
93
|
size_t input_size) {
|
99
|
-
return
|
94
|
+
return NUMBUCKETS * sizeof(uint32_t);
|
100
95
|
BROTLI_UNUSED(params);
|
101
96
|
BROTLI_UNUSED(one_shot);
|
102
97
|
BROTLI_UNUSED(input_size);
|
103
98
|
}
|
104
99
|
|
105
|
-
static BROTLI_INLINE void FN(Store)(
|
100
|
+
static BROTLI_INLINE void FN(Store)(HashRolling* BROTLI_RESTRICT self,
|
106
101
|
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
|
107
|
-
BROTLI_UNUSED(
|
102
|
+
BROTLI_UNUSED(self);
|
108
103
|
BROTLI_UNUSED(data);
|
109
104
|
BROTLI_UNUSED(mask);
|
110
105
|
BROTLI_UNUSED(ix);
|
111
106
|
}
|
112
107
|
|
113
|
-
static BROTLI_INLINE void FN(StoreRange)(
|
114
|
-
const uint8_t* data, const size_t mask,
|
115
|
-
const size_t ix_end) {
|
116
|
-
BROTLI_UNUSED(
|
108
|
+
static BROTLI_INLINE void FN(StoreRange)(HashRolling* BROTLI_RESTRICT self,
|
109
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
|
110
|
+
const size_t ix_start, const size_t ix_end) {
|
111
|
+
BROTLI_UNUSED(self);
|
117
112
|
BROTLI_UNUSED(data);
|
118
113
|
BROTLI_UNUSED(mask);
|
119
114
|
BROTLI_UNUSED(ix_start);
|
120
115
|
BROTLI_UNUSED(ix_end);
|
121
116
|
}
|
122
117
|
|
123
|
-
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
118
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
119
|
+
HashRolling* BROTLI_RESTRICT self,
|
124
120
|
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
125
121
|
size_t ring_buffer_mask) {
|
126
122
|
/* In this case we must re-initialize the hasher from scratch from the
|
127
123
|
current position. */
|
128
|
-
HashRolling* self = FN(Self)(handle);
|
129
124
|
size_t position_masked;
|
130
125
|
size_t available = num_bytes;
|
131
126
|
if ((position & (JUMP - 1)) != 0) {
|
@@ -139,28 +134,29 @@ static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
|
|
139
134
|
available = ring_buffer_mask - position_masked;
|
140
135
|
}
|
141
136
|
|
142
|
-
FN(Prepare)(
|
137
|
+
FN(Prepare)(self, BROTLI_FALSE, available,
|
143
138
|
ringbuffer + (position & ring_buffer_mask));
|
144
139
|
self->next_ix = position;
|
145
140
|
BROTLI_UNUSED(num_bytes);
|
146
141
|
}
|
147
142
|
|
148
143
|
static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
149
|
-
|
150
|
-
|
144
|
+
HashRolling* BROTLI_RESTRICT self,
|
145
|
+
int* BROTLI_RESTRICT distance_cache) {
|
146
|
+
BROTLI_UNUSED(self);
|
151
147
|
BROTLI_UNUSED(distance_cache);
|
152
148
|
}
|
153
149
|
|
154
|
-
static BROTLI_INLINE void FN(FindLongestMatch)(
|
150
|
+
static BROTLI_INLINE void FN(FindLongestMatch)(
|
151
|
+
HashRolling* BROTLI_RESTRICT self,
|
155
152
|
const BrotliEncoderDictionary* dictionary,
|
156
153
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
157
154
|
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
158
155
|
const size_t max_length, const size_t max_backward,
|
159
|
-
const size_t
|
156
|
+
const size_t dictionary_distance, const size_t max_distance,
|
160
157
|
HasherSearchResult* BROTLI_RESTRICT out) {
|
161
|
-
HashRolling* self = FN(Self)(handle);
|
162
158
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
163
|
-
size_t pos
|
159
|
+
size_t pos;
|
164
160
|
|
165
161
|
if ((cur_ix & (JUMP - 1)) != 0) return;
|
166
162
|
|
@@ -209,7 +205,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
209
205
|
backup-hasher, the main hasher already searches in it. */
|
210
206
|
BROTLI_UNUSED(dictionary);
|
211
207
|
BROTLI_UNUSED(distance_cache);
|
212
|
-
BROTLI_UNUSED(
|
208
|
+
BROTLI_UNUSED(dictionary_distance);
|
213
209
|
BROTLI_UNUSED(max_distance);
|
214
210
|
}
|
215
211
|
|
@@ -24,7 +24,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
|
|
24
24
|
return MAX_TREE_COMP_LENGTH;
|
25
25
|
}
|
26
26
|
|
27
|
-
static uint32_t FN(HashBytes)(const uint8_t* data) {
|
27
|
+
static uint32_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data) {
|
28
28
|
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
29
29
|
/* The higher bits contain more mixture from the multiplication,
|
30
30
|
so we take our results from there. */
|
@@ -38,7 +38,7 @@ typedef struct HashToBinaryTree {
|
|
38
38
|
/* Hash table that maps the 4-byte hashes of the sequence to the last
|
39
39
|
position where this hash was found, which is the root of the binary
|
40
40
|
tree of sequences that share this hash bucket. */
|
41
|
-
uint32_t buckets_[BUCKET_SIZE];
|
41
|
+
uint32_t* buckets_; /* uint32_t[BUCKET_SIZE]; */
|
42
42
|
|
43
43
|
/* A position used to mark a non-existent sequence, i.e. a tree is empty if
|
44
44
|
its root is at invalid_pos_ and a node is a leaf if both its children
|
@@ -51,34 +51,30 @@ typedef struct HashToBinaryTree {
|
|
51
51
|
corresponding to a hash is a sequence starting at buckets_[hash] and
|
52
52
|
the left and right children of a sequence starting at pos are
|
53
53
|
forest_[2 * pos] and forest_[2 * pos + 1]. */
|
54
|
-
/* uint32_t
|
54
|
+
uint32_t* forest_; /* uint32_t[2 * num_nodes] */
|
55
55
|
} HashToBinaryTree;
|
56
56
|
|
57
|
-
static BROTLI_INLINE HashToBinaryTree* FN(Self)(HasherHandle handle) {
|
58
|
-
return (HashToBinaryTree*)&(GetHasherCommon(handle)[1]);
|
59
|
-
}
|
60
|
-
|
61
|
-
static BROTLI_INLINE uint32_t* FN(Forest)(HashToBinaryTree* self) {
|
62
|
-
return (uint32_t*)(&self[1]);
|
63
|
-
}
|
64
|
-
|
65
57
|
static void FN(Initialize)(
|
66
|
-
|
67
|
-
|
58
|
+
HasherCommon* common, HashToBinaryTree* BROTLI_RESTRICT self,
|
59
|
+
const BrotliEncoderParams* params) {
|
60
|
+
self->buckets_ = (uint32_t*)common->extra;
|
61
|
+
self->forest_ = &self->buckets_[BUCKET_SIZE];
|
62
|
+
|
68
63
|
self->window_mask_ = (1u << params->lgwin) - 1u;
|
69
64
|
self->invalid_pos_ = (uint32_t)(0 - self->window_mask_);
|
70
65
|
}
|
71
66
|
|
72
|
-
static void FN(Prepare)
|
73
|
-
|
74
|
-
|
67
|
+
static void FN(Prepare)
|
68
|
+
(HashToBinaryTree* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
|
69
|
+
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
|
75
70
|
uint32_t invalid_pos = self->invalid_pos_;
|
76
71
|
uint32_t i;
|
72
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
77
73
|
BROTLI_UNUSED(data);
|
78
74
|
BROTLI_UNUSED(one_shot);
|
79
75
|
BROTLI_UNUSED(input_size);
|
80
76
|
for (i = 0; i < BUCKET_SIZE; i++) {
|
81
|
-
|
77
|
+
buckets[i] = invalid_pos;
|
82
78
|
}
|
83
79
|
}
|
84
80
|
|
@@ -89,15 +85,17 @@ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
|
89
85
|
if (one_shot && input_size < num_nodes) {
|
90
86
|
num_nodes = input_size;
|
91
87
|
}
|
92
|
-
return sizeof(
|
88
|
+
return sizeof(uint32_t) * BUCKET_SIZE + 2 * sizeof(uint32_t) * num_nodes;
|
93
89
|
}
|
94
90
|
|
95
|
-
static BROTLI_INLINE size_t FN(LeftChildIndex)(
|
91
|
+
static BROTLI_INLINE size_t FN(LeftChildIndex)(
|
92
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
96
93
|
const size_t pos) {
|
97
94
|
return 2 * (pos & self->window_mask_);
|
98
95
|
}
|
99
96
|
|
100
|
-
static BROTLI_INLINE size_t FN(RightChildIndex)(
|
97
|
+
static BROTLI_INLINE size_t FN(RightChildIndex)(
|
98
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
101
99
|
const size_t pos) {
|
102
100
|
return 2 * (pos & self->window_mask_) + 1;
|
103
101
|
}
|
@@ -113,7 +111,7 @@ static BROTLI_INLINE size_t FN(RightChildIndex)(HashToBinaryTree* self,
|
|
113
111
|
|
114
112
|
This function must be called with increasing cur_ix positions. */
|
115
113
|
static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
116
|
-
HashToBinaryTree* self, const uint8_t*
|
114
|
+
HashToBinaryTree* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
|
117
115
|
const size_t cur_ix, const size_t ring_buffer_mask, const size_t max_length,
|
118
116
|
const size_t max_backward, size_t* const BROTLI_RESTRICT best_len,
|
119
117
|
BackwardMatch* BROTLI_RESTRICT matches) {
|
@@ -123,8 +121,9 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
123
121
|
const BROTLI_BOOL should_reroot_tree =
|
124
122
|
TO_BROTLI_BOOL(max_length >= MAX_TREE_COMP_LENGTH);
|
125
123
|
const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
|
126
|
-
uint32_t*
|
127
|
-
|
124
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
125
|
+
uint32_t* BROTLI_RESTRICT forest = self->forest_;
|
126
|
+
size_t prev_ix = buckets[key];
|
128
127
|
/* The forest index of the rightmost node of the left subtree of the new
|
129
128
|
root, updated as we traverse and re-root the tree of the hash bucket. */
|
130
129
|
size_t node_left = FN(LeftChildIndex)(self, cur_ix);
|
@@ -139,7 +138,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
139
138
|
size_t best_len_right = 0;
|
140
139
|
size_t depth_remaining;
|
141
140
|
if (should_reroot_tree) {
|
142
|
-
|
141
|
+
buckets[key] = (uint32_t)cur_ix;
|
143
142
|
}
|
144
143
|
for (depth_remaining = MAX_TREE_SEARCH_DEPTH; ; --depth_remaining) {
|
145
144
|
const size_t backward = cur_ix - prev_ix;
|
@@ -199,11 +198,13 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
199
198
|
matches in matches[0] to matches[*num_matches - 1]. The matches will be
|
200
199
|
sorted by strictly increasing length and (non-strictly) increasing
|
201
200
|
distance. */
|
202
|
-
static BROTLI_INLINE size_t FN(FindAllMatches)(
|
203
|
-
|
201
|
+
static BROTLI_INLINE size_t FN(FindAllMatches)(
|
202
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
203
|
+
const BrotliEncoderDictionary* dictionary,
|
204
|
+
const uint8_t* BROTLI_RESTRICT data,
|
204
205
|
const size_t ring_buffer_mask, const size_t cur_ix,
|
205
206
|
const size_t max_length, const size_t max_backward,
|
206
|
-
const size_t
|
207
|
+
const size_t dictionary_distance, const BrotliEncoderParams* params,
|
207
208
|
BackwardMatch* matches) {
|
208
209
|
BackwardMatch* const orig_matches = matches;
|
209
210
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
@@ -236,7 +237,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
236
237
|
}
|
237
238
|
}
|
238
239
|
if (best_len < max_length) {
|
239
|
-
matches = FN(StoreAndFindMatches)(
|
240
|
+
matches = FN(StoreAndFindMatches)(self, data, cur_ix,
|
240
241
|
ring_buffer_mask, max_length, max_backward, &best_len, matches);
|
241
242
|
}
|
242
243
|
for (i = 0; i <= BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN; ++i) {
|
@@ -252,7 +253,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
252
253
|
for (l = minlen; l <= maxlen; ++l) {
|
253
254
|
uint32_t dict_id = dict_matches[l];
|
254
255
|
if (dict_id < kInvalidMatch) {
|
255
|
-
size_t distance =
|
256
|
+
size_t distance = dictionary_distance + (dict_id >> 5) + 1;
|
256
257
|
if (distance <= params->dist.max_distance) {
|
257
258
|
InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
|
258
259
|
}
|
@@ -266,18 +267,18 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
266
267
|
/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
|
267
268
|
current sequence, without returning any matches.
|
268
269
|
REQUIRES: ix + MAX_TREE_COMP_LENGTH <= end-of-current-block */
|
269
|
-
static BROTLI_INLINE void FN(Store)(
|
270
|
+
static BROTLI_INLINE void FN(Store)(HashToBinaryTree* BROTLI_RESTRICT self,
|
271
|
+
const uint8_t* BROTLI_RESTRICT data,
|
270
272
|
const size_t mask, const size_t ix) {
|
271
|
-
HashToBinaryTree* self = FN(Self)(handle);
|
272
273
|
/* Maximum distance is window size - 16, see section 9.1. of the spec. */
|
273
274
|
const size_t max_backward = self->window_mask_ - BROTLI_WINDOW_GAP + 1;
|
274
275
|
FN(StoreAndFindMatches)(self, data, ix, mask, MAX_TREE_COMP_LENGTH,
|
275
276
|
max_backward, NULL, NULL);
|
276
277
|
}
|
277
278
|
|
278
|
-
static BROTLI_INLINE void FN(StoreRange)(
|
279
|
-
const uint8_t* data, const size_t mask,
|
280
|
-
const size_t ix_end) {
|
279
|
+
static BROTLI_INLINE void FN(StoreRange)(HashToBinaryTree* BROTLI_RESTRICT self,
|
280
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
|
281
|
+
const size_t ix_start, const size_t ix_end) {
|
281
282
|
size_t i = ix_start;
|
282
283
|
size_t j = ix_start;
|
283
284
|
if (ix_start + 63 <= ix_end) {
|
@@ -285,18 +286,18 @@ static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
|
285
286
|
}
|
286
287
|
if (ix_start + 512 <= i) {
|
287
288
|
for (; j < i; j += 8) {
|
288
|
-
FN(Store)(
|
289
|
+
FN(Store)(self, data, mask, j);
|
289
290
|
}
|
290
291
|
}
|
291
292
|
for (; i < ix_end; ++i) {
|
292
|
-
FN(Store)(
|
293
|
+
FN(Store)(self, data, mask, i);
|
293
294
|
}
|
294
295
|
}
|
295
296
|
|
296
|
-
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
297
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
298
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
297
299
|
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
298
300
|
size_t ringbuffer_mask) {
|
299
|
-
HashToBinaryTree* self = FN(Self)(handle);
|
300
301
|
if (num_bytes >= FN(HashTypeLength)() - 1 &&
|
301
302
|
position >= MAX_TREE_COMP_LENGTH) {
|
302
303
|
/* Store the last `MAX_TREE_COMP_LENGTH - 1` positions in the hasher.
|
@@ -56,6 +56,18 @@ BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
|
|
56
56
|
#define BROTLI_IS_OOM(M) (!!(M)->is_oom)
|
57
57
|
#endif /* BROTLI_ENCODER_EXIT_ON_OOM */
|
58
58
|
|
59
|
+
/*
|
60
|
+
BROTLI_IS_NULL is a fake check, BROTLI_IS_OOM does the heavy lifting.
|
61
|
+
The only purpose of it is to explain static analyzers the state of things.
|
62
|
+
NB: use ONLY together with BROTLI_IS_OOM
|
63
|
+
AND ONLY for allocations in the current scope.
|
64
|
+
*/
|
65
|
+
#if defined(__clang_analyzer__) && !defined(BROTLI_ENCODER_EXIT_ON_OOM)
|
66
|
+
#define BROTLI_IS_NULL(A) ((A) == nullptr)
|
67
|
+
#else /* defined(__clang_analyzer__) */
|
68
|
+
#define BROTLI_IS_NULL(A) (!!0)
|
69
|
+
#endif /* defined(__clang_analyzer__) */
|
70
|
+
|
59
71
|
BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m);
|
60
72
|
|
61
73
|
/*
|
@@ -66,18 +78,18 @@ A: array
|
|
66
78
|
C: capacity
|
67
79
|
R: requested size
|
68
80
|
*/
|
69
|
-
#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) {
|
70
|
-
if (C < (R)) {
|
71
|
-
size_t _new_size = (C == 0) ? (R) : C;
|
72
|
-
T* new_array;
|
73
|
-
while (_new_size < (R)) _new_size *= 2;
|
74
|
-
new_array = BROTLI_ALLOC((M), T, _new_size);
|
75
|
-
if (!BROTLI_IS_OOM(M) && C != 0)
|
76
|
-
memcpy(new_array, A, C * sizeof(T));
|
77
|
-
BROTLI_FREE((M), A);
|
78
|
-
A = new_array;
|
79
|
-
C = _new_size;
|
80
|
-
}
|
81
|
+
#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \
|
82
|
+
if (C < (R)) { \
|
83
|
+
size_t _new_size = (C == 0) ? (R) : C; \
|
84
|
+
T* new_array; \
|
85
|
+
while (_new_size < (R)) _new_size *= 2; \
|
86
|
+
new_array = BROTLI_ALLOC((M), T, _new_size); \
|
87
|
+
if (!BROTLI_IS_OOM(M) && !BROTLI_IS_NULL(new_array) && C != 0) \
|
88
|
+
memcpy(new_array, A, C * sizeof(T)); \
|
89
|
+
BROTLI_FREE((M), A); \
|
90
|
+
A = new_array; \
|
91
|
+
C = _new_size; \
|
92
|
+
} \
|
81
93
|
}
|
82
94
|
|
83
95
|
/*
|
@@ -28,34 +28,30 @@ extern "C" {
|
|
28
28
|
void BrotliInitDistanceParams(BrotliEncoderParams* params,
|
29
29
|
uint32_t npostfix, uint32_t ndirect) {
|
30
30
|
BrotliDistanceParams* dist_params = ¶ms->dist;
|
31
|
-
uint32_t
|
31
|
+
uint32_t alphabet_size_max;
|
32
|
+
uint32_t alphabet_size_limit;
|
33
|
+
uint32_t max_distance;
|
32
34
|
|
33
35
|
dist_params->distance_postfix_bits = npostfix;
|
34
36
|
dist_params->num_direct_distance_codes = ndirect;
|
35
37
|
|
36
|
-
|
38
|
+
alphabet_size_max = BROTLI_DISTANCE_ALPHABET_SIZE(
|
37
39
|
npostfix, ndirect, BROTLI_MAX_DISTANCE_BITS);
|
40
|
+
alphabet_size_limit = alphabet_size_max;
|
38
41
|
max_distance = ndirect + (1U << (BROTLI_MAX_DISTANCE_BITS + npostfix + 2)) -
|
39
42
|
(1U << (npostfix + 2));
|
40
43
|
|
41
44
|
if (params->large_window) {
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
+
BrotliDistanceCodeLimit limit = BrotliCalculateDistanceCodeLimit(
|
46
|
+
BROTLI_MAX_ALLOWED_DISTANCE, npostfix, ndirect);
|
47
|
+
alphabet_size_max = BROTLI_DISTANCE_ALPHABET_SIZE(
|
45
48
|
npostfix, ndirect, BROTLI_LARGE_MAX_DISTANCE_BITS);
|
46
|
-
|
47
|
-
|
48
|
-
its extra bits set. */
|
49
|
-
if (ndirect < bound[npostfix]) {
|
50
|
-
max_distance = BROTLI_MAX_ALLOWED_DISTANCE - (bound[npostfix] - ndirect);
|
51
|
-
} else if (ndirect >= bound[npostfix] + postfix) {
|
52
|
-
max_distance = (3U << 29) - 4 + (ndirect - bound[npostfix]);
|
53
|
-
} else {
|
54
|
-
max_distance = BROTLI_MAX_ALLOWED_DISTANCE;
|
55
|
-
}
|
49
|
+
alphabet_size_limit = limit.max_alphabet_size;
|
50
|
+
max_distance = limit.max_distance;
|
56
51
|
}
|
57
52
|
|
58
|
-
dist_params->
|
53
|
+
dist_params->alphabet_size_max = alphabet_size_max;
|
54
|
+
dist_params->alphabet_size_limit = alphabet_size_limit;
|
59
55
|
dist_params->max_distance = max_distance;
|
60
56
|
}
|
61
57
|
|
@@ -200,7 +196,7 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
200
196
|
literal_context_multiplier = 1 << BROTLI_LITERAL_CONTEXT_BITS;
|
201
197
|
literal_context_modes =
|
202
198
|
BROTLI_ALLOC(m, ContextType, mb->literal_split.num_types);
|
203
|
-
if (BROTLI_IS_OOM(m)) return;
|
199
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(literal_context_modes)) return;
|
204
200
|
for (i = 0; i < mb->literal_split.num_types; ++i) {
|
205
201
|
literal_context_modes[i] = literal_context_mode;
|
206
202
|
}
|
@@ -210,21 +206,21 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
210
206
|
mb->literal_split.num_types * literal_context_multiplier;
|
211
207
|
literal_histograms =
|
212
208
|
BROTLI_ALLOC(m, HistogramLiteral, literal_histograms_size);
|
213
|
-
if (BROTLI_IS_OOM(m)) return;
|
209
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(literal_histograms)) return;
|
214
210
|
ClearHistogramsLiteral(literal_histograms, literal_histograms_size);
|
215
211
|
|
216
212
|
distance_histograms_size =
|
217
213
|
mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;
|
218
214
|
distance_histograms =
|
219
215
|
BROTLI_ALLOC(m, HistogramDistance, distance_histograms_size);
|
220
|
-
if (BROTLI_IS_OOM(m)) return;
|
216
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(distance_histograms)) return;
|
221
217
|
ClearHistogramsDistance(distance_histograms, distance_histograms_size);
|
222
218
|
|
223
219
|
BROTLI_DCHECK(mb->command_histograms == 0);
|
224
220
|
mb->command_histograms_size = mb->command_split.num_types;
|
225
221
|
mb->command_histograms =
|
226
222
|
BROTLI_ALLOC(m, HistogramCommand, mb->command_histograms_size);
|
227
|
-
if (BROTLI_IS_OOM(m)) return;
|
223
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->command_histograms)) return;
|
228
224
|
ClearHistogramsCommand(mb->command_histograms, mb->command_histograms_size);
|
229
225
|
|
230
226
|
BrotliBuildHistogramsWithContext(cmds, num_commands,
|
@@ -238,13 +234,13 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
238
234
|
mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
|
239
235
|
mb->literal_context_map =
|
240
236
|
BROTLI_ALLOC(m, uint32_t, mb->literal_context_map_size);
|
241
|
-
if (BROTLI_IS_OOM(m)) return;
|
237
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->literal_context_map)) return;
|
242
238
|
|
243
239
|
BROTLI_DCHECK(mb->literal_histograms == 0);
|
244
240
|
mb->literal_histograms_size = mb->literal_context_map_size;
|
245
241
|
mb->literal_histograms =
|
246
242
|
BROTLI_ALLOC(m, HistogramLiteral, mb->literal_histograms_size);
|
247
|
-
if (BROTLI_IS_OOM(m)) return;
|
243
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->literal_histograms)) return;
|
248
244
|
|
249
245
|
BrotliClusterHistogramsLiteral(m, literal_histograms, literal_histograms_size,
|
250
246
|
kMaxNumberOfHistograms, mb->literal_histograms,
|
@@ -269,13 +265,13 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
269
265
|
mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;
|
270
266
|
mb->distance_context_map =
|
271
267
|
BROTLI_ALLOC(m, uint32_t, mb->distance_context_map_size);
|
272
|
-
if (BROTLI_IS_OOM(m)) return;
|
268
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->distance_context_map)) return;
|
273
269
|
|
274
270
|
BROTLI_DCHECK(mb->distance_histograms == 0);
|
275
271
|
mb->distance_histograms_size = mb->distance_context_map_size;
|
276
272
|
mb->distance_histograms =
|
277
273
|
BROTLI_ALLOC(m, HistogramDistance, mb->distance_histograms_size);
|
278
|
-
if (BROTLI_IS_OOM(m)) return;
|
274
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->distance_histograms)) return;
|
279
275
|
|
280
276
|
BrotliClusterHistogramsDistance(m, distance_histograms,
|
281
277
|
mb->distance_context_map_size,
|
@@ -373,7 +369,7 @@ static void InitContextBlockSplitter(
|
|
373
369
|
*histograms_size = max_num_types * num_contexts;
|
374
370
|
*histograms = BROTLI_ALLOC(m, HistogramLiteral, *histograms_size);
|
375
371
|
self->histograms_ = *histograms;
|
376
|
-
if (BROTLI_IS_OOM(m)) return;
|
372
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(*histograms)) return;
|
377
373
|
/* Clear only current histogram. */
|
378
374
|
ClearHistogramsLiteral(&self->histograms_[0], num_contexts);
|
379
375
|
self->last_histogram_ix_[0] = self->last_histogram_ix_[1] = 0;
|
@@ -423,7 +419,7 @@ static void ContextBlockSplitterFinishBlock(
|
|
423
419
|
double combined_entropy[2 * BROTLI_MAX_STATIC_CONTEXTS];
|
424
420
|
double diff[2] = { 0.0 };
|
425
421
|
size_t i;
|
426
|
-
if (BROTLI_IS_OOM(m)) return;
|
422
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(combined_histo)) return;
|
427
423
|
for (i = 0; i < num_contexts; ++i) {
|
428
424
|
size_t curr_histo_ix = self->curr_histogram_ix_ + i;
|
429
425
|
size_t j;
|
@@ -527,7 +523,7 @@ static void MapStaticContexts(MemoryManager* m,
|
|
527
523
|
mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
|
528
524
|
mb->literal_context_map =
|
529
525
|
BROTLI_ALLOC(m, uint32_t, mb->literal_context_map_size);
|
530
|
-
if (BROTLI_IS_OOM(m)) return;
|
526
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->literal_context_map)) return;
|
531
527
|
|
532
528
|
for (i = 0; i < mb->literal_split.num_types; ++i) {
|
533
529
|
uint32_t offset = (uint32_t)(i * num_contexts);
|