brotli 0.2.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +34 -0
- data/.github/workflows/publish.yml +34 -0
- data/Gemfile +6 -3
- data/Rakefile +16 -9
- data/brotli.gemspec +7 -13
- data/ext/brotli/brotli.c +209 -31
- data/ext/brotli/buffer.c +1 -7
- data/ext/brotli/buffer.h +1 -1
- data/ext/brotli/extconf.rb +20 -18
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +104 -0
- data/test/brotli_writer_test.rb +36 -0
- data/test/test_helper.rb +8 -0
- data/vendor/brotli/c/common/constants.c +15 -0
- data/vendor/brotli/c/common/constants.h +136 -0
- data/vendor/brotli/c/common/context.c +156 -0
- data/vendor/brotli/c/common/context.h +4 -152
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +10 -1
- data/vendor/brotli/c/common/platform.c +22 -0
- data/vendor/brotli/c/common/platform.h +43 -17
- data/vendor/brotli/c/common/transform.c +59 -3
- data/vendor/brotli/c/common/transform.h +5 -0
- data/vendor/brotli/c/common/version.h +2 -2
- data/vendor/brotli/c/dec/bit_reader.c +28 -0
- data/vendor/brotli/c/dec/bit_reader.h +58 -16
- data/vendor/brotli/c/dec/decode.c +353 -251
- data/vendor/brotli/c/dec/huffman.h +6 -12
- data/vendor/brotli/c/dec/prefix.h +0 -18
- data/vendor/brotli/c/dec/state.c +9 -14
- data/vendor/brotli/c/dec/state.h +144 -37
- data/vendor/brotli/c/enc/backward_references.c +8 -7
- data/vendor/brotli/c/enc/backward_references.h +5 -4
- data/vendor/brotli/c/enc/backward_references_hq.c +51 -33
- data/vendor/brotli/c/enc/backward_references_hq.h +11 -8
- data/vendor/brotli/c/enc/backward_references_inc.h +24 -14
- data/vendor/brotli/c/enc/block_splitter.c +3 -3
- data/vendor/brotli/c/enc/block_splitter_inc.h +15 -6
- data/vendor/brotli/c/enc/brotli_bit_stream.c +13 -30
- data/vendor/brotli/c/enc/cluster_inc.h +6 -3
- data/vendor/brotli/c/enc/command.c +28 -0
- data/vendor/brotli/c/enc/command.h +12 -12
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1826 -1100
- data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
- data/vendor/brotli/c/enc/encode.c +104 -39
- data/vendor/brotli/c/enc/encoder_dict.c +3 -2
- data/vendor/brotli/c/enc/encoder_dict.h +3 -1
- data/vendor/brotli/c/enc/entropy_encode.c +2 -0
- data/vendor/brotli/c/enc/entropy_encode.h +2 -2
- data/vendor/brotli/c/enc/fast_log.c +105 -0
- data/vendor/brotli/c/enc/fast_log.h +19 -100
- data/vendor/brotli/c/enc/find_match_length.h +2 -3
- data/vendor/brotli/c/enc/hash.h +80 -90
- data/vendor/brotli/c/enc/hash_composite_inc.h +52 -63
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +88 -49
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +50 -50
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +53 -50
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +91 -60
- data/vendor/brotli/c/enc/hash_rolling_inc.h +23 -27
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +39 -38
- data/vendor/brotli/c/enc/memory.h +24 -12
- data/vendor/brotli/c/enc/metablock.c +23 -27
- data/vendor/brotli/c/enc/metablock_inc.h +1 -1
- data/vendor/brotli/c/enc/params.h +3 -1
- data/vendor/brotli/c/enc/ringbuffer.h +4 -1
- data/vendor/brotli/c/enc/utf8_util.c +1 -1
- data/vendor/brotli/c/enc/write_bits.h +27 -25
- data/vendor/brotli/c/include/brotli/encode.h +22 -1
- data/vendor/brotli/c/include/brotli/port.h +14 -0
- metadata +17 -97
- data/.travis.yml +0 -31
- data/docs/Brotli.html +0 -485
- data/docs/Brotli/Error.html +0 -124
- data/docs/_index.html +0 -122
- data/docs/class_list.html +0 -51
- data/docs/css/common.css +0 -1
- data/docs/css/full_list.css +0 -58
- data/docs/css/style.css +0 -496
- data/docs/file.README.html +0 -127
- data/docs/file_list.html +0 -56
- data/docs/frames.html +0 -17
- data/docs/index.html +0 -127
- data/docs/js/app.js +0 -292
- data/docs/js/full_list.js +0 -216
- data/docs/js/jquery.js +0 -4
- data/docs/method_list.html +0 -67
- data/docs/top-level-namespace.html +0 -110
- data/spec/brotli_spec.rb +0 -88
- data/spec/inflate_spec.rb +0 -75
- data/spec/spec_helper.rb +0 -4
@@ -51,13 +51,9 @@ typedef struct HashRolling {
|
|
51
51
|
uint32_t factor_remove;
|
52
52
|
} HashRolling;
|
53
53
|
|
54
|
-
static BROTLI_INLINE HashRolling* FN(Self)(HasherHandle handle) {
|
55
|
-
return (HashRolling*)&(GetHasherCommon(handle)[1]);
|
56
|
-
}
|
57
|
-
|
58
54
|
static void FN(Initialize)(
|
59
|
-
|
60
|
-
|
55
|
+
HasherCommon* common, HashRolling* BROTLI_RESTRICT self,
|
56
|
+
const BrotliEncoderParams* params) {
|
61
57
|
size_t i;
|
62
58
|
self->state = 0;
|
63
59
|
self->next_ix = 0;
|
@@ -71,7 +67,7 @@ static void FN(Initialize)(
|
|
71
67
|
self->factor_remove *= self->factor;
|
72
68
|
}
|
73
69
|
|
74
|
-
self->table = (uint32_t*)
|
70
|
+
self->table = (uint32_t*)common->extra;
|
75
71
|
for (i = 0; i < NUMBUCKETS; i++) {
|
76
72
|
self->table[i] = FN(kInvalidPos);
|
77
73
|
}
|
@@ -79,9 +75,8 @@ static void FN(Initialize)(
|
|
79
75
|
BROTLI_UNUSED(params);
|
80
76
|
}
|
81
77
|
|
82
|
-
static void FN(Prepare)(
|
83
|
-
size_t input_size, const uint8_t* data) {
|
84
|
-
HashRolling* self = FN(Self)(handle);
|
78
|
+
static void FN(Prepare)(HashRolling* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
|
79
|
+
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
|
85
80
|
size_t i;
|
86
81
|
/* Too small size, cannot use this hasher. */
|
87
82
|
if (input_size < CHUNKLEN) return;
|
@@ -96,36 +91,36 @@ static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
|
|
96
91
|
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
97
92
|
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
98
93
|
size_t input_size) {
|
99
|
-
return
|
94
|
+
return NUMBUCKETS * sizeof(uint32_t);
|
100
95
|
BROTLI_UNUSED(params);
|
101
96
|
BROTLI_UNUSED(one_shot);
|
102
97
|
BROTLI_UNUSED(input_size);
|
103
98
|
}
|
104
99
|
|
105
|
-
static BROTLI_INLINE void FN(Store)(
|
100
|
+
static BROTLI_INLINE void FN(Store)(HashRolling* BROTLI_RESTRICT self,
|
106
101
|
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
|
107
|
-
BROTLI_UNUSED(
|
102
|
+
BROTLI_UNUSED(self);
|
108
103
|
BROTLI_UNUSED(data);
|
109
104
|
BROTLI_UNUSED(mask);
|
110
105
|
BROTLI_UNUSED(ix);
|
111
106
|
}
|
112
107
|
|
113
|
-
static BROTLI_INLINE void FN(StoreRange)(
|
114
|
-
const uint8_t* data, const size_t mask,
|
115
|
-
const size_t ix_end) {
|
116
|
-
BROTLI_UNUSED(
|
108
|
+
static BROTLI_INLINE void FN(StoreRange)(HashRolling* BROTLI_RESTRICT self,
|
109
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
|
110
|
+
const size_t ix_start, const size_t ix_end) {
|
111
|
+
BROTLI_UNUSED(self);
|
117
112
|
BROTLI_UNUSED(data);
|
118
113
|
BROTLI_UNUSED(mask);
|
119
114
|
BROTLI_UNUSED(ix_start);
|
120
115
|
BROTLI_UNUSED(ix_end);
|
121
116
|
}
|
122
117
|
|
123
|
-
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
118
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
119
|
+
HashRolling* BROTLI_RESTRICT self,
|
124
120
|
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
125
121
|
size_t ring_buffer_mask) {
|
126
122
|
/* In this case we must re-initialize the hasher from scratch from the
|
127
123
|
current position. */
|
128
|
-
HashRolling* self = FN(Self)(handle);
|
129
124
|
size_t position_masked;
|
130
125
|
size_t available = num_bytes;
|
131
126
|
if ((position & (JUMP - 1)) != 0) {
|
@@ -139,28 +134,29 @@ static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
|
|
139
134
|
available = ring_buffer_mask - position_masked;
|
140
135
|
}
|
141
136
|
|
142
|
-
FN(Prepare)(
|
137
|
+
FN(Prepare)(self, BROTLI_FALSE, available,
|
143
138
|
ringbuffer + (position & ring_buffer_mask));
|
144
139
|
self->next_ix = position;
|
145
140
|
BROTLI_UNUSED(num_bytes);
|
146
141
|
}
|
147
142
|
|
148
143
|
static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
149
|
-
|
150
|
-
|
144
|
+
HashRolling* BROTLI_RESTRICT self,
|
145
|
+
int* BROTLI_RESTRICT distance_cache) {
|
146
|
+
BROTLI_UNUSED(self);
|
151
147
|
BROTLI_UNUSED(distance_cache);
|
152
148
|
}
|
153
149
|
|
154
|
-
static BROTLI_INLINE void FN(FindLongestMatch)(
|
150
|
+
static BROTLI_INLINE void FN(FindLongestMatch)(
|
151
|
+
HashRolling* BROTLI_RESTRICT self,
|
155
152
|
const BrotliEncoderDictionary* dictionary,
|
156
153
|
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
157
154
|
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
158
155
|
const size_t max_length, const size_t max_backward,
|
159
|
-
const size_t
|
156
|
+
const size_t dictionary_distance, const size_t max_distance,
|
160
157
|
HasherSearchResult* BROTLI_RESTRICT out) {
|
161
|
-
HashRolling* self = FN(Self)(handle);
|
162
158
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
163
|
-
size_t pos
|
159
|
+
size_t pos;
|
164
160
|
|
165
161
|
if ((cur_ix & (JUMP - 1)) != 0) return;
|
166
162
|
|
@@ -209,7 +205,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
209
205
|
backup-hasher, the main hasher already searches in it. */
|
210
206
|
BROTLI_UNUSED(dictionary);
|
211
207
|
BROTLI_UNUSED(distance_cache);
|
212
|
-
BROTLI_UNUSED(
|
208
|
+
BROTLI_UNUSED(dictionary_distance);
|
213
209
|
BROTLI_UNUSED(max_distance);
|
214
210
|
}
|
215
211
|
|
@@ -24,7 +24,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
|
|
24
24
|
return MAX_TREE_COMP_LENGTH;
|
25
25
|
}
|
26
26
|
|
27
|
-
static uint32_t FN(HashBytes)(const uint8_t* data) {
|
27
|
+
static uint32_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data) {
|
28
28
|
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
29
29
|
/* The higher bits contain more mixture from the multiplication,
|
30
30
|
so we take our results from there. */
|
@@ -38,7 +38,7 @@ typedef struct HashToBinaryTree {
|
|
38
38
|
/* Hash table that maps the 4-byte hashes of the sequence to the last
|
39
39
|
position where this hash was found, which is the root of the binary
|
40
40
|
tree of sequences that share this hash bucket. */
|
41
|
-
uint32_t buckets_[BUCKET_SIZE];
|
41
|
+
uint32_t* buckets_; /* uint32_t[BUCKET_SIZE]; */
|
42
42
|
|
43
43
|
/* A position used to mark a non-existent sequence, i.e. a tree is empty if
|
44
44
|
its root is at invalid_pos_ and a node is a leaf if both its children
|
@@ -51,34 +51,30 @@ typedef struct HashToBinaryTree {
|
|
51
51
|
corresponding to a hash is a sequence starting at buckets_[hash] and
|
52
52
|
the left and right children of a sequence starting at pos are
|
53
53
|
forest_[2 * pos] and forest_[2 * pos + 1]. */
|
54
|
-
/* uint32_t
|
54
|
+
uint32_t* forest_; /* uint32_t[2 * num_nodes] */
|
55
55
|
} HashToBinaryTree;
|
56
56
|
|
57
|
-
static BROTLI_INLINE HashToBinaryTree* FN(Self)(HasherHandle handle) {
|
58
|
-
return (HashToBinaryTree*)&(GetHasherCommon(handle)[1]);
|
59
|
-
}
|
60
|
-
|
61
|
-
static BROTLI_INLINE uint32_t* FN(Forest)(HashToBinaryTree* self) {
|
62
|
-
return (uint32_t*)(&self[1]);
|
63
|
-
}
|
64
|
-
|
65
57
|
static void FN(Initialize)(
|
66
|
-
|
67
|
-
|
58
|
+
HasherCommon* common, HashToBinaryTree* BROTLI_RESTRICT self,
|
59
|
+
const BrotliEncoderParams* params) {
|
60
|
+
self->buckets_ = (uint32_t*)common->extra;
|
61
|
+
self->forest_ = &self->buckets_[BUCKET_SIZE];
|
62
|
+
|
68
63
|
self->window_mask_ = (1u << params->lgwin) - 1u;
|
69
64
|
self->invalid_pos_ = (uint32_t)(0 - self->window_mask_);
|
70
65
|
}
|
71
66
|
|
72
|
-
static void FN(Prepare)
|
73
|
-
|
74
|
-
|
67
|
+
static void FN(Prepare)
|
68
|
+
(HashToBinaryTree* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
|
69
|
+
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
|
75
70
|
uint32_t invalid_pos = self->invalid_pos_;
|
76
71
|
uint32_t i;
|
72
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
77
73
|
BROTLI_UNUSED(data);
|
78
74
|
BROTLI_UNUSED(one_shot);
|
79
75
|
BROTLI_UNUSED(input_size);
|
80
76
|
for (i = 0; i < BUCKET_SIZE; i++) {
|
81
|
-
|
77
|
+
buckets[i] = invalid_pos;
|
82
78
|
}
|
83
79
|
}
|
84
80
|
|
@@ -89,15 +85,17 @@ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
|
89
85
|
if (one_shot && input_size < num_nodes) {
|
90
86
|
num_nodes = input_size;
|
91
87
|
}
|
92
|
-
return sizeof(
|
88
|
+
return sizeof(uint32_t) * BUCKET_SIZE + 2 * sizeof(uint32_t) * num_nodes;
|
93
89
|
}
|
94
90
|
|
95
|
-
static BROTLI_INLINE size_t FN(LeftChildIndex)(
|
91
|
+
static BROTLI_INLINE size_t FN(LeftChildIndex)(
|
92
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
96
93
|
const size_t pos) {
|
97
94
|
return 2 * (pos & self->window_mask_);
|
98
95
|
}
|
99
96
|
|
100
|
-
static BROTLI_INLINE size_t FN(RightChildIndex)(
|
97
|
+
static BROTLI_INLINE size_t FN(RightChildIndex)(
|
98
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
101
99
|
const size_t pos) {
|
102
100
|
return 2 * (pos & self->window_mask_) + 1;
|
103
101
|
}
|
@@ -113,7 +111,7 @@ static BROTLI_INLINE size_t FN(RightChildIndex)(HashToBinaryTree* self,
|
|
113
111
|
|
114
112
|
This function must be called with increasing cur_ix positions. */
|
115
113
|
static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
116
|
-
HashToBinaryTree* self, const uint8_t*
|
114
|
+
HashToBinaryTree* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
|
117
115
|
const size_t cur_ix, const size_t ring_buffer_mask, const size_t max_length,
|
118
116
|
const size_t max_backward, size_t* const BROTLI_RESTRICT best_len,
|
119
117
|
BackwardMatch* BROTLI_RESTRICT matches) {
|
@@ -123,8 +121,9 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
123
121
|
const BROTLI_BOOL should_reroot_tree =
|
124
122
|
TO_BROTLI_BOOL(max_length >= MAX_TREE_COMP_LENGTH);
|
125
123
|
const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
|
126
|
-
uint32_t*
|
127
|
-
|
124
|
+
uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
|
125
|
+
uint32_t* BROTLI_RESTRICT forest = self->forest_;
|
126
|
+
size_t prev_ix = buckets[key];
|
128
127
|
/* The forest index of the rightmost node of the left subtree of the new
|
129
128
|
root, updated as we traverse and re-root the tree of the hash bucket. */
|
130
129
|
size_t node_left = FN(LeftChildIndex)(self, cur_ix);
|
@@ -139,7 +138,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
139
138
|
size_t best_len_right = 0;
|
140
139
|
size_t depth_remaining;
|
141
140
|
if (should_reroot_tree) {
|
142
|
-
|
141
|
+
buckets[key] = (uint32_t)cur_ix;
|
143
142
|
}
|
144
143
|
for (depth_remaining = MAX_TREE_SEARCH_DEPTH; ; --depth_remaining) {
|
145
144
|
const size_t backward = cur_ix - prev_ix;
|
@@ -199,11 +198,13 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
|
199
198
|
matches in matches[0] to matches[*num_matches - 1]. The matches will be
|
200
199
|
sorted by strictly increasing length and (non-strictly) increasing
|
201
200
|
distance. */
|
202
|
-
static BROTLI_INLINE size_t FN(FindAllMatches)(
|
203
|
-
|
201
|
+
static BROTLI_INLINE size_t FN(FindAllMatches)(
|
202
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
203
|
+
const BrotliEncoderDictionary* dictionary,
|
204
|
+
const uint8_t* BROTLI_RESTRICT data,
|
204
205
|
const size_t ring_buffer_mask, const size_t cur_ix,
|
205
206
|
const size_t max_length, const size_t max_backward,
|
206
|
-
const size_t
|
207
|
+
const size_t dictionary_distance, const BrotliEncoderParams* params,
|
207
208
|
BackwardMatch* matches) {
|
208
209
|
BackwardMatch* const orig_matches = matches;
|
209
210
|
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
|
@@ -236,7 +237,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
236
237
|
}
|
237
238
|
}
|
238
239
|
if (best_len < max_length) {
|
239
|
-
matches = FN(StoreAndFindMatches)(
|
240
|
+
matches = FN(StoreAndFindMatches)(self, data, cur_ix,
|
240
241
|
ring_buffer_mask, max_length, max_backward, &best_len, matches);
|
241
242
|
}
|
242
243
|
for (i = 0; i <= BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN; ++i) {
|
@@ -252,7 +253,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
252
253
|
for (l = minlen; l <= maxlen; ++l) {
|
253
254
|
uint32_t dict_id = dict_matches[l];
|
254
255
|
if (dict_id < kInvalidMatch) {
|
255
|
-
size_t distance =
|
256
|
+
size_t distance = dictionary_distance + (dict_id >> 5) + 1;
|
256
257
|
if (distance <= params->dist.max_distance) {
|
257
258
|
InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
|
258
259
|
}
|
@@ -266,18 +267,18 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
|
|
266
267
|
/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
|
267
268
|
current sequence, without returning any matches.
|
268
269
|
REQUIRES: ix + MAX_TREE_COMP_LENGTH <= end-of-current-block */
|
269
|
-
static BROTLI_INLINE void FN(Store)(
|
270
|
+
static BROTLI_INLINE void FN(Store)(HashToBinaryTree* BROTLI_RESTRICT self,
|
271
|
+
const uint8_t* BROTLI_RESTRICT data,
|
270
272
|
const size_t mask, const size_t ix) {
|
271
|
-
HashToBinaryTree* self = FN(Self)(handle);
|
272
273
|
/* Maximum distance is window size - 16, see section 9.1. of the spec. */
|
273
274
|
const size_t max_backward = self->window_mask_ - BROTLI_WINDOW_GAP + 1;
|
274
275
|
FN(StoreAndFindMatches)(self, data, ix, mask, MAX_TREE_COMP_LENGTH,
|
275
276
|
max_backward, NULL, NULL);
|
276
277
|
}
|
277
278
|
|
278
|
-
static BROTLI_INLINE void FN(StoreRange)(
|
279
|
-
const uint8_t* data, const size_t mask,
|
280
|
-
const size_t ix_end) {
|
279
|
+
static BROTLI_INLINE void FN(StoreRange)(HashToBinaryTree* BROTLI_RESTRICT self,
|
280
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask,
|
281
|
+
const size_t ix_start, const size_t ix_end) {
|
281
282
|
size_t i = ix_start;
|
282
283
|
size_t j = ix_start;
|
283
284
|
if (ix_start + 63 <= ix_end) {
|
@@ -285,18 +286,18 @@ static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
|
285
286
|
}
|
286
287
|
if (ix_start + 512 <= i) {
|
287
288
|
for (; j < i; j += 8) {
|
288
|
-
FN(Store)(
|
289
|
+
FN(Store)(self, data, mask, j);
|
289
290
|
}
|
290
291
|
}
|
291
292
|
for (; i < ix_end; ++i) {
|
292
|
-
FN(Store)(
|
293
|
+
FN(Store)(self, data, mask, i);
|
293
294
|
}
|
294
295
|
}
|
295
296
|
|
296
|
-
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
297
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(
|
298
|
+
HashToBinaryTree* BROTLI_RESTRICT self,
|
297
299
|
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
298
300
|
size_t ringbuffer_mask) {
|
299
|
-
HashToBinaryTree* self = FN(Self)(handle);
|
300
301
|
if (num_bytes >= FN(HashTypeLength)() - 1 &&
|
301
302
|
position >= MAX_TREE_COMP_LENGTH) {
|
302
303
|
/* Store the last `MAX_TREE_COMP_LENGTH - 1` positions in the hasher.
|
@@ -56,6 +56,18 @@ BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
|
|
56
56
|
#define BROTLI_IS_OOM(M) (!!(M)->is_oom)
|
57
57
|
#endif /* BROTLI_ENCODER_EXIT_ON_OOM */
|
58
58
|
|
59
|
+
/*
|
60
|
+
BROTLI_IS_NULL is a fake check, BROTLI_IS_OOM does the heavy lifting.
|
61
|
+
The only purpose of it is to explain static analyzers the state of things.
|
62
|
+
NB: use ONLY together with BROTLI_IS_OOM
|
63
|
+
AND ONLY for allocations in the current scope.
|
64
|
+
*/
|
65
|
+
#if defined(__clang_analyzer__) && !defined(BROTLI_ENCODER_EXIT_ON_OOM)
|
66
|
+
#define BROTLI_IS_NULL(A) ((A) == nullptr)
|
67
|
+
#else /* defined(__clang_analyzer__) */
|
68
|
+
#define BROTLI_IS_NULL(A) (!!0)
|
69
|
+
#endif /* defined(__clang_analyzer__) */
|
70
|
+
|
59
71
|
BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m);
|
60
72
|
|
61
73
|
/*
|
@@ -66,18 +78,18 @@ A: array
|
|
66
78
|
C: capacity
|
67
79
|
R: requested size
|
68
80
|
*/
|
69
|
-
#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) {
|
70
|
-
if (C < (R)) {
|
71
|
-
size_t _new_size = (C == 0) ? (R) : C;
|
72
|
-
T* new_array;
|
73
|
-
while (_new_size < (R)) _new_size *= 2;
|
74
|
-
new_array = BROTLI_ALLOC((M), T, _new_size);
|
75
|
-
if (!BROTLI_IS_OOM(M) && C != 0)
|
76
|
-
memcpy(new_array, A, C * sizeof(T));
|
77
|
-
BROTLI_FREE((M), A);
|
78
|
-
A = new_array;
|
79
|
-
C = _new_size;
|
80
|
-
}
|
81
|
+
#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \
|
82
|
+
if (C < (R)) { \
|
83
|
+
size_t _new_size = (C == 0) ? (R) : C; \
|
84
|
+
T* new_array; \
|
85
|
+
while (_new_size < (R)) _new_size *= 2; \
|
86
|
+
new_array = BROTLI_ALLOC((M), T, _new_size); \
|
87
|
+
if (!BROTLI_IS_OOM(M) && !BROTLI_IS_NULL(new_array) && C != 0) \
|
88
|
+
memcpy(new_array, A, C * sizeof(T)); \
|
89
|
+
BROTLI_FREE((M), A); \
|
90
|
+
A = new_array; \
|
91
|
+
C = _new_size; \
|
92
|
+
} \
|
81
93
|
}
|
82
94
|
|
83
95
|
/*
|
@@ -28,34 +28,30 @@ extern "C" {
|
|
28
28
|
void BrotliInitDistanceParams(BrotliEncoderParams* params,
|
29
29
|
uint32_t npostfix, uint32_t ndirect) {
|
30
30
|
BrotliDistanceParams* dist_params = ¶ms->dist;
|
31
|
-
uint32_t
|
31
|
+
uint32_t alphabet_size_max;
|
32
|
+
uint32_t alphabet_size_limit;
|
33
|
+
uint32_t max_distance;
|
32
34
|
|
33
35
|
dist_params->distance_postfix_bits = npostfix;
|
34
36
|
dist_params->num_direct_distance_codes = ndirect;
|
35
37
|
|
36
|
-
|
38
|
+
alphabet_size_max = BROTLI_DISTANCE_ALPHABET_SIZE(
|
37
39
|
npostfix, ndirect, BROTLI_MAX_DISTANCE_BITS);
|
40
|
+
alphabet_size_limit = alphabet_size_max;
|
38
41
|
max_distance = ndirect + (1U << (BROTLI_MAX_DISTANCE_BITS + npostfix + 2)) -
|
39
42
|
(1U << (npostfix + 2));
|
40
43
|
|
41
44
|
if (params->large_window) {
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
+
BrotliDistanceCodeLimit limit = BrotliCalculateDistanceCodeLimit(
|
46
|
+
BROTLI_MAX_ALLOWED_DISTANCE, npostfix, ndirect);
|
47
|
+
alphabet_size_max = BROTLI_DISTANCE_ALPHABET_SIZE(
|
45
48
|
npostfix, ndirect, BROTLI_LARGE_MAX_DISTANCE_BITS);
|
46
|
-
|
47
|
-
|
48
|
-
its extra bits set. */
|
49
|
-
if (ndirect < bound[npostfix]) {
|
50
|
-
max_distance = BROTLI_MAX_ALLOWED_DISTANCE - (bound[npostfix] - ndirect);
|
51
|
-
} else if (ndirect >= bound[npostfix] + postfix) {
|
52
|
-
max_distance = (3U << 29) - 4 + (ndirect - bound[npostfix]);
|
53
|
-
} else {
|
54
|
-
max_distance = BROTLI_MAX_ALLOWED_DISTANCE;
|
55
|
-
}
|
49
|
+
alphabet_size_limit = limit.max_alphabet_size;
|
50
|
+
max_distance = limit.max_distance;
|
56
51
|
}
|
57
52
|
|
58
|
-
dist_params->
|
53
|
+
dist_params->alphabet_size_max = alphabet_size_max;
|
54
|
+
dist_params->alphabet_size_limit = alphabet_size_limit;
|
59
55
|
dist_params->max_distance = max_distance;
|
60
56
|
}
|
61
57
|
|
@@ -200,7 +196,7 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
200
196
|
literal_context_multiplier = 1 << BROTLI_LITERAL_CONTEXT_BITS;
|
201
197
|
literal_context_modes =
|
202
198
|
BROTLI_ALLOC(m, ContextType, mb->literal_split.num_types);
|
203
|
-
if (BROTLI_IS_OOM(m)) return;
|
199
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(literal_context_modes)) return;
|
204
200
|
for (i = 0; i < mb->literal_split.num_types; ++i) {
|
205
201
|
literal_context_modes[i] = literal_context_mode;
|
206
202
|
}
|
@@ -210,21 +206,21 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
210
206
|
mb->literal_split.num_types * literal_context_multiplier;
|
211
207
|
literal_histograms =
|
212
208
|
BROTLI_ALLOC(m, HistogramLiteral, literal_histograms_size);
|
213
|
-
if (BROTLI_IS_OOM(m)) return;
|
209
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(literal_histograms)) return;
|
214
210
|
ClearHistogramsLiteral(literal_histograms, literal_histograms_size);
|
215
211
|
|
216
212
|
distance_histograms_size =
|
217
213
|
mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;
|
218
214
|
distance_histograms =
|
219
215
|
BROTLI_ALLOC(m, HistogramDistance, distance_histograms_size);
|
220
|
-
if (BROTLI_IS_OOM(m)) return;
|
216
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(distance_histograms)) return;
|
221
217
|
ClearHistogramsDistance(distance_histograms, distance_histograms_size);
|
222
218
|
|
223
219
|
BROTLI_DCHECK(mb->command_histograms == 0);
|
224
220
|
mb->command_histograms_size = mb->command_split.num_types;
|
225
221
|
mb->command_histograms =
|
226
222
|
BROTLI_ALLOC(m, HistogramCommand, mb->command_histograms_size);
|
227
|
-
if (BROTLI_IS_OOM(m)) return;
|
223
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->command_histograms)) return;
|
228
224
|
ClearHistogramsCommand(mb->command_histograms, mb->command_histograms_size);
|
229
225
|
|
230
226
|
BrotliBuildHistogramsWithContext(cmds, num_commands,
|
@@ -238,13 +234,13 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
238
234
|
mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
|
239
235
|
mb->literal_context_map =
|
240
236
|
BROTLI_ALLOC(m, uint32_t, mb->literal_context_map_size);
|
241
|
-
if (BROTLI_IS_OOM(m)) return;
|
237
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->literal_context_map)) return;
|
242
238
|
|
243
239
|
BROTLI_DCHECK(mb->literal_histograms == 0);
|
244
240
|
mb->literal_histograms_size = mb->literal_context_map_size;
|
245
241
|
mb->literal_histograms =
|
246
242
|
BROTLI_ALLOC(m, HistogramLiteral, mb->literal_histograms_size);
|
247
|
-
if (BROTLI_IS_OOM(m)) return;
|
243
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->literal_histograms)) return;
|
248
244
|
|
249
245
|
BrotliClusterHistogramsLiteral(m, literal_histograms, literal_histograms_size,
|
250
246
|
kMaxNumberOfHistograms, mb->literal_histograms,
|
@@ -269,13 +265,13 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
269
265
|
mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;
|
270
266
|
mb->distance_context_map =
|
271
267
|
BROTLI_ALLOC(m, uint32_t, mb->distance_context_map_size);
|
272
|
-
if (BROTLI_IS_OOM(m)) return;
|
268
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->distance_context_map)) return;
|
273
269
|
|
274
270
|
BROTLI_DCHECK(mb->distance_histograms == 0);
|
275
271
|
mb->distance_histograms_size = mb->distance_context_map_size;
|
276
272
|
mb->distance_histograms =
|
277
273
|
BROTLI_ALLOC(m, HistogramDistance, mb->distance_histograms_size);
|
278
|
-
if (BROTLI_IS_OOM(m)) return;
|
274
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->distance_histograms)) return;
|
279
275
|
|
280
276
|
BrotliClusterHistogramsDistance(m, distance_histograms,
|
281
277
|
mb->distance_context_map_size,
|
@@ -373,7 +369,7 @@ static void InitContextBlockSplitter(
|
|
373
369
|
*histograms_size = max_num_types * num_contexts;
|
374
370
|
*histograms = BROTLI_ALLOC(m, HistogramLiteral, *histograms_size);
|
375
371
|
self->histograms_ = *histograms;
|
376
|
-
if (BROTLI_IS_OOM(m)) return;
|
372
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(*histograms)) return;
|
377
373
|
/* Clear only current histogram. */
|
378
374
|
ClearHistogramsLiteral(&self->histograms_[0], num_contexts);
|
379
375
|
self->last_histogram_ix_[0] = self->last_histogram_ix_[1] = 0;
|
@@ -423,7 +419,7 @@ static void ContextBlockSplitterFinishBlock(
|
|
423
419
|
double combined_entropy[2 * BROTLI_MAX_STATIC_CONTEXTS];
|
424
420
|
double diff[2] = { 0.0 };
|
425
421
|
size_t i;
|
426
|
-
if (BROTLI_IS_OOM(m)) return;
|
422
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(combined_histo)) return;
|
427
423
|
for (i = 0; i < num_contexts; ++i) {
|
428
424
|
size_t curr_histo_ix = self->curr_histogram_ix_ + i;
|
429
425
|
size_t j;
|
@@ -527,7 +523,7 @@ static void MapStaticContexts(MemoryManager* m,
|
|
527
523
|
mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
|
528
524
|
mb->literal_context_map =
|
529
525
|
BROTLI_ALLOC(m, uint32_t, mb->literal_context_map_size);
|
530
|
-
if (BROTLI_IS_OOM(m)) return;
|
526
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(mb->literal_context_map)) return;
|
531
527
|
|
532
528
|
for (i = 0; i < mb->literal_split.num_types; ++i) {
|
533
529
|
uint32_t offset = (uint32_t)(i * num_contexts);
|