brotli 0.2.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +34 -0
- data/.github/workflows/publish.yml +34 -0
- data/Gemfile +6 -3
- data/Rakefile +16 -9
- data/brotli.gemspec +7 -13
- data/ext/brotli/brotli.c +209 -31
- data/ext/brotli/buffer.c +1 -7
- data/ext/brotli/buffer.h +1 -1
- data/ext/brotli/extconf.rb +20 -18
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +104 -0
- data/test/brotli_writer_test.rb +36 -0
- data/test/test_helper.rb +8 -0
- data/vendor/brotli/c/common/constants.c +15 -0
- data/vendor/brotli/c/common/constants.h +136 -0
- data/vendor/brotli/c/common/context.c +156 -0
- data/vendor/brotli/c/common/context.h +4 -152
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +10 -1
- data/vendor/brotli/c/common/platform.c +22 -0
- data/vendor/brotli/c/common/platform.h +43 -17
- data/vendor/brotli/c/common/transform.c +59 -3
- data/vendor/brotli/c/common/transform.h +5 -0
- data/vendor/brotli/c/common/version.h +2 -2
- data/vendor/brotli/c/dec/bit_reader.c +28 -0
- data/vendor/brotli/c/dec/bit_reader.h +58 -16
- data/vendor/brotli/c/dec/decode.c +353 -251
- data/vendor/brotli/c/dec/huffman.h +6 -12
- data/vendor/brotli/c/dec/prefix.h +0 -18
- data/vendor/brotli/c/dec/state.c +9 -14
- data/vendor/brotli/c/dec/state.h +144 -37
- data/vendor/brotli/c/enc/backward_references.c +8 -7
- data/vendor/brotli/c/enc/backward_references.h +5 -4
- data/vendor/brotli/c/enc/backward_references_hq.c +51 -33
- data/vendor/brotli/c/enc/backward_references_hq.h +11 -8
- data/vendor/brotli/c/enc/backward_references_inc.h +24 -14
- data/vendor/brotli/c/enc/block_splitter.c +3 -3
- data/vendor/brotli/c/enc/block_splitter_inc.h +15 -6
- data/vendor/brotli/c/enc/brotli_bit_stream.c +13 -30
- data/vendor/brotli/c/enc/cluster_inc.h +6 -3
- data/vendor/brotli/c/enc/command.c +28 -0
- data/vendor/brotli/c/enc/command.h +12 -12
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1826 -1100
- data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
- data/vendor/brotli/c/enc/encode.c +104 -39
- data/vendor/brotli/c/enc/encoder_dict.c +3 -2
- data/vendor/brotli/c/enc/encoder_dict.h +3 -1
- data/vendor/brotli/c/enc/entropy_encode.c +2 -0
- data/vendor/brotli/c/enc/entropy_encode.h +2 -2
- data/vendor/brotli/c/enc/fast_log.c +105 -0
- data/vendor/brotli/c/enc/fast_log.h +19 -100
- data/vendor/brotli/c/enc/find_match_length.h +2 -3
- data/vendor/brotli/c/enc/hash.h +80 -90
- data/vendor/brotli/c/enc/hash_composite_inc.h +52 -63
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +88 -49
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +50 -50
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +53 -50
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +91 -60
- data/vendor/brotli/c/enc/hash_rolling_inc.h +23 -27
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +39 -38
- data/vendor/brotli/c/enc/memory.h +24 -12
- data/vendor/brotli/c/enc/metablock.c +23 -27
- data/vendor/brotli/c/enc/metablock_inc.h +1 -1
- data/vendor/brotli/c/enc/params.h +3 -1
- data/vendor/brotli/c/enc/ringbuffer.h +4 -1
- data/vendor/brotli/c/enc/utf8_util.c +1 -1
- data/vendor/brotli/c/enc/write_bits.h +27 -25
- data/vendor/brotli/c/include/brotli/encode.h +22 -1
- data/vendor/brotli/c/include/brotli/port.h +14 -0
- metadata +17 -97
- data/.travis.yml +0 -31
- data/docs/Brotli.html +0 -485
- data/docs/Brotli/Error.html +0 -124
- data/docs/_index.html +0 -122
- data/docs/class_list.html +0 -51
- data/docs/css/common.css +0 -1
- data/docs/css/full_list.css +0 -58
- data/docs/css/style.css +0 -496
- data/docs/file.README.html +0 -127
- data/docs/file_list.html +0 -56
- data/docs/frames.html +0 -17
- data/docs/index.html +0 -127
- data/docs/js/app.js +0 -292
- data/docs/js/full_list.js +0 -216
- data/docs/js/jquery.js +0 -4
- data/docs/method_list.html +0 -67
- data/docs/top-level-namespace.html +0 -110
- data/spec/brotli_spec.rb +0 -88
- data/spec/inflate_spec.rb +0 -75
- data/spec/spec_helper.rb +0 -4
@@ -10,6 +10,7 @@
|
|
10
10
|
#define BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
|
11
11
|
|
12
12
|
#include "../common/constants.h"
|
13
|
+
#include "../common/context.h"
|
13
14
|
#include "../common/dictionary.h"
|
14
15
|
#include "../common/platform.h"
|
15
16
|
#include <brotli/types.h>
|
@@ -23,15 +24,17 @@ extern "C" {
|
|
23
24
|
#endif
|
24
25
|
|
25
26
|
BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(MemoryManager* m,
|
26
|
-
size_t num_bytes,
|
27
|
-
size_t
|
28
|
-
|
27
|
+
size_t num_bytes,
|
28
|
+
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
29
|
+
ContextLut literal_context_lut, const BrotliEncoderParams* params,
|
30
|
+
Hasher* hasher, int* dist_cache, size_t* last_insert_len,
|
29
31
|
Command* commands, size_t* num_commands, size_t* num_literals);
|
30
32
|
|
31
33
|
BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m,
|
32
|
-
size_t num_bytes,
|
33
|
-
size_t
|
34
|
-
|
34
|
+
size_t num_bytes,
|
35
|
+
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
36
|
+
ContextLut literal_context_lut, const BrotliEncoderParams* params,
|
37
|
+
Hasher* hasher, int* dist_cache, size_t* last_insert_len,
|
35
38
|
Command* commands, size_t* num_commands, size_t* num_literals);
|
36
39
|
|
37
40
|
typedef struct ZopfliNode {
|
@@ -77,8 +80,8 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
|
|
77
80
|
BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
|
78
81
|
MemoryManager* m, size_t num_bytes,
|
79
82
|
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
80
|
-
const BrotliEncoderParams* params,
|
81
|
-
const int* dist_cache,
|
83
|
+
ContextLut literal_context_lut, const BrotliEncoderParams* params,
|
84
|
+
const int* dist_cache, Hasher* hasher, ZopfliNode* nodes);
|
82
85
|
|
83
86
|
BROTLI_INTERNAL void BrotliZopfliCreateCommands(
|
84
87
|
const size_t num_bytes, const size_t block_start, const ZopfliNode* nodes,
|
@@ -10,11 +10,13 @@
|
|
10
10
|
static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
|
11
11
|
size_t num_bytes, size_t position,
|
12
12
|
const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
13
|
-
const BrotliEncoderParams* params,
|
14
|
-
|
13
|
+
ContextLut literal_context_lut, const BrotliEncoderParams* params,
|
14
|
+
Hasher* hasher, int* dist_cache, size_t* last_insert_len,
|
15
15
|
Command* commands, size_t* num_commands, size_t* num_literals) {
|
16
|
+
HASHER()* privat = &hasher->privat.FN(_);
|
16
17
|
/* Set maximum distance, see section 9.1. of the spec. */
|
17
18
|
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
|
19
|
+
const size_t position_offset = params->stream_offset;
|
18
20
|
|
19
21
|
const Command* const orig_commands = commands;
|
20
22
|
size_t insert_length = *last_insert_len;
|
@@ -31,19 +33,23 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
|
|
31
33
|
/* Minimum score to accept a backward reference. */
|
32
34
|
const score_t kMinScore = BROTLI_SCORE_BASE + 100;
|
33
35
|
|
34
|
-
|
36
|
+
BROTLI_UNUSED(literal_context_lut);
|
37
|
+
|
38
|
+
FN(PrepareDistanceCache)(privat, dist_cache);
|
35
39
|
|
36
40
|
while (position + FN(HashTypeLength)() < pos_end) {
|
37
41
|
size_t max_length = pos_end - position;
|
38
42
|
size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
|
43
|
+
size_t dictionary_start = BROTLI_MIN(size_t,
|
44
|
+
position + position_offset, max_backward_limit);
|
39
45
|
HasherSearchResult sr;
|
40
46
|
sr.len = 0;
|
41
47
|
sr.len_code_delta = 0;
|
42
48
|
sr.distance = 0;
|
43
49
|
sr.score = kMinScore;
|
44
|
-
FN(FindLongestMatch)(
|
50
|
+
FN(FindLongestMatch)(privat, ¶ms->dictionary,
|
45
51
|
ringbuffer, ringbuffer_mask, dist_cache, position, max_length,
|
46
|
-
max_distance, gap, params->dist.max_distance, &sr);
|
52
|
+
max_distance, dictionary_start + gap, params->dist.max_distance, &sr);
|
47
53
|
if (sr.score > kMinScore) {
|
48
54
|
/* Found a match. Let's look for something even better ahead. */
|
49
55
|
int delayed_backward_references_in_row = 0;
|
@@ -57,10 +63,13 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
|
|
57
63
|
sr2.distance = 0;
|
58
64
|
sr2.score = kMinScore;
|
59
65
|
max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
|
60
|
-
|
66
|
+
dictionary_start = BROTLI_MIN(size_t,
|
67
|
+
position + 1 + position_offset, max_backward_limit);
|
68
|
+
FN(FindLongestMatch)(privat,
|
61
69
|
¶ms->dictionary,
|
62
70
|
ringbuffer, ringbuffer_mask, dist_cache, position + 1, max_length,
|
63
|
-
max_distance, gap, params->dist.max_distance,
|
71
|
+
max_distance, dictionary_start + gap, params->dist.max_distance,
|
72
|
+
&sr2);
|
64
73
|
if (sr2.score >= sr.score + cost_diff_lazy) {
|
65
74
|
/* Ok, let's just write one byte for now and start a match from the
|
66
75
|
next byte. */
|
@@ -76,18 +85,19 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
|
|
76
85
|
}
|
77
86
|
apply_random_heuristics =
|
78
87
|
position + 2 * sr.len + random_heuristics_window_size;
|
79
|
-
|
88
|
+
dictionary_start = BROTLI_MIN(size_t,
|
89
|
+
position + position_offset, max_backward_limit);
|
80
90
|
{
|
81
91
|
/* The first 16 codes are special short-codes,
|
82
92
|
and the minimum offset is 1. */
|
83
93
|
size_t distance_code = ComputeDistanceCode(
|
84
|
-
sr.distance,
|
85
|
-
if ((sr.distance <= (
|
94
|
+
sr.distance, dictionary_start + gap, dist_cache);
|
95
|
+
if ((sr.distance <= (dictionary_start + gap)) && distance_code > 0) {
|
86
96
|
dist_cache[3] = dist_cache[2];
|
87
97
|
dist_cache[2] = dist_cache[1];
|
88
98
|
dist_cache[1] = dist_cache[0];
|
89
99
|
dist_cache[0] = (int)sr.distance;
|
90
|
-
FN(PrepareDistanceCache)(
|
100
|
+
FN(PrepareDistanceCache)(privat, dist_cache);
|
91
101
|
}
|
92
102
|
InitCommand(commands++, ¶ms->dist, insert_length,
|
93
103
|
sr.len, sr.len_code_delta, distance_code);
|
@@ -105,7 +115,7 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
|
|
105
115
|
range_start = BROTLI_MIN(size_t, range_end, BROTLI_MAX(size_t,
|
106
116
|
range_start, position + sr.len - (sr.distance << 2)));
|
107
117
|
}
|
108
|
-
FN(StoreRange)(
|
118
|
+
FN(StoreRange)(privat, ringbuffer, ringbuffer_mask, range_start,
|
109
119
|
range_end);
|
110
120
|
}
|
111
121
|
position += sr.len;
|
@@ -131,7 +141,7 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
|
|
131
141
|
size_t pos_jump =
|
132
142
|
BROTLI_MIN(size_t, position + 16, pos_end - kMargin);
|
133
143
|
for (; position < pos_jump; position += 4) {
|
134
|
-
FN(Store)(
|
144
|
+
FN(Store)(privat, ringbuffer, ringbuffer_mask, position);
|
135
145
|
insert_length += 4;
|
136
146
|
}
|
137
147
|
} else {
|
@@ -140,7 +150,7 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
|
|
140
150
|
size_t pos_jump =
|
141
151
|
BROTLI_MIN(size_t, position + 8, pos_end - kMargin);
|
142
152
|
for (; position < pos_jump; position += 2) {
|
143
|
-
FN(Store)(
|
153
|
+
FN(Store)(privat, ringbuffer, ringbuffer_mask, position);
|
144
154
|
insert_length += 2;
|
145
155
|
}
|
146
156
|
}
|
@@ -132,7 +132,7 @@ void BrotliSplitBlock(MemoryManager* m,
|
|
132
132
|
{
|
133
133
|
size_t literals_count = CountLiterals(cmds, num_commands);
|
134
134
|
uint8_t* literals = BROTLI_ALLOC(m, uint8_t, literals_count);
|
135
|
-
if (BROTLI_IS_OOM(m)) return;
|
135
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(literals)) return;
|
136
136
|
/* Create a continuous array of literals. */
|
137
137
|
CopyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals);
|
138
138
|
/* Create the block split on the array of literals.
|
@@ -150,7 +150,7 @@ void BrotliSplitBlock(MemoryManager* m,
|
|
150
150
|
/* Compute prefix codes for commands. */
|
151
151
|
uint16_t* insert_and_copy_codes = BROTLI_ALLOC(m, uint16_t, num_commands);
|
152
152
|
size_t i;
|
153
|
-
if (BROTLI_IS_OOM(m)) return;
|
153
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(insert_and_copy_codes)) return;
|
154
154
|
for (i = 0; i < num_commands; ++i) {
|
155
155
|
insert_and_copy_codes[i] = cmds[i].cmd_prefix_;
|
156
156
|
}
|
@@ -170,7 +170,7 @@ void BrotliSplitBlock(MemoryManager* m,
|
|
170
170
|
uint16_t* distance_prefixes = BROTLI_ALLOC(m, uint16_t, num_commands);
|
171
171
|
size_t j = 0;
|
172
172
|
size_t i;
|
173
|
-
if (BROTLI_IS_OOM(m)) return;
|
173
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(distance_prefixes)) return;
|
174
174
|
for (i = 0; i < num_commands; ++i) {
|
175
175
|
const Command* cmd = &cmds[i];
|
176
176
|
if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
|
@@ -219,7 +219,12 @@ static void FN(ClusterBlocks)(MemoryManager* m,
|
|
219
219
|
uint32_t symbols[HISTOGRAMS_PER_BATCH] = { 0 };
|
220
220
|
uint32_t remap[HISTOGRAMS_PER_BATCH] = { 0 };
|
221
221
|
|
222
|
-
if (BROTLI_IS_OOM(m))
|
222
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(histogram_symbols) ||
|
223
|
+
BROTLI_IS_NULL(block_lengths) || BROTLI_IS_NULL(all_histograms) ||
|
224
|
+
BROTLI_IS_NULL(cluster_size) || BROTLI_IS_NULL(histograms) ||
|
225
|
+
BROTLI_IS_NULL(pairs)) {
|
226
|
+
return;
|
227
|
+
}
|
223
228
|
|
224
229
|
memset(block_lengths, 0, num_blocks * sizeof(uint32_t));
|
225
230
|
|
@@ -278,11 +283,11 @@ static void FN(ClusterBlocks)(MemoryManager* m,
|
|
278
283
|
if (pairs_capacity < max_num_pairs + 1) {
|
279
284
|
BROTLI_FREE(m, pairs);
|
280
285
|
pairs = BROTLI_ALLOC(m, HistogramPair, max_num_pairs + 1);
|
281
|
-
if (BROTLI_IS_OOM(m)) return;
|
286
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(pairs)) return;
|
282
287
|
}
|
283
288
|
|
284
289
|
clusters = BROTLI_ALLOC(m, uint32_t, num_clusters);
|
285
|
-
if (BROTLI_IS_OOM(m)) return;
|
290
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(clusters)) return;
|
286
291
|
for (i = 0; i < num_clusters; ++i) {
|
287
292
|
clusters[i] = (uint32_t)i;
|
288
293
|
}
|
@@ -294,7 +299,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
|
|
294
299
|
BROTLI_FREE(m, cluster_size);
|
295
300
|
|
296
301
|
new_index = BROTLI_ALLOC(m, uint32_t, num_clusters);
|
297
|
-
if (BROTLI_IS_OOM(m)) return;
|
302
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_index)) return;
|
298
303
|
for (i = 0; i < num_clusters; ++i) new_index[i] = kInvalidIndex;
|
299
304
|
pos = 0;
|
300
305
|
{
|
@@ -386,7 +391,7 @@ static void FN(SplitByteVector)(MemoryManager* m,
|
|
386
391
|
return;
|
387
392
|
}
|
388
393
|
histograms = BROTLI_ALLOC(m, HistogramType, num_histograms);
|
389
|
-
if (BROTLI_IS_OOM(m)) return;
|
394
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(histograms)) return;
|
390
395
|
/* Find good entropy codes. */
|
391
396
|
FN(InitialEntropyCodes)(data, length,
|
392
397
|
sampling_stride_length,
|
@@ -405,7 +410,11 @@ static void FN(SplitByteVector)(MemoryManager* m,
|
|
405
410
|
uint16_t* new_id = BROTLI_ALLOC(m, uint16_t, num_histograms);
|
406
411
|
const size_t iters = params->quality < HQ_ZOPFLIFICATION_QUALITY ? 3 : 10;
|
407
412
|
size_t i;
|
408
|
-
if (BROTLI_IS_OOM(m))
|
413
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(block_ids) ||
|
414
|
+
BROTLI_IS_NULL(insert_cost) || BROTLI_IS_NULL(cost) ||
|
415
|
+
BROTLI_IS_NULL(switch_signal) || BROTLI_IS_NULL(new_id)) {
|
416
|
+
return;
|
417
|
+
}
|
409
418
|
for (i = 0; i < iters; ++i) {
|
410
419
|
num_blocks = FN(FindBlocks)(data, length,
|
411
420
|
block_switch_cost,
|
@@ -34,33 +34,18 @@ extern "C" {
|
|
34
34
|
BROTLI_DISTANCE_ALPHABET_SIZE(0, 0, BROTLI_LARGE_MAX_DISTANCE_BITS)
|
35
35
|
/* MAX_SIMPLE_DISTANCE_ALPHABET_SIZE == 140 */
|
36
36
|
|
37
|
-
/* Represents the range of values belonging to a prefix code:
|
38
|
-
[offset, offset + 2^nbits) */
|
39
|
-
typedef struct PrefixCodeRange {
|
40
|
-
uint32_t offset;
|
41
|
-
uint32_t nbits;
|
42
|
-
} PrefixCodeRange;
|
43
|
-
|
44
|
-
static const PrefixCodeRange
|
45
|
-
kBlockLengthPrefixCode[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = {
|
46
|
-
{ 1, 2}, { 5, 2}, { 9, 2}, {13, 2}, {17, 3}, { 25, 3}, { 33, 3},
|
47
|
-
{41, 3}, {49, 4}, {65, 4}, {81, 4}, {97, 4}, {113, 5}, {145, 5},
|
48
|
-
{177, 5}, { 209, 5}, { 241, 6}, { 305, 6}, { 369, 7}, { 497, 8},
|
49
|
-
{753, 9}, {1265, 10}, {2289, 11}, {4337, 12}, {8433, 13}, {16625, 24}
|
50
|
-
};
|
51
|
-
|
52
37
|
static BROTLI_INLINE uint32_t BlockLengthPrefixCode(uint32_t len) {
|
53
38
|
uint32_t code = (len >= 177) ? (len >= 753 ? 20 : 14) : (len >= 41 ? 7 : 0);
|
54
39
|
while (code < (BROTLI_NUM_BLOCK_LEN_SYMBOLS - 1) &&
|
55
|
-
len >=
|
40
|
+
len >= _kBrotliPrefixCodeRanges[code + 1].offset) ++code;
|
56
41
|
return code;
|
57
42
|
}
|
58
43
|
|
59
44
|
static BROTLI_INLINE void GetBlockLengthPrefixCode(uint32_t len, size_t* code,
|
60
45
|
uint32_t* n_extra, uint32_t* extra) {
|
61
46
|
*code = BlockLengthPrefixCode(len);
|
62
|
-
*n_extra =
|
63
|
-
*extra = len -
|
47
|
+
*n_extra = _kBrotliPrefixCodeRanges[*code].nbits;
|
48
|
+
*extra = len - _kBrotliPrefixCodeRanges[*code].offset;
|
64
49
|
}
|
65
50
|
|
66
51
|
typedef struct BlockTypeCodeCalculator {
|
@@ -450,7 +435,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
|
|
450
435
|
const size_t max_tree_size = 2 * length + 1;
|
451
436
|
HuffmanTree* tree = BROTLI_ALLOC(m, HuffmanTree, max_tree_size);
|
452
437
|
uint32_t count_limit;
|
453
|
-
if (BROTLI_IS_OOM(m)) return;
|
438
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
|
454
439
|
for (count_limit = 1; ; count_limit *= 2) {
|
455
440
|
HuffmanTree* node = tree;
|
456
441
|
size_t l;
|
@@ -714,7 +699,7 @@ static void EncodeContextMap(MemoryManager* m,
|
|
714
699
|
}
|
715
700
|
|
716
701
|
rle_symbols = BROTLI_ALLOC(m, uint32_t, context_map_size);
|
717
|
-
if (BROTLI_IS_OOM(m)) return;
|
702
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(rle_symbols)) return;
|
718
703
|
MoveToFrontTransform(context_map, context_map_size, rle_symbols);
|
719
704
|
RunLengthCodeZeros(context_map_size, rle_symbols,
|
720
705
|
&num_rle_symbols, &max_run_length_prefix);
|
@@ -956,23 +941,21 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
956
941
|
|
957
942
|
size_t pos = start_pos;
|
958
943
|
size_t i;
|
959
|
-
uint32_t num_distance_symbols = params->dist.
|
960
|
-
uint32_t num_effective_distance_symbols =
|
944
|
+
uint32_t num_distance_symbols = params->dist.alphabet_size_max;
|
945
|
+
uint32_t num_effective_distance_symbols = params->dist.alphabet_size_limit;
|
961
946
|
HuffmanTree* tree;
|
962
947
|
ContextLut literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
|
963
948
|
BlockEncoder literal_enc;
|
964
949
|
BlockEncoder command_enc;
|
965
950
|
BlockEncoder distance_enc;
|
966
951
|
const BrotliDistanceParams* dist = ¶ms->dist;
|
967
|
-
|
968
|
-
num_effective_distance_symbols
|
969
|
-
num_effective_distance_symbols = BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS;
|
970
|
-
}
|
952
|
+
BROTLI_DCHECK(
|
953
|
+
num_effective_distance_symbols <= BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS);
|
971
954
|
|
972
955
|
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
|
973
956
|
|
974
957
|
tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
|
975
|
-
if (BROTLI_IS_OOM(m)) return;
|
958
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
|
976
959
|
InitBlockEncoder(&literal_enc, BROTLI_NUM_LITERAL_SYMBOLS,
|
977
960
|
mb->literal_split.num_types, mb->literal_split.types,
|
978
961
|
mb->literal_split.lengths, mb->literal_split.num_blocks);
|
@@ -1163,7 +1146,7 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
|
1163
1146
|
uint8_t dist_depth[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1164
1147
|
uint16_t dist_bits[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1165
1148
|
HuffmanTree* tree;
|
1166
|
-
uint32_t num_distance_symbols = params->dist.
|
1149
|
+
uint32_t num_distance_symbols = params->dist.alphabet_size_max;
|
1167
1150
|
|
1168
1151
|
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
|
1169
1152
|
|
@@ -1177,7 +1160,7 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
|
1177
1160
|
BrotliWriteBits(13, 0, storage_ix, storage);
|
1178
1161
|
|
1179
1162
|
tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
|
1180
|
-
if (BROTLI_IS_OOM(m)) return;
|
1163
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
|
1181
1164
|
BuildAndStoreHuffmanTree(lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS,
|
1182
1165
|
BROTLI_NUM_LITERAL_SYMBOLS, tree,
|
1183
1166
|
lit_depth, lit_bits,
|
@@ -1206,7 +1189,7 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
|
|
1206
1189
|
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
|
1207
1190
|
const Command* commands, size_t n_commands,
|
1208
1191
|
size_t* storage_ix, uint8_t* storage) {
|
1209
|
-
uint32_t num_distance_symbols = params->dist.
|
1192
|
+
uint32_t num_distance_symbols = params->dist.alphabet_size_max;
|
1210
1193
|
uint32_t distance_alphabet_bits =
|
1211
1194
|
Log2FloorNonZero(num_distance_symbols - 1) + 1;
|
1212
1195
|
|
@@ -215,7 +215,7 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramReindex)(MemoryManager* m,
|
|
215
215
|
uint32_t next_index;
|
216
216
|
HistogramType* tmp;
|
217
217
|
size_t i;
|
218
|
-
if (BROTLI_IS_OOM(m)) return 0;
|
218
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_index)) return 0;
|
219
219
|
for (i = 0; i < length; ++i) {
|
220
220
|
new_index[i] = kInvalidIndex;
|
221
221
|
}
|
@@ -229,7 +229,7 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramReindex)(MemoryManager* m,
|
|
229
229
|
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
|
230
230
|
tmp and reduce the number of copying by the factor of 2. */
|
231
231
|
tmp = BROTLI_ALLOC(m, HistogramType, next_index);
|
232
|
-
if (BROTLI_IS_OOM(m)) return 0;
|
232
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tmp)) return 0;
|
233
233
|
next_index = 0;
|
234
234
|
for (i = 0; i < length; ++i) {
|
235
235
|
if (new_index[symbols[i]] == next_index) {
|
@@ -259,7 +259,10 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
|
|
259
259
|
HistogramPair* pairs = BROTLI_ALLOC(m, HistogramPair, pairs_capacity + 1);
|
260
260
|
size_t i;
|
261
261
|
|
262
|
-
if (BROTLI_IS_OOM(m))
|
262
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(cluster_size) ||
|
263
|
+
BROTLI_IS_NULL(clusters) || BROTLI_IS_NULL(pairs)) {
|
264
|
+
return;
|
265
|
+
}
|
263
266
|
|
264
267
|
for (i = 0; i < in_size; ++i) {
|
265
268
|
cluster_size[i] = 1;
|
@@ -0,0 +1,28 @@
|
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
#include "./command.h"
|
8
|
+
|
9
|
+
#include <brotli/types.h>
|
10
|
+
|
11
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
12
|
+
extern "C" {
|
13
|
+
#endif
|
14
|
+
|
15
|
+
const uint32_t kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES] = {
|
16
|
+
0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26,
|
17
|
+
34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210, 22594};
|
18
|
+
const uint32_t kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES] = {
|
19
|
+
0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24};
|
20
|
+
const uint32_t kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES] = {
|
21
|
+
2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18,
|
22
|
+
22, 30, 38, 54, 70, 102, 134, 198, 326, 582, 1094, 2118};
|
23
|
+
const uint32_t kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES] = {
|
24
|
+
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24};
|
25
|
+
|
26
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
27
|
+
} /* extern "C" */
|
28
|
+
#endif
|
@@ -20,14 +20,14 @@
|
|
20
20
|
extern "C" {
|
21
21
|
#endif
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
23
|
+
BROTLI_INTERNAL extern const uint32_t
|
24
|
+
kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES];
|
25
|
+
BROTLI_INTERNAL extern const uint32_t
|
26
|
+
kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES];
|
27
|
+
BROTLI_INTERNAL extern const uint32_t
|
28
|
+
kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES];
|
29
|
+
BROTLI_INTERNAL extern const uint32_t
|
30
|
+
kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES];
|
31
31
|
|
32
32
|
static BROTLI_INLINE uint16_t GetInsertLengthCode(size_t insertlen) {
|
33
33
|
if (insertlen < 6) {
|
@@ -89,19 +89,19 @@ static BROTLI_INLINE void GetLengthCode(size_t insertlen, size_t copylen,
|
|
89
89
|
}
|
90
90
|
|
91
91
|
static BROTLI_INLINE uint32_t GetInsertBase(uint16_t inscode) {
|
92
|
-
return
|
92
|
+
return kBrotliInsBase[inscode];
|
93
93
|
}
|
94
94
|
|
95
95
|
static BROTLI_INLINE uint32_t GetInsertExtra(uint16_t inscode) {
|
96
|
-
return
|
96
|
+
return kBrotliInsExtra[inscode];
|
97
97
|
}
|
98
98
|
|
99
99
|
static BROTLI_INLINE uint32_t GetCopyBase(uint16_t copycode) {
|
100
|
-
return
|
100
|
+
return kBrotliCopyBase[copycode];
|
101
101
|
}
|
102
102
|
|
103
103
|
static BROTLI_INLINE uint32_t GetCopyExtra(uint16_t copycode) {
|
104
|
-
return
|
104
|
+
return kBrotliCopyExtra[copycode];
|
105
105
|
}
|
106
106
|
|
107
107
|
typedef struct Command {
|
@@ -524,7 +524,7 @@ static void StoreCommands(MemoryManager* m,
|
|
524
524
|
static BROTLI_BOOL ShouldCompress(
|
525
525
|
const uint8_t* input, size_t input_size, size_t num_literals) {
|
526
526
|
double corpus_size = (double)input_size;
|
527
|
-
if (num_literals < MIN_RATIO * corpus_size) {
|
527
|
+
if ((double)num_literals < MIN_RATIO * corpus_size) {
|
528
528
|
return BROTLI_TRUE;
|
529
529
|
} else {
|
530
530
|
uint32_t literal_histo[256] = { 0 };
|