brotli 0.1.8 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +7 -3
- data/brotli.gemspec +1 -1
- data/ext/brotli/brotli.c +4 -4
- data/ext/brotli/brotli.h +2 -2
- data/ext/brotli/extconf.rb +9 -16
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/{common → c/common}/constants.h +11 -1
- data/vendor/brotli/c/common/dictionary.bin +432 -0
- data/vendor/brotli/c/common/dictionary.c +5905 -0
- data/vendor/brotli/c/common/dictionary.h +64 -0
- data/vendor/brotli/c/common/version.h +19 -0
- data/vendor/brotli/{dec → c/dec}/bit_reader.c +2 -2
- data/vendor/brotli/{dec → c/dec}/bit_reader.h +11 -34
- data/vendor/brotli/{dec → c/dec}/context.h +1 -1
- data/vendor/brotli/{dec → c/dec}/decode.c +389 -356
- data/vendor/brotli/{dec → c/dec}/huffman.c +24 -23
- data/vendor/brotli/{dec → c/dec}/huffman.h +1 -1
- data/vendor/brotli/{dec → c/dec}/port.h +19 -10
- data/vendor/brotli/{dec → c/dec}/prefix.h +1 -1
- data/vendor/brotli/{dec → c/dec}/state.c +23 -19
- data/vendor/brotli/{dec → c/dec}/state.h +18 -17
- data/vendor/brotli/{dec → c/dec}/transform.h +2 -2
- data/vendor/brotli/c/enc/backward_references.c +134 -0
- data/vendor/brotli/c/enc/backward_references.h +39 -0
- data/vendor/brotli/{enc/backward_references.c → c/enc/backward_references_hq.c} +144 -232
- data/vendor/brotli/{enc/backward_references.h → c/enc/backward_references_hq.h} +28 -31
- data/vendor/brotli/{enc → c/enc}/backward_references_inc.h +37 -31
- data/vendor/brotli/{enc → c/enc}/bit_cost.c +1 -1
- data/vendor/brotli/{enc → c/enc}/bit_cost.h +1 -1
- data/vendor/brotli/{enc → c/enc}/bit_cost_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/block_encoder_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/block_splitter.c +2 -4
- data/vendor/brotli/{enc → c/enc}/block_splitter.h +1 -1
- data/vendor/brotli/{enc → c/enc}/block_splitter_inc.h +6 -7
- data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.c +22 -26
- data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.h +1 -5
- data/vendor/brotli/{enc → c/enc}/cluster.c +1 -1
- data/vendor/brotli/{enc → c/enc}/cluster.h +1 -1
- data/vendor/brotli/{enc → c/enc}/cluster_inc.h +2 -0
- data/vendor/brotli/{enc → c/enc}/command.h +34 -17
- data/vendor/brotli/{enc → c/enc}/compress_fragment.c +97 -53
- data/vendor/brotli/{enc → c/enc}/compress_fragment.h +5 -2
- data/vendor/brotli/{enc → c/enc}/compress_fragment_two_pass.c +106 -51
- data/vendor/brotli/{enc → c/enc}/compress_fragment_two_pass.h +5 -2
- data/vendor/brotli/{enc → c/enc}/context.h +3 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +1120 -0
- data/vendor/brotli/c/enc/dictionary_hash.h +24 -0
- data/vendor/brotli/{enc → c/enc}/encode.c +442 -240
- data/vendor/brotli/{enc → c/enc}/entropy_encode.c +9 -9
- data/vendor/brotli/{enc → c/enc}/entropy_encode.h +4 -4
- data/vendor/brotli/{enc → c/enc}/entropy_encode_static.h +4 -4
- data/vendor/brotli/{enc → c/enc}/fast_log.h +3 -3
- data/vendor/brotli/{enc → c/enc}/find_match_length.h +8 -8
- data/vendor/brotli/c/enc/hash.h +446 -0
- data/vendor/brotli/{enc → c/enc}/hash_forgetful_chain_inc.h +72 -68
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +266 -0
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +258 -0
- data/vendor/brotli/{enc → c/enc}/hash_longest_match_quickly_inc.h +81 -77
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +326 -0
- data/vendor/brotli/{enc → c/enc}/histogram.c +4 -2
- data/vendor/brotli/{enc → c/enc}/histogram.h +1 -1
- data/vendor/brotli/{enc → c/enc}/histogram_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/literal_cost.c +4 -7
- data/vendor/brotli/{enc → c/enc}/literal_cost.h +2 -2
- data/vendor/brotli/{enc → c/enc}/memory.c +1 -1
- data/vendor/brotli/{enc → c/enc}/memory.h +3 -2
- data/vendor/brotli/{enc → c/enc}/metablock.c +136 -123
- data/vendor/brotli/{enc → c/enc}/metablock.h +2 -12
- data/vendor/brotli/{enc → c/enc}/metablock_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/port.h +49 -33
- data/vendor/brotli/{enc → c/enc}/prefix.h +4 -2
- data/vendor/brotli/{enc → c/enc}/quality.h +47 -17
- data/vendor/brotli/{enc → c/enc}/ringbuffer.h +6 -6
- data/vendor/brotli/{enc → c/enc}/static_dict.c +26 -22
- data/vendor/brotli/{enc → c/enc}/static_dict.h +3 -1
- data/vendor/brotli/c/enc/static_dict_lut.h +5864 -0
- data/vendor/brotli/{enc → c/enc}/utf8_util.c +1 -1
- data/vendor/brotli/{enc → c/enc}/utf8_util.h +2 -2
- data/vendor/brotli/{enc → c/enc}/write_bits.h +3 -3
- data/vendor/brotli/c/include/brotli/decode.h +339 -0
- data/vendor/brotli/c/include/brotli/encode.h +402 -0
- data/vendor/brotli/c/include/brotli/port.h +146 -0
- data/vendor/brotli/c/include/brotli/types.h +90 -0
- metadata +80 -79
- data/vendor/brotli/common/dictionary.c +0 -9474
- data/vendor/brotli/common/dictionary.h +0 -29
- data/vendor/brotli/common/port.h +0 -107
- data/vendor/brotli/common/types.h +0 -58
- data/vendor/brotli/dec/decode.h +0 -188
- data/vendor/brotli/enc/compressor.cc +0 -139
- data/vendor/brotli/enc/compressor.h +0 -161
- data/vendor/brotli/enc/dictionary_hash.h +0 -4121
- data/vendor/brotli/enc/encode.h +0 -221
- data/vendor/brotli/enc/encode_parallel.cc +0 -289
- data/vendor/brotli/enc/encode_parallel.h +0 -27
- data/vendor/brotli/enc/hash.h +0 -717
- data/vendor/brotli/enc/hash_longest_match_inc.h +0 -241
- data/vendor/brotli/enc/static_dict_lut.h +0 -11241
- data/vendor/brotli/enc/streams.cc +0 -114
- data/vendor/brotli/enc/streams.h +0 -121
@@ -6,10 +6,12 @@
|
|
6
6
|
|
7
7
|
/* Function to find backward reference copies. */
|
8
8
|
|
9
|
-
#ifndef
|
10
|
-
#define
|
9
|
+
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
|
10
|
+
#define BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
|
11
11
|
|
12
|
-
#include "../common/
|
12
|
+
#include "../common/constants.h"
|
13
|
+
#include "../common/dictionary.h"
|
14
|
+
#include <brotli/types.h>
|
13
15
|
#include "./command.h"
|
14
16
|
#include "./hash.h"
|
15
17
|
#include "./memory.h"
|
@@ -20,14 +22,17 @@
|
|
20
22
|
extern "C" {
|
21
23
|
#endif
|
22
24
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
25
|
+
BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(
|
26
|
+
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
|
27
|
+
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
28
|
+
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
|
29
|
+
size_t* last_insert_len, Command* commands, size_t* num_commands,
|
30
|
+
size_t* num_literals);
|
31
|
+
|
32
|
+
BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(
|
33
|
+
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
|
34
|
+
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
35
|
+
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
|
31
36
|
size_t* last_insert_len, Command* commands, size_t* num_commands,
|
32
37
|
size_t* num_literals);
|
33
38
|
|
@@ -44,7 +49,7 @@ typedef struct ZopfliNode {
|
|
44
49
|
|
45
50
|
/* This union holds information used by dynamic-programming. During forward
|
46
51
|
pass |cost| it used to store the goal function. When node is processed its
|
47
|
-
|cost| is invalidated in favor of |shortcut|. On path
|
52
|
+
|cost| is invalidated in favor of |shortcut|. On path back-tracing pass
|
48
53
|
|next| is assigned the offset to next node on the path. */
|
49
54
|
union {
|
50
55
|
/* Smallest cost to get to this byte from the beginning, as found so far. */
|
@@ -63,7 +68,7 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
|
|
63
68
|
position + num_bytes.
|
64
69
|
|
65
70
|
On return, path->size() is the number of commands found and path[i] is the
|
66
|
-
length of the
|
71
|
+
length of the i-th command (copy length plus insert length).
|
67
72
|
Note that the sum of the lengths of all commands can be less than num_bytes.
|
68
73
|
|
69
74
|
On return, the nodes[0..num_bytes] array will have the following
|
@@ -73,27 +78,19 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
|
|
73
78
|
(2) nodes[i].command_length() <= i and
|
74
79
|
(3) nodes[i - nodes[i].command_length()].cost < kInfinity */
|
75
80
|
BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
|
76
|
-
MemoryManager* m,
|
77
|
-
const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
81
|
+
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
|
82
|
+
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
78
83
|
const BrotliEncoderParams* params, const size_t max_backward_limit,
|
79
|
-
const int* dist_cache,
|
80
|
-
|
81
|
-
BROTLI_INTERNAL void BrotliZopfliCreateCommands(
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
size_t* last_insert_len,
|
87
|
-
Command* commands,
|
88
|
-
size_t* num_literals);
|
89
|
-
|
90
|
-
/* Maximum distance, see section 9.1. of the spec. */
|
91
|
-
static BROTLI_INLINE size_t MaxBackwardLimit(int lgwin) {
|
92
|
-
return (1u << lgwin) - 16;
|
93
|
-
}
|
84
|
+
const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes);
|
85
|
+
|
86
|
+
BROTLI_INTERNAL void BrotliZopfliCreateCommands(
|
87
|
+
const size_t num_bytes, const size_t block_start,
|
88
|
+
const size_t max_backward_limit, const ZopfliNode* nodes,
|
89
|
+
int* dist_cache, size_t* last_insert_len, const BrotliEncoderParams* params,
|
90
|
+
Command* commands, size_t* num_literals);
|
94
91
|
|
95
92
|
#if defined(__cplusplus) || defined(c_plusplus)
|
96
93
|
} /* extern "C" */
|
97
94
|
#endif
|
98
95
|
|
99
|
-
#endif /*
|
96
|
+
#endif /* BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_ */
|
@@ -5,18 +5,17 @@
|
|
5
5
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
6
6
|
*/
|
7
7
|
|
8
|
-
/* template parameters: FN */
|
8
|
+
/* template parameters: EXPORT_FN, FN */
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
MemoryManager* m, size_t num_bytes, size_t position, BROTLI_BOOL is_last,
|
10
|
+
static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
|
11
|
+
const BrotliDictionary* dictionary,
|
12
|
+
const uint16_t* dictionary_hash, size_t num_bytes, size_t position,
|
14
13
|
const uint8_t* ringbuffer, size_t ringbuffer_mask,
|
15
|
-
const BrotliEncoderParams* params,
|
14
|
+
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
|
16
15
|
size_t* last_insert_len, Command* commands, size_t* num_commands,
|
17
16
|
size_t* num_literals) {
|
18
17
|
/* Set maximum distance, see section 9.1. of the spec. */
|
19
|
-
const size_t max_backward_limit =
|
18
|
+
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
|
20
19
|
|
21
20
|
const Command* const orig_commands = commands;
|
22
21
|
size_t insert_length = *last_insert_len;
|
@@ -28,42 +27,41 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
|
|
28
27
|
const size_t random_heuristics_window_size =
|
29
28
|
LiteralSpreeLengthForSparseSearch(params);
|
30
29
|
size_t apply_random_heuristics = position + random_heuristics_window_size;
|
30
|
+
const size_t gap = 0;
|
31
31
|
|
32
32
|
/* Minimum score to accept a backward reference. */
|
33
|
-
const score_t kMinScore = BROTLI_SCORE_BASE +
|
33
|
+
const score_t kMinScore = BROTLI_SCORE_BASE + 100;
|
34
34
|
|
35
|
-
FN(
|
36
|
-
if (BROTLI_IS_OOM(m)) return;
|
37
|
-
FN(StitchToPreviousBlock)(hasher, num_bytes, position,
|
38
|
-
ringbuffer, ringbuffer_mask);
|
35
|
+
FN(PrepareDistanceCache)(hasher, dist_cache);
|
39
36
|
|
40
37
|
while (position + FN(HashTypeLength)() < pos_end) {
|
41
38
|
size_t max_length = pos_end - position;
|
42
39
|
size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
|
43
40
|
HasherSearchResult sr;
|
44
41
|
sr.len = 0;
|
45
|
-
sr.
|
42
|
+
sr.len_code_delta = 0;
|
46
43
|
sr.distance = 0;
|
47
44
|
sr.score = kMinScore;
|
48
|
-
|
49
|
-
|
45
|
+
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
|
46
|
+
ringbuffer_mask, dist_cache, position,
|
47
|
+
max_length, max_distance, gap, &sr);
|
48
|
+
if (sr.score > kMinScore) {
|
50
49
|
/* Found a match. Let's look for something even better ahead. */
|
51
50
|
int delayed_backward_references_in_row = 0;
|
52
51
|
--max_length;
|
53
52
|
for (;; --max_length) {
|
54
|
-
const score_t cost_diff_lazy =
|
55
|
-
BROTLI_BOOL is_match_found;
|
53
|
+
const score_t cost_diff_lazy = 175;
|
56
54
|
HasherSearchResult sr2;
|
57
55
|
sr2.len = params->quality < MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH ?
|
58
56
|
BROTLI_MIN(size_t, sr.len - 1, max_length) : 0;
|
59
|
-
sr2.
|
57
|
+
sr2.len_code_delta = 0;
|
60
58
|
sr2.distance = 0;
|
61
59
|
sr2.score = kMinScore;
|
62
60
|
max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
if (
|
61
|
+
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
|
62
|
+
ringbuffer_mask, dist_cache, position + 1,
|
63
|
+
max_length, max_distance, gap, &sr2);
|
64
|
+
if (sr2.score >= sr.score + cost_diff_lazy) {
|
67
65
|
/* Ok, let's just write one byte for now and start a match from the
|
68
66
|
next byte. */
|
69
67
|
++position;
|
@@ -80,26 +78,36 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
|
|
80
78
|
position + 2 * sr.len + random_heuristics_window_size;
|
81
79
|
max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
|
82
80
|
{
|
83
|
-
/* The first 16 codes are special
|
81
|
+
/* The first 16 codes are special short-codes,
|
84
82
|
and the minimum offset is 1. */
|
85
83
|
size_t distance_code =
|
86
|
-
ComputeDistanceCode(sr.distance, max_distance, dist_cache);
|
87
|
-
if (sr.distance <= max_distance && distance_code > 0) {
|
84
|
+
ComputeDistanceCode(sr.distance, max_distance + gap, dist_cache);
|
85
|
+
if ((sr.distance <= (max_distance + gap)) && distance_code > 0) {
|
88
86
|
dist_cache[3] = dist_cache[2];
|
89
87
|
dist_cache[2] = dist_cache[1];
|
90
88
|
dist_cache[1] = dist_cache[0];
|
91
89
|
dist_cache[0] = (int)sr.distance;
|
90
|
+
FN(PrepareDistanceCache)(hasher, dist_cache);
|
92
91
|
}
|
93
|
-
InitCommand(commands++, insert_length, sr.len, sr.
|
92
|
+
InitCommand(commands++, insert_length, sr.len, sr.len_code_delta,
|
94
93
|
distance_code);
|
95
94
|
}
|
96
95
|
*num_literals += insert_length;
|
97
96
|
insert_length = 0;
|
98
97
|
/* Put the hash keys into the table, if there are enough bytes left.
|
99
98
|
Depending on the hasher implementation, it can push all positions
|
100
|
-
in the given range or only a subset of them.
|
101
|
-
|
102
|
-
|
99
|
+
in the given range or only a subset of them.
|
100
|
+
Avoid hash poisoning with RLE data. */
|
101
|
+
{
|
102
|
+
size_t range_start = position + 2;
|
103
|
+
size_t range_end = BROTLI_MIN(size_t, position + sr.len, store_end);
|
104
|
+
if (sr.distance < (sr.len >> 2)) {
|
105
|
+
range_start = BROTLI_MIN(size_t, range_end, BROTLI_MAX(size_t,
|
106
|
+
range_start, position + sr.len - (sr.distance << 2)));
|
107
|
+
}
|
108
|
+
FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, range_start,
|
109
|
+
range_end);
|
110
|
+
}
|
103
111
|
position += sr.len;
|
104
112
|
} else {
|
105
113
|
++insert_length;
|
@@ -143,5 +151,3 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
|
|
143
151
|
*last_insert_len = insert_length;
|
144
152
|
*num_commands += (size_t)(commands - orig_commands);
|
145
153
|
}
|
146
|
-
|
147
|
-
#undef Hasher
|
File without changes
|
File without changes
|
@@ -74,11 +74,9 @@ static void CopyLiteralsToByteArray(const Command* cmds,
|
|
74
74
|
}
|
75
75
|
}
|
76
76
|
|
77
|
-
static BROTLI_INLINE
|
77
|
+
static BROTLI_INLINE uint32_t MyRand(uint32_t* seed) {
|
78
|
+
/* Initial seed should be 7. In this case, loop length is (1 << 29). */
|
78
79
|
*seed *= 16807U;
|
79
|
-
if (*seed == 0) {
|
80
|
-
*seed = 1;
|
81
|
-
}
|
82
80
|
return *seed;
|
83
81
|
}
|
84
82
|
|
@@ -13,7 +13,7 @@ static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
|
|
13
13
|
size_t stride,
|
14
14
|
size_t num_histograms,
|
15
15
|
HistogramType* histograms) {
|
16
|
-
|
16
|
+
uint32_t seed = 7;
|
17
17
|
size_t block_length = length / num_histograms;
|
18
18
|
size_t i;
|
19
19
|
FN(ClearHistograms)(histograms, num_histograms);
|
@@ -29,14 +29,13 @@ static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
|
|
29
29
|
}
|
30
30
|
}
|
31
31
|
|
32
|
-
static void FN(RandomSample)(
|
32
|
+
static void FN(RandomSample)(uint32_t* seed,
|
33
33
|
const DataType* data,
|
34
34
|
size_t length,
|
35
35
|
size_t stride,
|
36
36
|
HistogramType* sample) {
|
37
37
|
size_t pos = 0;
|
38
38
|
if (stride >= length) {
|
39
|
-
pos = 0;
|
40
39
|
stride = length;
|
41
40
|
} else {
|
42
41
|
pos = MyRand(seed) % (length - stride + 1);
|
@@ -50,7 +49,7 @@ static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
|
|
50
49
|
HistogramType* histograms) {
|
51
50
|
size_t iters =
|
52
51
|
kIterMulForRefining * length / stride + kMinItersForRefining;
|
53
|
-
|
52
|
+
uint32_t seed = 7;
|
54
53
|
size_t iter;
|
55
54
|
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms;
|
56
55
|
for (iter = 0; iter < iters; ++iter) {
|
@@ -61,7 +60,7 @@ static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
|
|
61
60
|
}
|
62
61
|
}
|
63
62
|
|
64
|
-
/* Assigns a block id from the range [0,
|
63
|
+
/* Assigns a block id from the range [0, num_histograms) to each data element
|
65
64
|
in data[0..length) and fills in block_id[0..length) with the assigned values.
|
66
65
|
Returns the number of blocks, i.e. one plus the number of block switches. */
|
67
66
|
static size_t FN(FindBlocks)(const DataType* data, const size_t length,
|
@@ -214,7 +213,6 @@ static void FN(ClusterBlocks)(MemoryManager* m,
|
|
214
213
|
size_t num_final_clusters;
|
215
214
|
static const uint32_t kInvalidIndex = BROTLI_UINT32_MAX;
|
216
215
|
uint32_t* new_index;
|
217
|
-
uint8_t max_type = 0;
|
218
216
|
size_t i;
|
219
217
|
uint32_t sizes[HISTOGRAMS_PER_BATCH] = { 0 };
|
220
218
|
uint32_t new_clusters[HISTOGRAMS_PER_BATCH] = { 0 };
|
@@ -337,6 +335,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
|
|
337
335
|
{
|
338
336
|
uint32_t cur_length = 0;
|
339
337
|
size_t block_idx = 0;
|
338
|
+
uint8_t max_type = 0;
|
340
339
|
for (i = 0; i < num_blocks; ++i) {
|
341
340
|
cur_length += block_lengths[i];
|
342
341
|
if (i + 1 == num_blocks ||
|
@@ -398,7 +397,7 @@ static void FN(SplitByteVector)(MemoryManager* m,
|
|
398
397
|
{
|
399
398
|
/* Find a good path through literals with the good entropy codes. */
|
400
399
|
uint8_t* block_ids = BROTLI_ALLOC(m, uint8_t, length);
|
401
|
-
size_t num_blocks;
|
400
|
+
size_t num_blocks = 0;
|
402
401
|
const size_t bitmaplen = (num_histograms + 7) >> 3;
|
403
402
|
double* insert_cost = BROTLI_ALLOC(m, double, data_size * num_histograms);
|
404
403
|
double* cost = BROTLI_ALLOC(m, double, num_histograms);
|
@@ -13,7 +13,7 @@
|
|
13
13
|
#include <string.h> /* memcpy, memset */
|
14
14
|
|
15
15
|
#include "../common/constants.h"
|
16
|
-
#include
|
16
|
+
#include <brotli/types.h>
|
17
17
|
#include "./context.h"
|
18
18
|
#include "./entropy_encode.h"
|
19
19
|
#include "./entropy_encode_static.h"
|
@@ -27,6 +27,12 @@ extern "C" {
|
|
27
27
|
#endif
|
28
28
|
|
29
29
|
#define MAX_HUFFMAN_TREE_SIZE (2 * BROTLI_NUM_COMMAND_SYMBOLS + 1)
|
30
|
+
/* The size of Huffman dictionary for distances assuming that NPOSTFIX = 0 and
|
31
|
+
NDIRECT = 0. */
|
32
|
+
#define SIMPLE_DISTANCE_ALPHABET_SIZE (BROTLI_NUM_DISTANCE_SHORT_CODES + \
|
33
|
+
(2 * BROTLI_MAX_DISTANCE_BITS))
|
34
|
+
/* SIMPLE_DISTANCE_ALPHABET_SIZE == 64 */
|
35
|
+
#define SIMPLE_DISTANCE_ALPHABET_BITS 6
|
30
36
|
|
31
37
|
/* Represents the range of values belonging to a prefix code:
|
32
38
|
[offset, offset + 2^nbits) */
|
@@ -76,7 +82,7 @@ static BROTLI_INLINE size_t NextBlockTypeCode(
|
|
76
82
|
return type_code;
|
77
83
|
}
|
78
84
|
|
79
|
-
/* nibblesbits represents the 2 bits to encode MNIBBLES (0-3)
|
85
|
+
/* |nibblesbits| represents the 2 bits to encode MNIBBLES (0-3)
|
80
86
|
REQUIRES: length > 0
|
81
87
|
REQUIRES: length <= (1 << 24) */
|
82
88
|
static void BrotliEncodeMlen(size_t length, uint64_t* bits,
|
@@ -343,7 +349,7 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
|
|
343
349
|
code_length_bitdepth[code] = 0;
|
344
350
|
}
|
345
351
|
|
346
|
-
/* Store the real
|
352
|
+
/* Store the real Huffman tree now. */
|
347
353
|
BrotliStoreHuffmanTreeToBitMask(huffman_tree_size,
|
348
354
|
huffman_tree,
|
349
355
|
huffman_tree_extra_bits,
|
@@ -450,7 +456,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
|
|
450
456
|
for (l = length; l != 0;) {
|
451
457
|
--l;
|
452
458
|
if (histogram[l]) {
|
453
|
-
if (
|
459
|
+
if (BROTLI_PREDICT_TRUE(histogram[l] >= count_limit)) {
|
454
460
|
InitHuffmanTree(node, histogram[l], -1, (int16_t)l);
|
455
461
|
} else {
|
456
462
|
InitHuffmanTree(node, count_limit, -1, (int16_t)l);
|
@@ -548,7 +554,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
|
|
548
554
|
/* Complex Huffman Tree */
|
549
555
|
StoreStaticCodeLengthCode(storage_ix, storage);
|
550
556
|
|
551
|
-
/* Actual
|
557
|
+
/* Actual RLE coding. */
|
552
558
|
for (i = 0; i < length;) {
|
553
559
|
const uint8_t value = depth[i];
|
554
560
|
size_t reps = 1;
|
@@ -1151,12 +1157,12 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
|
1151
1157
|
HistogramLiteral lit_histo;
|
1152
1158
|
HistogramCommand cmd_histo;
|
1153
1159
|
HistogramDistance dist_histo;
|
1154
|
-
uint8_t lit_depth[
|
1155
|
-
uint16_t lit_bits[
|
1160
|
+
uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS];
|
1161
|
+
uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
|
1156
1162
|
uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
|
1157
1163
|
uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
|
1158
|
-
uint8_t dist_depth[
|
1159
|
-
uint16_t dist_bits[
|
1164
|
+
uint8_t dist_depth[SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1165
|
+
uint16_t dist_bits[SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1160
1166
|
HuffmanTree* tree;
|
1161
1167
|
|
1162
1168
|
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
|
@@ -1172,13 +1178,14 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
|
1172
1178
|
|
1173
1179
|
tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
|
1174
1180
|
if (BROTLI_IS_OOM(m)) return;
|
1175
|
-
BuildAndStoreHuffmanTree(lit_histo.data_,
|
1181
|
+
BuildAndStoreHuffmanTree(lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS, tree,
|
1176
1182
|
lit_depth, lit_bits,
|
1177
1183
|
storage_ix, storage);
|
1178
1184
|
BuildAndStoreHuffmanTree(cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS, tree,
|
1179
1185
|
cmd_depth, cmd_bits,
|
1180
1186
|
storage_ix, storage);
|
1181
|
-
BuildAndStoreHuffmanTree(dist_histo.data_,
|
1187
|
+
BuildAndStoreHuffmanTree(dist_histo.data_, SIMPLE_DISTANCE_ALPHABET_SIZE,
|
1188
|
+
tree,
|
1182
1189
|
dist_depth, dist_bits,
|
1183
1190
|
storage_ix, storage);
|
1184
1191
|
BROTLI_FREE(m, tree);
|
@@ -1245,8 +1252,8 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
|
|
1245
1252
|
uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
|
1246
1253
|
uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
|
1247
1254
|
uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
|
1248
|
-
uint8_t dist_depth[
|
1249
|
-
uint16_t dist_bits[
|
1255
|
+
uint8_t dist_depth[SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1256
|
+
uint16_t dist_bits[SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1250
1257
|
HistogramClearLiteral(&lit_histo);
|
1251
1258
|
HistogramClearCommand(&cmd_histo);
|
1252
1259
|
HistogramClearDistance(&dist_histo);
|
@@ -1266,7 +1273,8 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
|
|
1266
1273
|
if (BROTLI_IS_OOM(m)) return;
|
1267
1274
|
BrotliBuildAndStoreHuffmanTreeFast(m, dist_histo.data_,
|
1268
1275
|
dist_histo.total_count_,
|
1269
|
-
/* max_bits = */
|
1276
|
+
/* max_bits = */
|
1277
|
+
SIMPLE_DISTANCE_ALPHABET_BITS,
|
1270
1278
|
dist_depth, dist_bits,
|
1271
1279
|
storage_ix, storage);
|
1272
1280
|
if (BROTLI_IS_OOM(m)) return;
|
@@ -1317,18 +1325,6 @@ void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
|
|
1317
1325
|
}
|
1318
1326
|
}
|
1319
1327
|
|
1320
|
-
void BrotliStoreSyncMetaBlock(size_t* BROTLI_RESTRICT storage_ix,
|
1321
|
-
uint8_t* BROTLI_RESTRICT storage) {
|
1322
|
-
/* Empty metadata meta-block bit pattern:
|
1323
|
-
1 bit: is_last (0)
|
1324
|
-
2 bits: num nibbles (3)
|
1325
|
-
1 bit: reserved (0)
|
1326
|
-
2 bits: metadata length bytes (0) */
|
1327
|
-
BrotliWriteBits(6, 6, storage_ix, storage);
|
1328
|
-
JumpToByteBoundary(storage_ix, storage);
|
1329
|
-
}
|
1330
|
-
|
1331
|
-
|
1332
1328
|
#if defined(__cplusplus) || defined(c_plusplus)
|
1333
1329
|
} /* extern "C" */
|
1334
1330
|
#endif
|