brotli 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Gemfile +1 -0
- data/Rakefile +6 -1
- data/brotli.gemspec +1 -1
- data/docs/Brotli.html +485 -0
- data/docs/Brotli/Error.html +124 -0
- data/docs/_index.html +122 -0
- data/docs/class_list.html +51 -0
- data/docs/css/common.css +1 -0
- data/docs/css/full_list.css +58 -0
- data/docs/css/style.css +496 -0
- data/docs/file.README.html +127 -0
- data/docs/file_list.html +56 -0
- data/docs/frames.html +17 -0
- data/docs/index.html +127 -0
- data/docs/js/app.js +292 -0
- data/docs/js/full_list.js +216 -0
- data/docs/js/jquery.js +4 -0
- data/docs/method_list.html +67 -0
- data/docs/top-level-namespace.html +110 -0
- data/ext/brotli/brotli.c +20 -0
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/c/common/constants.h +13 -6
- data/vendor/brotli/c/{dec → common}/context.h +182 -172
- data/vendor/brotli/c/common/dictionary.bin +0 -0
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +1 -1
- data/vendor/brotli/c/common/dictionary.h +4 -4
- data/vendor/brotli/c/common/platform.h +509 -0
- data/vendor/brotli/c/common/transform.c +235 -0
- data/vendor/brotli/c/common/transform.h +80 -0
- data/vendor/brotli/c/common/version.h +8 -1
- data/vendor/brotli/c/dec/bit_reader.c +1 -1
- data/vendor/brotli/c/dec/bit_reader.h +35 -86
- data/vendor/brotli/c/dec/decode.c +322 -205
- data/vendor/brotli/c/dec/huffman.c +35 -37
- data/vendor/brotli/c/dec/huffman.h +13 -9
- data/vendor/brotli/c/dec/prefix.h +3 -4
- data/vendor/brotli/c/dec/state.c +26 -34
- data/vendor/brotli/c/dec/state.h +34 -23
- data/vendor/brotli/c/enc/backward_references.c +25 -15
- data/vendor/brotli/c/enc/backward_references.h +5 -6
- data/vendor/brotli/c/enc/backward_references_hq.c +94 -68
- data/vendor/brotli/c/enc/backward_references_hq.h +22 -25
- data/vendor/brotli/c/enc/backward_references_inc.h +10 -10
- data/vendor/brotli/c/enc/bit_cost.c +1 -1
- data/vendor/brotli/c/enc/bit_cost.h +5 -5
- data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
- data/vendor/brotli/c/enc/block_splitter.c +2 -3
- data/vendor/brotli/c/enc/block_splitter.h +1 -1
- data/vendor/brotli/c/enc/block_splitter_inc.h +11 -11
- data/vendor/brotli/c/enc/brotli_bit_stream.c +102 -101
- data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
- data/vendor/brotli/c/enc/cluster.c +1 -1
- data/vendor/brotli/c/enc/cluster.h +1 -1
- data/vendor/brotli/c/enc/command.h +40 -30
- data/vendor/brotli/c/enc/compress_fragment.c +21 -22
- data/vendor/brotli/c/enc/compress_fragment.h +1 -1
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +101 -68
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1 -1
- data/vendor/brotli/c/enc/encode.c +262 -162
- data/vendor/brotli/c/enc/encoder_dict.c +32 -0
- data/vendor/brotli/c/enc/encoder_dict.h +41 -0
- data/vendor/brotli/c/enc/entropy_encode.c +14 -14
- data/vendor/brotli/c/enc/entropy_encode.h +5 -5
- data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
- data/vendor/brotli/c/enc/fast_log.h +4 -2
- data/vendor/brotli/c/enc/find_match_length.h +3 -3
- data/vendor/brotli/c/enc/hash.h +75 -24
- data/vendor/brotli/c/enc/hash_composite_inc.h +133 -0
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +9 -8
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +10 -9
- data/vendor/brotli/c/enc/hash_rolling_inc.h +215 -0
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +9 -8
- data/vendor/brotli/c/enc/histogram.c +9 -6
- data/vendor/brotli/c/enc/histogram.h +6 -3
- data/vendor/brotli/c/enc/histogram_inc.h +1 -1
- data/vendor/brotli/c/enc/literal_cost.c +5 -5
- data/vendor/brotli/c/enc/literal_cost.h +2 -2
- data/vendor/brotli/c/enc/memory.c +5 -16
- data/vendor/brotli/c/enc/memory.h +40 -1
- data/vendor/brotli/c/enc/metablock.c +163 -25
- data/vendor/brotli/c/enc/metablock.h +13 -8
- data/vendor/brotli/c/enc/metablock_inc.h +1 -1
- data/vendor/brotli/c/enc/params.h +44 -0
- data/vendor/brotli/c/enc/prefix.h +3 -4
- data/vendor/brotli/c/enc/quality.h +29 -24
- data/vendor/brotli/c/enc/ringbuffer.h +15 -11
- data/vendor/brotli/c/enc/static_dict.c +49 -45
- data/vendor/brotli/c/enc/static_dict.h +4 -3
- data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
- data/vendor/brotli/c/enc/utf8_util.c +20 -20
- data/vendor/brotli/c/enc/utf8_util.h +1 -1
- data/vendor/brotli/c/enc/write_bits.h +16 -21
- data/vendor/brotli/c/include/brotli/decode.h +13 -8
- data/vendor/brotli/c/include/brotli/encode.h +33 -8
- data/vendor/brotli/c/include/brotli/port.h +211 -83
- data/vendor/brotli/c/include/brotli/types.h +0 -7
- metadata +33 -12
- data/vendor/brotli/c/dec/port.h +0 -168
- data/vendor/brotli/c/dec/transform.h +0 -300
- data/vendor/brotli/c/enc/context.h +0 -184
- data/vendor/brotli/c/enc/port.h +0 -184
@@ -8,9 +8,9 @@
|
|
8
8
|
|
9
9
|
#include "./histogram.h"
|
10
10
|
|
11
|
+
#include "../common/context.h"
|
11
12
|
#include "./block_splitter.h"
|
12
13
|
#include "./command.h"
|
13
|
-
#include "./context.h"
|
14
14
|
|
15
15
|
#if defined(__cplusplus) || defined(c_plusplus)
|
16
16
|
extern "C" {
|
@@ -63,13 +63,16 @@ void BrotliBuildHistogramsWithContext(
|
|
63
63
|
BlockSplitIteratorNext(&insert_and_copy_it);
|
64
64
|
HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_],
|
65
65
|
cmd->cmd_prefix_);
|
66
|
+
/* TODO: unwrap iterator blocks. */
|
66
67
|
for (j = cmd->insert_len_; j != 0; --j) {
|
67
68
|
size_t context;
|
68
69
|
BlockSplitIteratorNext(&literal_it);
|
69
|
-
context =
|
70
|
-
|
71
|
-
|
72
|
-
|
70
|
+
context = literal_it.type_;
|
71
|
+
if (context_modes) {
|
72
|
+
ContextLut lut = BROTLI_CONTEXT_LUT(context_modes[context]);
|
73
|
+
context = (context << BROTLI_LITERAL_CONTEXT_BITS) +
|
74
|
+
BROTLI_CONTEXT(prev_byte, prev_byte2, lut);
|
75
|
+
}
|
73
76
|
HistogramAddLiteral(&literal_histograms[context],
|
74
77
|
ringbuffer[pos & mask]);
|
75
78
|
prev_byte2 = prev_byte;
|
@@ -86,7 +89,7 @@ void BrotliBuildHistogramsWithContext(
|
|
86
89
|
context = (dist_it.type_ << BROTLI_DISTANCE_CONTEXT_BITS) +
|
87
90
|
CommandDistanceContext(cmd);
|
88
91
|
HistogramAddDistance(©_dist_histograms[context],
|
89
|
-
cmd->dist_prefix_);
|
92
|
+
cmd->dist_prefix_ & 0x3FF);
|
90
93
|
}
|
91
94
|
}
|
92
95
|
}
|
@@ -12,16 +12,19 @@
|
|
12
12
|
#include <string.h> /* memset */
|
13
13
|
|
14
14
|
#include "../common/constants.h"
|
15
|
+
#include "../common/context.h"
|
16
|
+
#include "../common/platform.h"
|
15
17
|
#include <brotli/types.h>
|
16
18
|
#include "./block_splitter.h"
|
17
19
|
#include "./command.h"
|
18
|
-
#include "./context.h"
|
19
|
-
#include "./port.h"
|
20
20
|
|
21
21
|
#if defined(__cplusplus) || defined(c_plusplus)
|
22
22
|
extern "C" {
|
23
23
|
#endif
|
24
24
|
|
25
|
+
/* The distance symbols effectively used by "Large Window Brotli" (32-bit). */
|
26
|
+
#define BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS 544
|
27
|
+
|
25
28
|
#define FN(X) X ## Literal
|
26
29
|
#define DATA_SIZE BROTLI_NUM_LITERAL_SYMBOLS
|
27
30
|
#define DataType uint8_t
|
@@ -38,7 +41,7 @@ extern "C" {
|
|
38
41
|
#undef FN
|
39
42
|
|
40
43
|
#define FN(X) X ## Distance
|
41
|
-
#define DATA_SIZE
|
44
|
+
#define DATA_SIZE BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS
|
42
45
|
#include "./histogram_inc.h" /* NOLINT(build/include) */
|
43
46
|
#undef DataType
|
44
47
|
#undef DATA_SIZE
|
@@ -33,7 +33,7 @@ static BROTLI_INLINE void FN(HistogramAdd)(FN(Histogram)* self, size_t val) {
|
|
33
33
|
}
|
34
34
|
|
35
35
|
static BROTLI_INLINE void FN(HistogramAddVector)(FN(Histogram)* self,
|
36
|
-
const DataType
|
36
|
+
const DataType* p, size_t n) {
|
37
37
|
self->total_count_ += n;
|
38
38
|
n += 1;
|
39
39
|
while (--n) ++self->data_[*p++];
|
@@ -9,9 +9,9 @@
|
|
9
9
|
|
10
10
|
#include "./literal_cost.h"
|
11
11
|
|
12
|
+
#include "../common/platform.h"
|
12
13
|
#include <brotli/types.h>
|
13
14
|
#include "./fast_log.h"
|
14
|
-
#include "./port.h"
|
15
15
|
#include "./utf8_util.h"
|
16
16
|
|
17
17
|
#if defined(__cplusplus) || defined(c_plusplus)
|
@@ -25,7 +25,7 @@ static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
|
|
25
25
|
return BROTLI_MIN(size_t, 1, clamp);
|
26
26
|
} else {
|
27
27
|
/* Let's decide over the last byte if this ends the sequence. */
|
28
|
-
if (last <
|
28
|
+
if (last < 0xE0) {
|
29
29
|
return 0; /* Completed two or three byte coding. */
|
30
30
|
} else { /* Next one is the 'Byte 3' of utf-8 encoding. */
|
31
31
|
return BROTLI_MIN(size_t, 2, clamp);
|
@@ -34,7 +34,7 @@ static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
|
|
34
34
|
}
|
35
35
|
|
36
36
|
static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
|
37
|
-
const uint8_t
|
37
|
+
const uint8_t* data) {
|
38
38
|
size_t counts[3] = { 0 };
|
39
39
|
size_t max_utf8 = 1; /* should be 2, but 1 compresses better. */
|
40
40
|
size_t last_c = 0;
|
@@ -54,7 +54,7 @@ static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
|
|
54
54
|
}
|
55
55
|
|
56
56
|
static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
57
|
-
const uint8_t
|
57
|
+
const uint8_t* data, float* cost) {
|
58
58
|
/* max_utf8 is 0 (normal ASCII single byte modeling),
|
59
59
|
1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
|
60
60
|
const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
|
@@ -125,7 +125,7 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
|
125
125
|
}
|
126
126
|
|
127
127
|
void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
|
128
|
-
const uint8_t
|
128
|
+
const uint8_t* data, float* cost) {
|
129
129
|
if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
|
130
130
|
EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost);
|
131
131
|
return;
|
@@ -10,8 +10,8 @@
|
|
10
10
|
#ifndef BROTLI_ENC_LITERAL_COST_H_
|
11
11
|
#define BROTLI_ENC_LITERAL_COST_H_
|
12
12
|
|
13
|
+
#include "../common/platform.h"
|
13
14
|
#include <brotli/types.h>
|
14
|
-
#include "./port.h"
|
15
15
|
|
16
16
|
#if defined(__cplusplus) || defined(c_plusplus)
|
17
17
|
extern "C" {
|
@@ -21,7 +21,7 @@ extern "C" {
|
|
21
21
|
ring-buffer (data, mask) will take entropy coded and writes these estimates
|
22
22
|
to the cost[0..len) array. */
|
23
23
|
BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals(
|
24
|
-
size_t pos, size_t len, size_t mask, const uint8_t
|
24
|
+
size_t pos, size_t len, size_t mask, const uint8_t* data, float* cost);
|
25
25
|
|
26
26
|
#if defined(__cplusplus) || defined(c_plusplus)
|
27
27
|
} /* extern "C" */
|
@@ -9,12 +9,11 @@
|
|
9
9
|
|
10
10
|
#include "./memory.h"
|
11
11
|
|
12
|
-
#include <assert.h>
|
13
12
|
#include <stdlib.h> /* exit, free, malloc */
|
14
13
|
#include <string.h> /* memcpy */
|
15
14
|
|
15
|
+
#include "../common/platform.h"
|
16
16
|
#include <brotli/types.h>
|
17
|
-
#include "./port.h"
|
18
17
|
|
19
18
|
#if defined(__cplusplus) || defined(c_plusplus)
|
20
19
|
extern "C" {
|
@@ -28,22 +27,12 @@ extern "C" {
|
|
28
27
|
#define NEW_ALLOCATED_OFFSET MAX_PERM_ALLOCATED
|
29
28
|
#define NEW_FREED_OFFSET (MAX_PERM_ALLOCATED + MAX_NEW_ALLOCATED)
|
30
29
|
|
31
|
-
static void* DefaultAllocFunc(void* opaque, size_t size) {
|
32
|
-
BROTLI_UNUSED(opaque);
|
33
|
-
return malloc(size);
|
34
|
-
}
|
35
|
-
|
36
|
-
static void DefaultFreeFunc(void* opaque, void* address) {
|
37
|
-
BROTLI_UNUSED(opaque);
|
38
|
-
free(address);
|
39
|
-
}
|
40
|
-
|
41
30
|
void BrotliInitMemoryManager(
|
42
31
|
MemoryManager* m, brotli_alloc_func alloc_func, brotli_free_func free_func,
|
43
32
|
void* opaque) {
|
44
33
|
if (!alloc_func) {
|
45
|
-
m->alloc_func =
|
46
|
-
m->free_func =
|
34
|
+
m->alloc_func = BrotliDefaultAllocFunc;
|
35
|
+
m->free_func = BrotliDefaultFreeFunc;
|
47
36
|
m->opaque = 0;
|
48
37
|
} else {
|
49
38
|
m->alloc_func = alloc_func;
|
@@ -132,11 +121,11 @@ static void CollectGarbagePointers(MemoryManager* m) {
|
|
132
121
|
m->pointers + NEW_FREED_OFFSET, m->new_freed);
|
133
122
|
m->perm_allocated -= annihilated;
|
134
123
|
m->new_freed -= annihilated;
|
135
|
-
|
124
|
+
BROTLI_DCHECK(m->new_freed == 0);
|
136
125
|
}
|
137
126
|
|
138
127
|
if (m->new_allocated != 0) {
|
139
|
-
|
128
|
+
BROTLI_DCHECK(m->perm_allocated + m->new_allocated <= MAX_PERM_ALLOCATED);
|
140
129
|
memcpy(m->pointers + PERM_ALLOCATED_OFFSET + m->perm_allocated,
|
141
130
|
m->pointers + NEW_ALLOCATED_OFFSET,
|
142
131
|
sizeof(void*) * m->new_allocated);
|
@@ -9,8 +9,10 @@
|
|
9
9
|
#ifndef BROTLI_ENC_MEMORY_H_
|
10
10
|
#define BROTLI_ENC_MEMORY_H_
|
11
11
|
|
12
|
+
#include <string.h> /* memcpy */
|
13
|
+
|
14
|
+
#include "../common/platform.h"
|
12
15
|
#include <brotli/types.h>
|
13
|
-
#include "./port.h"
|
14
16
|
|
15
17
|
#if defined(__cplusplus) || defined(c_plusplus)
|
16
18
|
extern "C" {
|
@@ -56,6 +58,43 @@ BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
|
|
56
58
|
|
57
59
|
BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m);
|
58
60
|
|
61
|
+
/*
|
62
|
+
Dynamically grows array capacity to at least the requested size
|
63
|
+
M: MemoryManager
|
64
|
+
T: data type
|
65
|
+
A: array
|
66
|
+
C: capacity
|
67
|
+
R: requested size
|
68
|
+
*/
|
69
|
+
#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \
|
70
|
+
if (C < (R)) { \
|
71
|
+
size_t _new_size = (C == 0) ? (R) : C; \
|
72
|
+
T* new_array; \
|
73
|
+
while (_new_size < (R)) _new_size *= 2; \
|
74
|
+
new_array = BROTLI_ALLOC((M), T, _new_size); \
|
75
|
+
if (!BROTLI_IS_OOM(M) && C != 0) \
|
76
|
+
memcpy(new_array, A, C * sizeof(T)); \
|
77
|
+
BROTLI_FREE((M), A); \
|
78
|
+
A = new_array; \
|
79
|
+
C = _new_size; \
|
80
|
+
} \
|
81
|
+
}
|
82
|
+
|
83
|
+
/*
|
84
|
+
Appends value and dynamically grows array capacity when needed
|
85
|
+
M: MemoryManager
|
86
|
+
T: data type
|
87
|
+
A: array
|
88
|
+
C: array capacity
|
89
|
+
S: array size
|
90
|
+
V: value to append
|
91
|
+
*/
|
92
|
+
#define BROTLI_ENSURE_CAPACITY_APPEND(M, T, A, C, S, V) { \
|
93
|
+
(S)++; \
|
94
|
+
BROTLI_ENSURE_CAPACITY(M, T, A, C, S); \
|
95
|
+
A[(S) - 1] = (V); \
|
96
|
+
}
|
97
|
+
|
59
98
|
#if defined(__cplusplus) || defined(c_plusplus)
|
60
99
|
} /* extern "C" */
|
61
100
|
#endif
|
@@ -10,29 +10,131 @@
|
|
10
10
|
#include "./metablock.h"
|
11
11
|
|
12
12
|
#include "../common/constants.h"
|
13
|
+
#include "../common/context.h"
|
14
|
+
#include "../common/platform.h"
|
13
15
|
#include <brotli/types.h>
|
14
16
|
#include "./bit_cost.h"
|
15
17
|
#include "./block_splitter.h"
|
16
18
|
#include "./cluster.h"
|
17
|
-
#include "./context.h"
|
18
19
|
#include "./entropy_encode.h"
|
19
20
|
#include "./histogram.h"
|
20
21
|
#include "./memory.h"
|
21
|
-
#include "./port.h"
|
22
22
|
#include "./quality.h"
|
23
23
|
|
24
24
|
#if defined(__cplusplus) || defined(c_plusplus)
|
25
25
|
extern "C" {
|
26
26
|
#endif
|
27
27
|
|
28
|
+
void BrotliInitDistanceParams(BrotliEncoderParams* params,
|
29
|
+
uint32_t npostfix, uint32_t ndirect) {
|
30
|
+
BrotliDistanceParams* dist_params = ¶ms->dist;
|
31
|
+
uint32_t alphabet_size, max_distance;
|
32
|
+
|
33
|
+
dist_params->distance_postfix_bits = npostfix;
|
34
|
+
dist_params->num_direct_distance_codes = ndirect;
|
35
|
+
|
36
|
+
alphabet_size = BROTLI_DISTANCE_ALPHABET_SIZE(
|
37
|
+
npostfix, ndirect, BROTLI_MAX_DISTANCE_BITS);
|
38
|
+
max_distance = ndirect + (1U << (BROTLI_MAX_DISTANCE_BITS + npostfix + 2)) -
|
39
|
+
(1U << (npostfix + 2));
|
40
|
+
|
41
|
+
if (params->large_window) {
|
42
|
+
static const uint32_t bound[BROTLI_MAX_NPOSTFIX + 1] = {0, 4, 12, 28};
|
43
|
+
uint32_t postfix = 1U << npostfix;
|
44
|
+
alphabet_size = BROTLI_DISTANCE_ALPHABET_SIZE(
|
45
|
+
npostfix, ndirect, BROTLI_LARGE_MAX_DISTANCE_BITS);
|
46
|
+
/* The maximum distance is set so that no distance symbol used can encode
|
47
|
+
a distance larger than BROTLI_MAX_ALLOWED_DISTANCE with all
|
48
|
+
its extra bits set. */
|
49
|
+
if (ndirect < bound[npostfix]) {
|
50
|
+
max_distance = BROTLI_MAX_ALLOWED_DISTANCE - (bound[npostfix] - ndirect);
|
51
|
+
} else if (ndirect >= bound[npostfix] + postfix) {
|
52
|
+
max_distance = (3U << 29) - 4 + (ndirect - bound[npostfix]);
|
53
|
+
} else {
|
54
|
+
max_distance = BROTLI_MAX_ALLOWED_DISTANCE;
|
55
|
+
}
|
56
|
+
}
|
57
|
+
|
58
|
+
dist_params->alphabet_size = alphabet_size;
|
59
|
+
dist_params->max_distance = max_distance;
|
60
|
+
}
|
61
|
+
|
62
|
+
static void RecomputeDistancePrefixes(Command* cmds,
|
63
|
+
size_t num_commands,
|
64
|
+
const BrotliDistanceParams* orig_params,
|
65
|
+
const BrotliDistanceParams* new_params) {
|
66
|
+
size_t i;
|
67
|
+
|
68
|
+
if (orig_params->distance_postfix_bits == new_params->distance_postfix_bits &&
|
69
|
+
orig_params->num_direct_distance_codes ==
|
70
|
+
new_params->num_direct_distance_codes) {
|
71
|
+
return;
|
72
|
+
}
|
73
|
+
|
74
|
+
for (i = 0; i < num_commands; ++i) {
|
75
|
+
Command* cmd = &cmds[i];
|
76
|
+
if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
|
77
|
+
PrefixEncodeCopyDistance(CommandRestoreDistanceCode(cmd, orig_params),
|
78
|
+
new_params->num_direct_distance_codes,
|
79
|
+
new_params->distance_postfix_bits,
|
80
|
+
&cmd->dist_prefix_,
|
81
|
+
&cmd->dist_extra_);
|
82
|
+
}
|
83
|
+
}
|
84
|
+
}
|
85
|
+
|
86
|
+
static BROTLI_BOOL ComputeDistanceCost(const Command* cmds,
|
87
|
+
size_t num_commands,
|
88
|
+
const BrotliDistanceParams* orig_params,
|
89
|
+
const BrotliDistanceParams* new_params,
|
90
|
+
double* cost) {
|
91
|
+
size_t i;
|
92
|
+
BROTLI_BOOL equal_params = BROTLI_FALSE;
|
93
|
+
uint16_t dist_prefix;
|
94
|
+
uint32_t dist_extra;
|
95
|
+
double extra_bits = 0.0;
|
96
|
+
HistogramDistance histo;
|
97
|
+
HistogramClearDistance(&histo);
|
98
|
+
|
99
|
+
if (orig_params->distance_postfix_bits == new_params->distance_postfix_bits &&
|
100
|
+
orig_params->num_direct_distance_codes ==
|
101
|
+
new_params->num_direct_distance_codes) {
|
102
|
+
equal_params = BROTLI_TRUE;
|
103
|
+
}
|
104
|
+
|
105
|
+
for (i = 0; i < num_commands; i++) {
|
106
|
+
const Command* cmd = &cmds[i];
|
107
|
+
if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
|
108
|
+
if (equal_params) {
|
109
|
+
dist_prefix = cmd->dist_prefix_;
|
110
|
+
} else {
|
111
|
+
uint32_t distance = CommandRestoreDistanceCode(cmd, orig_params);
|
112
|
+
if (distance > new_params->max_distance) {
|
113
|
+
return BROTLI_FALSE;
|
114
|
+
}
|
115
|
+
PrefixEncodeCopyDistance(distance,
|
116
|
+
new_params->num_direct_distance_codes,
|
117
|
+
new_params->distance_postfix_bits,
|
118
|
+
&dist_prefix,
|
119
|
+
&dist_extra);
|
120
|
+
}
|
121
|
+
HistogramAddDistance(&histo, dist_prefix & 0x3FF);
|
122
|
+
extra_bits += dist_prefix >> 10;
|
123
|
+
}
|
124
|
+
}
|
125
|
+
|
126
|
+
*cost = BrotliPopulationCostDistance(&histo) + extra_bits;
|
127
|
+
return BROTLI_TRUE;
|
128
|
+
}
|
129
|
+
|
28
130
|
void BrotliBuildMetaBlock(MemoryManager* m,
|
29
131
|
const uint8_t* ringbuffer,
|
30
132
|
const size_t pos,
|
31
133
|
const size_t mask,
|
32
|
-
|
134
|
+
BrotliEncoderParams* params,
|
33
135
|
uint8_t prev_byte,
|
34
136
|
uint8_t prev_byte2,
|
35
|
-
|
137
|
+
Command* cmds,
|
36
138
|
size_t num_commands,
|
37
139
|
ContextType literal_context_mode,
|
38
140
|
MetaBlockSplit* mb) {
|
@@ -45,6 +147,46 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
45
147
|
size_t distance_histograms_size;
|
46
148
|
size_t i;
|
47
149
|
size_t literal_context_multiplier = 1;
|
150
|
+
uint32_t npostfix;
|
151
|
+
uint32_t ndirect_msb = 0;
|
152
|
+
BROTLI_BOOL check_orig = BROTLI_TRUE;
|
153
|
+
double best_dist_cost = 1e99;
|
154
|
+
BrotliEncoderParams orig_params = *params;
|
155
|
+
BrotliEncoderParams new_params = *params;
|
156
|
+
|
157
|
+
for (npostfix = 0; npostfix <= BROTLI_MAX_NPOSTFIX; npostfix++) {
|
158
|
+
for (; ndirect_msb < 16; ndirect_msb++) {
|
159
|
+
uint32_t ndirect = ndirect_msb << npostfix;
|
160
|
+
BROTLI_BOOL skip;
|
161
|
+
double dist_cost;
|
162
|
+
BrotliInitDistanceParams(&new_params, npostfix, ndirect);
|
163
|
+
if (npostfix == orig_params.dist.distance_postfix_bits &&
|
164
|
+
ndirect == orig_params.dist.num_direct_distance_codes) {
|
165
|
+
check_orig = BROTLI_FALSE;
|
166
|
+
}
|
167
|
+
skip = !ComputeDistanceCost(
|
168
|
+
cmds, num_commands,
|
169
|
+
&orig_params.dist, &new_params.dist, &dist_cost);
|
170
|
+
if (skip || (dist_cost > best_dist_cost)) {
|
171
|
+
break;
|
172
|
+
}
|
173
|
+
best_dist_cost = dist_cost;
|
174
|
+
params->dist = new_params.dist;
|
175
|
+
}
|
176
|
+
if (ndirect_msb > 0) ndirect_msb--;
|
177
|
+
ndirect_msb /= 2;
|
178
|
+
}
|
179
|
+
if (check_orig) {
|
180
|
+
double dist_cost;
|
181
|
+
ComputeDistanceCost(cmds, num_commands,
|
182
|
+
&orig_params.dist, &orig_params.dist, &dist_cost);
|
183
|
+
if (dist_cost < best_dist_cost) {
|
184
|
+
best_dist_cost = dist_cost;
|
185
|
+
params->dist = orig_params.dist;
|
186
|
+
}
|
187
|
+
}
|
188
|
+
RecomputeDistancePrefixes(cmds, num_commands,
|
189
|
+
&orig_params.dist, ¶ms->dist);
|
48
190
|
|
49
191
|
BrotliSplitBlock(m, cmds, num_commands,
|
50
192
|
ringbuffer, pos, mask, params,
|
@@ -77,7 +219,7 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
77
219
|
if (BROTLI_IS_OOM(m)) return;
|
78
220
|
ClearHistogramsDistance(distance_histograms, distance_histograms_size);
|
79
221
|
|
80
|
-
|
222
|
+
BROTLI_DCHECK(mb->command_histograms == 0);
|
81
223
|
mb->command_histograms_size = mb->command_split.num_types;
|
82
224
|
mb->command_histograms =
|
83
225
|
BROTLI_ALLOC(m, HistogramCommand, mb->command_histograms_size);
|
@@ -90,14 +232,14 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
90
232
|
literal_histograms, mb->command_histograms, distance_histograms);
|
91
233
|
BROTLI_FREE(m, literal_context_modes);
|
92
234
|
|
93
|
-
|
235
|
+
BROTLI_DCHECK(mb->literal_context_map == 0);
|
94
236
|
mb->literal_context_map_size =
|
95
237
|
mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
|
96
238
|
mb->literal_context_map =
|
97
239
|
BROTLI_ALLOC(m, uint32_t, mb->literal_context_map_size);
|
98
240
|
if (BROTLI_IS_OOM(m)) return;
|
99
241
|
|
100
|
-
|
242
|
+
BROTLI_DCHECK(mb->literal_histograms == 0);
|
101
243
|
mb->literal_histograms_size = mb->literal_context_map_size;
|
102
244
|
mb->literal_histograms =
|
103
245
|
BROTLI_ALLOC(m, HistogramLiteral, mb->literal_histograms_size);
|
@@ -121,14 +263,14 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
121
263
|
}
|
122
264
|
}
|
123
265
|
|
124
|
-
|
266
|
+
BROTLI_DCHECK(mb->distance_context_map == 0);
|
125
267
|
mb->distance_context_map_size =
|
126
268
|
mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;
|
127
269
|
mb->distance_context_map =
|
128
270
|
BROTLI_ALLOC(m, uint32_t, mb->distance_context_map_size);
|
129
271
|
if (BROTLI_IS_OOM(m)) return;
|
130
272
|
|
131
|
-
|
273
|
+
BROTLI_DCHECK(mb->distance_histograms == 0);
|
132
274
|
mb->distance_histograms_size = mb->distance_context_map_size;
|
133
275
|
mb->distance_histograms =
|
134
276
|
BROTLI_ALLOC(m, HistogramDistance, mb->distance_histograms_size);
|
@@ -200,7 +342,7 @@ static void InitContextBlockSplitter(
|
|
200
342
|
size_t* histograms_size) {
|
201
343
|
size_t max_num_blocks = num_symbols / min_block_size + 1;
|
202
344
|
size_t max_num_types;
|
203
|
-
|
345
|
+
BROTLI_DCHECK(num_contexts <= BROTLI_MAX_STATIC_CONTEXTS);
|
204
346
|
|
205
347
|
self->alphabet_size_ = alphabet_size;
|
206
348
|
self->num_contexts_ = num_contexts;
|
@@ -226,7 +368,7 @@ static void InitContextBlockSplitter(
|
|
226
368
|
if (BROTLI_IS_OOM(m)) return;
|
227
369
|
split->num_blocks = max_num_blocks;
|
228
370
|
if (BROTLI_IS_OOM(m)) return;
|
229
|
-
|
371
|
+
BROTLI_DCHECK(*histograms == 0);
|
230
372
|
*histograms_size = max_num_types * num_contexts;
|
231
373
|
*histograms = BROTLI_ALLOC(m, HistogramLiteral, *histograms_size);
|
232
374
|
self->histograms_ = *histograms;
|
@@ -379,7 +521,7 @@ static void MapStaticContexts(MemoryManager* m,
|
|
379
521
|
const uint32_t* static_context_map,
|
380
522
|
MetaBlockSplit* mb) {
|
381
523
|
size_t i;
|
382
|
-
|
524
|
+
BROTLI_DCHECK(mb->literal_context_map == 0);
|
383
525
|
mb->literal_context_map_size =
|
384
526
|
mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
|
385
527
|
mb->literal_context_map =
|
@@ -398,9 +540,9 @@ static void MapStaticContexts(MemoryManager* m,
|
|
398
540
|
|
399
541
|
static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
|
400
542
|
MemoryManager* m, const uint8_t* ringbuffer, size_t pos, size_t mask,
|
401
|
-
uint8_t prev_byte, uint8_t prev_byte2,
|
543
|
+
uint8_t prev_byte, uint8_t prev_byte2, ContextLut literal_context_lut,
|
402
544
|
const size_t num_contexts, const uint32_t* static_context_map,
|
403
|
-
const Command
|
545
|
+
const Command* commands, size_t n_commands, MetaBlockSplit* mb) {
|
404
546
|
union {
|
405
547
|
BlockSplitterLiteral plain;
|
406
548
|
ContextBlockSplitter ctx;
|
@@ -441,7 +583,8 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
|
|
441
583
|
if (num_contexts == 1) {
|
442
584
|
BlockSplitterAddSymbolLiteral(&lit_blocks.plain, literal);
|
443
585
|
} else {
|
444
|
-
size_t context =
|
586
|
+
size_t context =
|
587
|
+
BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut);
|
445
588
|
ContextBlockSplitterAddSymbol(&lit_blocks.ctx, m, literal,
|
446
589
|
static_context_map[context]);
|
447
590
|
if (BROTLI_IS_OOM(m)) return;
|
@@ -455,7 +598,7 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
|
|
455
598
|
prev_byte2 = ringbuffer[(pos - 2) & mask];
|
456
599
|
prev_byte = ringbuffer[(pos - 1) & mask];
|
457
600
|
if (cmd.cmd_prefix_ >= 128) {
|
458
|
-
BlockSplitterAddSymbolDistance(&dist_blocks, cmd.dist_prefix_);
|
601
|
+
BlockSplitterAddSymbolDistance(&dist_blocks, cmd.dist_prefix_ & 0x3FF);
|
459
602
|
}
|
460
603
|
}
|
461
604
|
}
|
@@ -482,7 +625,7 @@ void BrotliBuildMetaBlockGreedy(MemoryManager* m,
|
|
482
625
|
size_t mask,
|
483
626
|
uint8_t prev_byte,
|
484
627
|
uint8_t prev_byte2,
|
485
|
-
|
628
|
+
ContextLut literal_context_lut,
|
486
629
|
size_t num_contexts,
|
487
630
|
const uint32_t* static_context_map,
|
488
631
|
const Command* commands,
|
@@ -490,19 +633,17 @@ void BrotliBuildMetaBlockGreedy(MemoryManager* m,
|
|
490
633
|
MetaBlockSplit* mb) {
|
491
634
|
if (num_contexts == 1) {
|
492
635
|
BrotliBuildMetaBlockGreedyInternal(m, ringbuffer, pos, mask, prev_byte,
|
493
|
-
prev_byte2,
|
636
|
+
prev_byte2, literal_context_lut, 1, NULL, commands, n_commands, mb);
|
494
637
|
} else {
|
495
638
|
BrotliBuildMetaBlockGreedyInternal(m, ringbuffer, pos, mask, prev_byte,
|
496
|
-
prev_byte2,
|
639
|
+
prev_byte2, literal_context_lut, num_contexts, static_context_map,
|
497
640
|
commands, n_commands, mb);
|
498
641
|
}
|
499
642
|
}
|
500
643
|
|
501
|
-
void BrotliOptimizeHistograms(
|
502
|
-
size_t distance_postfix_bits,
|
644
|
+
void BrotliOptimizeHistograms(uint32_t num_distance_codes,
|
503
645
|
MetaBlockSplit* mb) {
|
504
646
|
uint8_t good_for_rle[BROTLI_NUM_COMMAND_SYMBOLS];
|
505
|
-
size_t num_distance_codes;
|
506
647
|
size_t i;
|
507
648
|
for (i = 0; i < mb->literal_histograms_size; ++i) {
|
508
649
|
BrotliOptimizeHuffmanCountsForRle(256, mb->literal_histograms[i].data_,
|
@@ -513,9 +654,6 @@ void BrotliOptimizeHistograms(size_t num_direct_distance_codes,
|
|
513
654
|
mb->command_histograms[i].data_,
|
514
655
|
good_for_rle);
|
515
656
|
}
|
516
|
-
num_distance_codes = BROTLI_NUM_DISTANCE_SHORT_CODES +
|
517
|
-
num_direct_distance_codes +
|
518
|
-
((2 * BROTLI_MAX_DISTANCE_BITS) << distance_postfix_bits);
|
519
657
|
for (i = 0; i < mb->distance_histograms_size; ++i) {
|
520
658
|
BrotliOptimizeHuffmanCountsForRle(num_distance_codes,
|
521
659
|
mb->distance_histograms[i].data_,
|