brotli 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile +1 -0
- data/Rakefile +6 -1
- data/brotli.gemspec +1 -1
- data/docs/Brotli.html +485 -0
- data/docs/Brotli/Error.html +124 -0
- data/docs/_index.html +122 -0
- data/docs/class_list.html +51 -0
- data/docs/css/common.css +1 -0
- data/docs/css/full_list.css +58 -0
- data/docs/css/style.css +496 -0
- data/docs/file.README.html +127 -0
- data/docs/file_list.html +56 -0
- data/docs/frames.html +17 -0
- data/docs/index.html +127 -0
- data/docs/js/app.js +292 -0
- data/docs/js/full_list.js +216 -0
- data/docs/js/jquery.js +4 -0
- data/docs/method_list.html +67 -0
- data/docs/top-level-namespace.html +110 -0
- data/ext/brotli/brotli.c +20 -0
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/c/common/constants.h +13 -6
- data/vendor/brotli/c/{dec → common}/context.h +182 -172
- data/vendor/brotli/c/common/dictionary.bin +0 -0
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +1 -1
- data/vendor/brotli/c/common/dictionary.h +4 -4
- data/vendor/brotli/c/common/platform.h +509 -0
- data/vendor/brotli/c/common/transform.c +235 -0
- data/vendor/brotli/c/common/transform.h +80 -0
- data/vendor/brotli/c/common/version.h +8 -1
- data/vendor/brotli/c/dec/bit_reader.c +1 -1
- data/vendor/brotli/c/dec/bit_reader.h +35 -86
- data/vendor/brotli/c/dec/decode.c +322 -205
- data/vendor/brotli/c/dec/huffman.c +35 -37
- data/vendor/brotli/c/dec/huffman.h +13 -9
- data/vendor/brotli/c/dec/prefix.h +3 -4
- data/vendor/brotli/c/dec/state.c +26 -34
- data/vendor/brotli/c/dec/state.h +34 -23
- data/vendor/brotli/c/enc/backward_references.c +25 -15
- data/vendor/brotli/c/enc/backward_references.h +5 -6
- data/vendor/brotli/c/enc/backward_references_hq.c +94 -68
- data/vendor/brotli/c/enc/backward_references_hq.h +22 -25
- data/vendor/brotli/c/enc/backward_references_inc.h +10 -10
- data/vendor/brotli/c/enc/bit_cost.c +1 -1
- data/vendor/brotli/c/enc/bit_cost.h +5 -5
- data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
- data/vendor/brotli/c/enc/block_splitter.c +2 -3
- data/vendor/brotli/c/enc/block_splitter.h +1 -1
- data/vendor/brotli/c/enc/block_splitter_inc.h +11 -11
- data/vendor/brotli/c/enc/brotli_bit_stream.c +102 -101
- data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
- data/vendor/brotli/c/enc/cluster.c +1 -1
- data/vendor/brotli/c/enc/cluster.h +1 -1
- data/vendor/brotli/c/enc/command.h +40 -30
- data/vendor/brotli/c/enc/compress_fragment.c +21 -22
- data/vendor/brotli/c/enc/compress_fragment.h +1 -1
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +101 -68
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1 -1
- data/vendor/brotli/c/enc/encode.c +262 -162
- data/vendor/brotli/c/enc/encoder_dict.c +32 -0
- data/vendor/brotli/c/enc/encoder_dict.h +41 -0
- data/vendor/brotli/c/enc/entropy_encode.c +14 -14
- data/vendor/brotli/c/enc/entropy_encode.h +5 -5
- data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
- data/vendor/brotli/c/enc/fast_log.h +4 -2
- data/vendor/brotli/c/enc/find_match_length.h +3 -3
- data/vendor/brotli/c/enc/hash.h +75 -24
- data/vendor/brotli/c/enc/hash_composite_inc.h +133 -0
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +9 -8
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +10 -9
- data/vendor/brotli/c/enc/hash_rolling_inc.h +215 -0
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +9 -8
- data/vendor/brotli/c/enc/histogram.c +9 -6
- data/vendor/brotli/c/enc/histogram.h +6 -3
- data/vendor/brotli/c/enc/histogram_inc.h +1 -1
- data/vendor/brotli/c/enc/literal_cost.c +5 -5
- data/vendor/brotli/c/enc/literal_cost.h +2 -2
- data/vendor/brotli/c/enc/memory.c +5 -16
- data/vendor/brotli/c/enc/memory.h +40 -1
- data/vendor/brotli/c/enc/metablock.c +163 -25
- data/vendor/brotli/c/enc/metablock.h +13 -8
- data/vendor/brotli/c/enc/metablock_inc.h +1 -1
- data/vendor/brotli/c/enc/params.h +44 -0
- data/vendor/brotli/c/enc/prefix.h +3 -4
- data/vendor/brotli/c/enc/quality.h +29 -24
- data/vendor/brotli/c/enc/ringbuffer.h +15 -11
- data/vendor/brotli/c/enc/static_dict.c +49 -45
- data/vendor/brotli/c/enc/static_dict.h +4 -3
- data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
- data/vendor/brotli/c/enc/utf8_util.c +20 -20
- data/vendor/brotli/c/enc/utf8_util.h +1 -1
- data/vendor/brotli/c/enc/write_bits.h +16 -21
- data/vendor/brotli/c/include/brotli/decode.h +13 -8
- data/vendor/brotli/c/include/brotli/encode.h +33 -8
- data/vendor/brotli/c/include/brotli/port.h +211 -83
- data/vendor/brotli/c/include/brotli/types.h +0 -7
- metadata +33 -12
- data/vendor/brotli/c/dec/port.h +0 -168
- data/vendor/brotli/c/dec/transform.h +0 -300
- data/vendor/brotli/c/enc/context.h +0 -184
- data/vendor/brotli/c/enc/port.h +0 -184
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
|
|
9
9
|
#include "./histogram.h"
|
|
10
10
|
|
|
11
|
+
#include "../common/context.h"
|
|
11
12
|
#include "./block_splitter.h"
|
|
12
13
|
#include "./command.h"
|
|
13
|
-
#include "./context.h"
|
|
14
14
|
|
|
15
15
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
16
16
|
extern "C" {
|
|
@@ -63,13 +63,16 @@ void BrotliBuildHistogramsWithContext(
|
|
|
63
63
|
BlockSplitIteratorNext(&insert_and_copy_it);
|
|
64
64
|
HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_],
|
|
65
65
|
cmd->cmd_prefix_);
|
|
66
|
+
/* TODO: unwrap iterator blocks. */
|
|
66
67
|
for (j = cmd->insert_len_; j != 0; --j) {
|
|
67
68
|
size_t context;
|
|
68
69
|
BlockSplitIteratorNext(&literal_it);
|
|
69
|
-
context =
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
context = literal_it.type_;
|
|
71
|
+
if (context_modes) {
|
|
72
|
+
ContextLut lut = BROTLI_CONTEXT_LUT(context_modes[context]);
|
|
73
|
+
context = (context << BROTLI_LITERAL_CONTEXT_BITS) +
|
|
74
|
+
BROTLI_CONTEXT(prev_byte, prev_byte2, lut);
|
|
75
|
+
}
|
|
73
76
|
HistogramAddLiteral(&literal_histograms[context],
|
|
74
77
|
ringbuffer[pos & mask]);
|
|
75
78
|
prev_byte2 = prev_byte;
|
|
@@ -86,7 +89,7 @@ void BrotliBuildHistogramsWithContext(
|
|
|
86
89
|
context = (dist_it.type_ << BROTLI_DISTANCE_CONTEXT_BITS) +
|
|
87
90
|
CommandDistanceContext(cmd);
|
|
88
91
|
HistogramAddDistance(©_dist_histograms[context],
|
|
89
|
-
cmd->dist_prefix_);
|
|
92
|
+
cmd->dist_prefix_ & 0x3FF);
|
|
90
93
|
}
|
|
91
94
|
}
|
|
92
95
|
}
|
|
@@ -12,16 +12,19 @@
|
|
|
12
12
|
#include <string.h> /* memset */
|
|
13
13
|
|
|
14
14
|
#include "../common/constants.h"
|
|
15
|
+
#include "../common/context.h"
|
|
16
|
+
#include "../common/platform.h"
|
|
15
17
|
#include <brotli/types.h>
|
|
16
18
|
#include "./block_splitter.h"
|
|
17
19
|
#include "./command.h"
|
|
18
|
-
#include "./context.h"
|
|
19
|
-
#include "./port.h"
|
|
20
20
|
|
|
21
21
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
22
22
|
extern "C" {
|
|
23
23
|
#endif
|
|
24
24
|
|
|
25
|
+
/* The distance symbols effectively used by "Large Window Brotli" (32-bit). */
|
|
26
|
+
#define BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS 544
|
|
27
|
+
|
|
25
28
|
#define FN(X) X ## Literal
|
|
26
29
|
#define DATA_SIZE BROTLI_NUM_LITERAL_SYMBOLS
|
|
27
30
|
#define DataType uint8_t
|
|
@@ -38,7 +41,7 @@ extern "C" {
|
|
|
38
41
|
#undef FN
|
|
39
42
|
|
|
40
43
|
#define FN(X) X ## Distance
|
|
41
|
-
#define DATA_SIZE
|
|
44
|
+
#define DATA_SIZE BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS
|
|
42
45
|
#include "./histogram_inc.h" /* NOLINT(build/include) */
|
|
43
46
|
#undef DataType
|
|
44
47
|
#undef DATA_SIZE
|
|
@@ -33,7 +33,7 @@ static BROTLI_INLINE void FN(HistogramAdd)(FN(Histogram)* self, size_t val) {
|
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
static BROTLI_INLINE void FN(HistogramAddVector)(FN(Histogram)* self,
|
|
36
|
-
const DataType
|
|
36
|
+
const DataType* p, size_t n) {
|
|
37
37
|
self->total_count_ += n;
|
|
38
38
|
n += 1;
|
|
39
39
|
while (--n) ++self->data_[*p++];
|
|
@@ -9,9 +9,9 @@
|
|
|
9
9
|
|
|
10
10
|
#include "./literal_cost.h"
|
|
11
11
|
|
|
12
|
+
#include "../common/platform.h"
|
|
12
13
|
#include <brotli/types.h>
|
|
13
14
|
#include "./fast_log.h"
|
|
14
|
-
#include "./port.h"
|
|
15
15
|
#include "./utf8_util.h"
|
|
16
16
|
|
|
17
17
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
@@ -25,7 +25,7 @@ static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
|
|
|
25
25
|
return BROTLI_MIN(size_t, 1, clamp);
|
|
26
26
|
} else {
|
|
27
27
|
/* Let's decide over the last byte if this ends the sequence. */
|
|
28
|
-
if (last <
|
|
28
|
+
if (last < 0xE0) {
|
|
29
29
|
return 0; /* Completed two or three byte coding. */
|
|
30
30
|
} else { /* Next one is the 'Byte 3' of utf-8 encoding. */
|
|
31
31
|
return BROTLI_MIN(size_t, 2, clamp);
|
|
@@ -34,7 +34,7 @@ static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
|
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
|
|
37
|
-
const uint8_t
|
|
37
|
+
const uint8_t* data) {
|
|
38
38
|
size_t counts[3] = { 0 };
|
|
39
39
|
size_t max_utf8 = 1; /* should be 2, but 1 compresses better. */
|
|
40
40
|
size_t last_c = 0;
|
|
@@ -54,7 +54,7 @@ static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
|
57
|
-
const uint8_t
|
|
57
|
+
const uint8_t* data, float* cost) {
|
|
58
58
|
/* max_utf8 is 0 (normal ASCII single byte modeling),
|
|
59
59
|
1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
|
|
60
60
|
const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
|
|
@@ -125,7 +125,7 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
|
|
125
125
|
}
|
|
126
126
|
|
|
127
127
|
void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
|
|
128
|
-
const uint8_t
|
|
128
|
+
const uint8_t* data, float* cost) {
|
|
129
129
|
if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
|
|
130
130
|
EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost);
|
|
131
131
|
return;
|
|
@@ -10,8 +10,8 @@
|
|
|
10
10
|
#ifndef BROTLI_ENC_LITERAL_COST_H_
|
|
11
11
|
#define BROTLI_ENC_LITERAL_COST_H_
|
|
12
12
|
|
|
13
|
+
#include "../common/platform.h"
|
|
13
14
|
#include <brotli/types.h>
|
|
14
|
-
#include "./port.h"
|
|
15
15
|
|
|
16
16
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
17
17
|
extern "C" {
|
|
@@ -21,7 +21,7 @@ extern "C" {
|
|
|
21
21
|
ring-buffer (data, mask) will take entropy coded and writes these estimates
|
|
22
22
|
to the cost[0..len) array. */
|
|
23
23
|
BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals(
|
|
24
|
-
size_t pos, size_t len, size_t mask, const uint8_t
|
|
24
|
+
size_t pos, size_t len, size_t mask, const uint8_t* data, float* cost);
|
|
25
25
|
|
|
26
26
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
27
27
|
} /* extern "C" */
|
|
@@ -9,12 +9,11 @@
|
|
|
9
9
|
|
|
10
10
|
#include "./memory.h"
|
|
11
11
|
|
|
12
|
-
#include <assert.h>
|
|
13
12
|
#include <stdlib.h> /* exit, free, malloc */
|
|
14
13
|
#include <string.h> /* memcpy */
|
|
15
14
|
|
|
15
|
+
#include "../common/platform.h"
|
|
16
16
|
#include <brotli/types.h>
|
|
17
|
-
#include "./port.h"
|
|
18
17
|
|
|
19
18
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
20
19
|
extern "C" {
|
|
@@ -28,22 +27,12 @@ extern "C" {
|
|
|
28
27
|
#define NEW_ALLOCATED_OFFSET MAX_PERM_ALLOCATED
|
|
29
28
|
#define NEW_FREED_OFFSET (MAX_PERM_ALLOCATED + MAX_NEW_ALLOCATED)
|
|
30
29
|
|
|
31
|
-
static void* DefaultAllocFunc(void* opaque, size_t size) {
|
|
32
|
-
BROTLI_UNUSED(opaque);
|
|
33
|
-
return malloc(size);
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
static void DefaultFreeFunc(void* opaque, void* address) {
|
|
37
|
-
BROTLI_UNUSED(opaque);
|
|
38
|
-
free(address);
|
|
39
|
-
}
|
|
40
|
-
|
|
41
30
|
void BrotliInitMemoryManager(
|
|
42
31
|
MemoryManager* m, brotli_alloc_func alloc_func, brotli_free_func free_func,
|
|
43
32
|
void* opaque) {
|
|
44
33
|
if (!alloc_func) {
|
|
45
|
-
m->alloc_func =
|
|
46
|
-
m->free_func =
|
|
34
|
+
m->alloc_func = BrotliDefaultAllocFunc;
|
|
35
|
+
m->free_func = BrotliDefaultFreeFunc;
|
|
47
36
|
m->opaque = 0;
|
|
48
37
|
} else {
|
|
49
38
|
m->alloc_func = alloc_func;
|
|
@@ -132,11 +121,11 @@ static void CollectGarbagePointers(MemoryManager* m) {
|
|
|
132
121
|
m->pointers + NEW_FREED_OFFSET, m->new_freed);
|
|
133
122
|
m->perm_allocated -= annihilated;
|
|
134
123
|
m->new_freed -= annihilated;
|
|
135
|
-
|
|
124
|
+
BROTLI_DCHECK(m->new_freed == 0);
|
|
136
125
|
}
|
|
137
126
|
|
|
138
127
|
if (m->new_allocated != 0) {
|
|
139
|
-
|
|
128
|
+
BROTLI_DCHECK(m->perm_allocated + m->new_allocated <= MAX_PERM_ALLOCATED);
|
|
140
129
|
memcpy(m->pointers + PERM_ALLOCATED_OFFSET + m->perm_allocated,
|
|
141
130
|
m->pointers + NEW_ALLOCATED_OFFSET,
|
|
142
131
|
sizeof(void*) * m->new_allocated);
|
|
@@ -9,8 +9,10 @@
|
|
|
9
9
|
#ifndef BROTLI_ENC_MEMORY_H_
|
|
10
10
|
#define BROTLI_ENC_MEMORY_H_
|
|
11
11
|
|
|
12
|
+
#include <string.h> /* memcpy */
|
|
13
|
+
|
|
14
|
+
#include "../common/platform.h"
|
|
12
15
|
#include <brotli/types.h>
|
|
13
|
-
#include "./port.h"
|
|
14
16
|
|
|
15
17
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
16
18
|
extern "C" {
|
|
@@ -56,6 +58,43 @@ BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
|
|
|
56
58
|
|
|
57
59
|
BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m);
|
|
58
60
|
|
|
61
|
+
/*
|
|
62
|
+
Dynamically grows array capacity to at least the requested size
|
|
63
|
+
M: MemoryManager
|
|
64
|
+
T: data type
|
|
65
|
+
A: array
|
|
66
|
+
C: capacity
|
|
67
|
+
R: requested size
|
|
68
|
+
*/
|
|
69
|
+
#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \
|
|
70
|
+
if (C < (R)) { \
|
|
71
|
+
size_t _new_size = (C == 0) ? (R) : C; \
|
|
72
|
+
T* new_array; \
|
|
73
|
+
while (_new_size < (R)) _new_size *= 2; \
|
|
74
|
+
new_array = BROTLI_ALLOC((M), T, _new_size); \
|
|
75
|
+
if (!BROTLI_IS_OOM(M) && C != 0) \
|
|
76
|
+
memcpy(new_array, A, C * sizeof(T)); \
|
|
77
|
+
BROTLI_FREE((M), A); \
|
|
78
|
+
A = new_array; \
|
|
79
|
+
C = _new_size; \
|
|
80
|
+
} \
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/*
|
|
84
|
+
Appends value and dynamically grows array capacity when needed
|
|
85
|
+
M: MemoryManager
|
|
86
|
+
T: data type
|
|
87
|
+
A: array
|
|
88
|
+
C: array capacity
|
|
89
|
+
S: array size
|
|
90
|
+
V: value to append
|
|
91
|
+
*/
|
|
92
|
+
#define BROTLI_ENSURE_CAPACITY_APPEND(M, T, A, C, S, V) { \
|
|
93
|
+
(S)++; \
|
|
94
|
+
BROTLI_ENSURE_CAPACITY(M, T, A, C, S); \
|
|
95
|
+
A[(S) - 1] = (V); \
|
|
96
|
+
}
|
|
97
|
+
|
|
59
98
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
60
99
|
} /* extern "C" */
|
|
61
100
|
#endif
|
|
@@ -10,29 +10,131 @@
|
|
|
10
10
|
#include "./metablock.h"
|
|
11
11
|
|
|
12
12
|
#include "../common/constants.h"
|
|
13
|
+
#include "../common/context.h"
|
|
14
|
+
#include "../common/platform.h"
|
|
13
15
|
#include <brotli/types.h>
|
|
14
16
|
#include "./bit_cost.h"
|
|
15
17
|
#include "./block_splitter.h"
|
|
16
18
|
#include "./cluster.h"
|
|
17
|
-
#include "./context.h"
|
|
18
19
|
#include "./entropy_encode.h"
|
|
19
20
|
#include "./histogram.h"
|
|
20
21
|
#include "./memory.h"
|
|
21
|
-
#include "./port.h"
|
|
22
22
|
#include "./quality.h"
|
|
23
23
|
|
|
24
24
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
25
25
|
extern "C" {
|
|
26
26
|
#endif
|
|
27
27
|
|
|
28
|
+
void BrotliInitDistanceParams(BrotliEncoderParams* params,
|
|
29
|
+
uint32_t npostfix, uint32_t ndirect) {
|
|
30
|
+
BrotliDistanceParams* dist_params = ¶ms->dist;
|
|
31
|
+
uint32_t alphabet_size, max_distance;
|
|
32
|
+
|
|
33
|
+
dist_params->distance_postfix_bits = npostfix;
|
|
34
|
+
dist_params->num_direct_distance_codes = ndirect;
|
|
35
|
+
|
|
36
|
+
alphabet_size = BROTLI_DISTANCE_ALPHABET_SIZE(
|
|
37
|
+
npostfix, ndirect, BROTLI_MAX_DISTANCE_BITS);
|
|
38
|
+
max_distance = ndirect + (1U << (BROTLI_MAX_DISTANCE_BITS + npostfix + 2)) -
|
|
39
|
+
(1U << (npostfix + 2));
|
|
40
|
+
|
|
41
|
+
if (params->large_window) {
|
|
42
|
+
static const uint32_t bound[BROTLI_MAX_NPOSTFIX + 1] = {0, 4, 12, 28};
|
|
43
|
+
uint32_t postfix = 1U << npostfix;
|
|
44
|
+
alphabet_size = BROTLI_DISTANCE_ALPHABET_SIZE(
|
|
45
|
+
npostfix, ndirect, BROTLI_LARGE_MAX_DISTANCE_BITS);
|
|
46
|
+
/* The maximum distance is set so that no distance symbol used can encode
|
|
47
|
+
a distance larger than BROTLI_MAX_ALLOWED_DISTANCE with all
|
|
48
|
+
its extra bits set. */
|
|
49
|
+
if (ndirect < bound[npostfix]) {
|
|
50
|
+
max_distance = BROTLI_MAX_ALLOWED_DISTANCE - (bound[npostfix] - ndirect);
|
|
51
|
+
} else if (ndirect >= bound[npostfix] + postfix) {
|
|
52
|
+
max_distance = (3U << 29) - 4 + (ndirect - bound[npostfix]);
|
|
53
|
+
} else {
|
|
54
|
+
max_distance = BROTLI_MAX_ALLOWED_DISTANCE;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
dist_params->alphabet_size = alphabet_size;
|
|
59
|
+
dist_params->max_distance = max_distance;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
static void RecomputeDistancePrefixes(Command* cmds,
|
|
63
|
+
size_t num_commands,
|
|
64
|
+
const BrotliDistanceParams* orig_params,
|
|
65
|
+
const BrotliDistanceParams* new_params) {
|
|
66
|
+
size_t i;
|
|
67
|
+
|
|
68
|
+
if (orig_params->distance_postfix_bits == new_params->distance_postfix_bits &&
|
|
69
|
+
orig_params->num_direct_distance_codes ==
|
|
70
|
+
new_params->num_direct_distance_codes) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
for (i = 0; i < num_commands; ++i) {
|
|
75
|
+
Command* cmd = &cmds[i];
|
|
76
|
+
if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
|
|
77
|
+
PrefixEncodeCopyDistance(CommandRestoreDistanceCode(cmd, orig_params),
|
|
78
|
+
new_params->num_direct_distance_codes,
|
|
79
|
+
new_params->distance_postfix_bits,
|
|
80
|
+
&cmd->dist_prefix_,
|
|
81
|
+
&cmd->dist_extra_);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
static BROTLI_BOOL ComputeDistanceCost(const Command* cmds,
|
|
87
|
+
size_t num_commands,
|
|
88
|
+
const BrotliDistanceParams* orig_params,
|
|
89
|
+
const BrotliDistanceParams* new_params,
|
|
90
|
+
double* cost) {
|
|
91
|
+
size_t i;
|
|
92
|
+
BROTLI_BOOL equal_params = BROTLI_FALSE;
|
|
93
|
+
uint16_t dist_prefix;
|
|
94
|
+
uint32_t dist_extra;
|
|
95
|
+
double extra_bits = 0.0;
|
|
96
|
+
HistogramDistance histo;
|
|
97
|
+
HistogramClearDistance(&histo);
|
|
98
|
+
|
|
99
|
+
if (orig_params->distance_postfix_bits == new_params->distance_postfix_bits &&
|
|
100
|
+
orig_params->num_direct_distance_codes ==
|
|
101
|
+
new_params->num_direct_distance_codes) {
|
|
102
|
+
equal_params = BROTLI_TRUE;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
for (i = 0; i < num_commands; i++) {
|
|
106
|
+
const Command* cmd = &cmds[i];
|
|
107
|
+
if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
|
|
108
|
+
if (equal_params) {
|
|
109
|
+
dist_prefix = cmd->dist_prefix_;
|
|
110
|
+
} else {
|
|
111
|
+
uint32_t distance = CommandRestoreDistanceCode(cmd, orig_params);
|
|
112
|
+
if (distance > new_params->max_distance) {
|
|
113
|
+
return BROTLI_FALSE;
|
|
114
|
+
}
|
|
115
|
+
PrefixEncodeCopyDistance(distance,
|
|
116
|
+
new_params->num_direct_distance_codes,
|
|
117
|
+
new_params->distance_postfix_bits,
|
|
118
|
+
&dist_prefix,
|
|
119
|
+
&dist_extra);
|
|
120
|
+
}
|
|
121
|
+
HistogramAddDistance(&histo, dist_prefix & 0x3FF);
|
|
122
|
+
extra_bits += dist_prefix >> 10;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
*cost = BrotliPopulationCostDistance(&histo) + extra_bits;
|
|
127
|
+
return BROTLI_TRUE;
|
|
128
|
+
}
|
|
129
|
+
|
|
28
130
|
void BrotliBuildMetaBlock(MemoryManager* m,
|
|
29
131
|
const uint8_t* ringbuffer,
|
|
30
132
|
const size_t pos,
|
|
31
133
|
const size_t mask,
|
|
32
|
-
|
|
134
|
+
BrotliEncoderParams* params,
|
|
33
135
|
uint8_t prev_byte,
|
|
34
136
|
uint8_t prev_byte2,
|
|
35
|
-
|
|
137
|
+
Command* cmds,
|
|
36
138
|
size_t num_commands,
|
|
37
139
|
ContextType literal_context_mode,
|
|
38
140
|
MetaBlockSplit* mb) {
|
|
@@ -45,6 +147,46 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
|
45
147
|
size_t distance_histograms_size;
|
|
46
148
|
size_t i;
|
|
47
149
|
size_t literal_context_multiplier = 1;
|
|
150
|
+
uint32_t npostfix;
|
|
151
|
+
uint32_t ndirect_msb = 0;
|
|
152
|
+
BROTLI_BOOL check_orig = BROTLI_TRUE;
|
|
153
|
+
double best_dist_cost = 1e99;
|
|
154
|
+
BrotliEncoderParams orig_params = *params;
|
|
155
|
+
BrotliEncoderParams new_params = *params;
|
|
156
|
+
|
|
157
|
+
for (npostfix = 0; npostfix <= BROTLI_MAX_NPOSTFIX; npostfix++) {
|
|
158
|
+
for (; ndirect_msb < 16; ndirect_msb++) {
|
|
159
|
+
uint32_t ndirect = ndirect_msb << npostfix;
|
|
160
|
+
BROTLI_BOOL skip;
|
|
161
|
+
double dist_cost;
|
|
162
|
+
BrotliInitDistanceParams(&new_params, npostfix, ndirect);
|
|
163
|
+
if (npostfix == orig_params.dist.distance_postfix_bits &&
|
|
164
|
+
ndirect == orig_params.dist.num_direct_distance_codes) {
|
|
165
|
+
check_orig = BROTLI_FALSE;
|
|
166
|
+
}
|
|
167
|
+
skip = !ComputeDistanceCost(
|
|
168
|
+
cmds, num_commands,
|
|
169
|
+
&orig_params.dist, &new_params.dist, &dist_cost);
|
|
170
|
+
if (skip || (dist_cost > best_dist_cost)) {
|
|
171
|
+
break;
|
|
172
|
+
}
|
|
173
|
+
best_dist_cost = dist_cost;
|
|
174
|
+
params->dist = new_params.dist;
|
|
175
|
+
}
|
|
176
|
+
if (ndirect_msb > 0) ndirect_msb--;
|
|
177
|
+
ndirect_msb /= 2;
|
|
178
|
+
}
|
|
179
|
+
if (check_orig) {
|
|
180
|
+
double dist_cost;
|
|
181
|
+
ComputeDistanceCost(cmds, num_commands,
|
|
182
|
+
&orig_params.dist, &orig_params.dist, &dist_cost);
|
|
183
|
+
if (dist_cost < best_dist_cost) {
|
|
184
|
+
best_dist_cost = dist_cost;
|
|
185
|
+
params->dist = orig_params.dist;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
RecomputeDistancePrefixes(cmds, num_commands,
|
|
189
|
+
&orig_params.dist, ¶ms->dist);
|
|
48
190
|
|
|
49
191
|
BrotliSplitBlock(m, cmds, num_commands,
|
|
50
192
|
ringbuffer, pos, mask, params,
|
|
@@ -77,7 +219,7 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
|
77
219
|
if (BROTLI_IS_OOM(m)) return;
|
|
78
220
|
ClearHistogramsDistance(distance_histograms, distance_histograms_size);
|
|
79
221
|
|
|
80
|
-
|
|
222
|
+
BROTLI_DCHECK(mb->command_histograms == 0);
|
|
81
223
|
mb->command_histograms_size = mb->command_split.num_types;
|
|
82
224
|
mb->command_histograms =
|
|
83
225
|
BROTLI_ALLOC(m, HistogramCommand, mb->command_histograms_size);
|
|
@@ -90,14 +232,14 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
|
90
232
|
literal_histograms, mb->command_histograms, distance_histograms);
|
|
91
233
|
BROTLI_FREE(m, literal_context_modes);
|
|
92
234
|
|
|
93
|
-
|
|
235
|
+
BROTLI_DCHECK(mb->literal_context_map == 0);
|
|
94
236
|
mb->literal_context_map_size =
|
|
95
237
|
mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
|
|
96
238
|
mb->literal_context_map =
|
|
97
239
|
BROTLI_ALLOC(m, uint32_t, mb->literal_context_map_size);
|
|
98
240
|
if (BROTLI_IS_OOM(m)) return;
|
|
99
241
|
|
|
100
|
-
|
|
242
|
+
BROTLI_DCHECK(mb->literal_histograms == 0);
|
|
101
243
|
mb->literal_histograms_size = mb->literal_context_map_size;
|
|
102
244
|
mb->literal_histograms =
|
|
103
245
|
BROTLI_ALLOC(m, HistogramLiteral, mb->literal_histograms_size);
|
|
@@ -121,14 +263,14 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
|
121
263
|
}
|
|
122
264
|
}
|
|
123
265
|
|
|
124
|
-
|
|
266
|
+
BROTLI_DCHECK(mb->distance_context_map == 0);
|
|
125
267
|
mb->distance_context_map_size =
|
|
126
268
|
mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;
|
|
127
269
|
mb->distance_context_map =
|
|
128
270
|
BROTLI_ALLOC(m, uint32_t, mb->distance_context_map_size);
|
|
129
271
|
if (BROTLI_IS_OOM(m)) return;
|
|
130
272
|
|
|
131
|
-
|
|
273
|
+
BROTLI_DCHECK(mb->distance_histograms == 0);
|
|
132
274
|
mb->distance_histograms_size = mb->distance_context_map_size;
|
|
133
275
|
mb->distance_histograms =
|
|
134
276
|
BROTLI_ALLOC(m, HistogramDistance, mb->distance_histograms_size);
|
|
@@ -200,7 +342,7 @@ static void InitContextBlockSplitter(
|
|
|
200
342
|
size_t* histograms_size) {
|
|
201
343
|
size_t max_num_blocks = num_symbols / min_block_size + 1;
|
|
202
344
|
size_t max_num_types;
|
|
203
|
-
|
|
345
|
+
BROTLI_DCHECK(num_contexts <= BROTLI_MAX_STATIC_CONTEXTS);
|
|
204
346
|
|
|
205
347
|
self->alphabet_size_ = alphabet_size;
|
|
206
348
|
self->num_contexts_ = num_contexts;
|
|
@@ -226,7 +368,7 @@ static void InitContextBlockSplitter(
|
|
|
226
368
|
if (BROTLI_IS_OOM(m)) return;
|
|
227
369
|
split->num_blocks = max_num_blocks;
|
|
228
370
|
if (BROTLI_IS_OOM(m)) return;
|
|
229
|
-
|
|
371
|
+
BROTLI_DCHECK(*histograms == 0);
|
|
230
372
|
*histograms_size = max_num_types * num_contexts;
|
|
231
373
|
*histograms = BROTLI_ALLOC(m, HistogramLiteral, *histograms_size);
|
|
232
374
|
self->histograms_ = *histograms;
|
|
@@ -379,7 +521,7 @@ static void MapStaticContexts(MemoryManager* m,
|
|
|
379
521
|
const uint32_t* static_context_map,
|
|
380
522
|
MetaBlockSplit* mb) {
|
|
381
523
|
size_t i;
|
|
382
|
-
|
|
524
|
+
BROTLI_DCHECK(mb->literal_context_map == 0);
|
|
383
525
|
mb->literal_context_map_size =
|
|
384
526
|
mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
|
|
385
527
|
mb->literal_context_map =
|
|
@@ -398,9 +540,9 @@ static void MapStaticContexts(MemoryManager* m,
|
|
|
398
540
|
|
|
399
541
|
static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
|
|
400
542
|
MemoryManager* m, const uint8_t* ringbuffer, size_t pos, size_t mask,
|
|
401
|
-
uint8_t prev_byte, uint8_t prev_byte2,
|
|
543
|
+
uint8_t prev_byte, uint8_t prev_byte2, ContextLut literal_context_lut,
|
|
402
544
|
const size_t num_contexts, const uint32_t* static_context_map,
|
|
403
|
-
const Command
|
|
545
|
+
const Command* commands, size_t n_commands, MetaBlockSplit* mb) {
|
|
404
546
|
union {
|
|
405
547
|
BlockSplitterLiteral plain;
|
|
406
548
|
ContextBlockSplitter ctx;
|
|
@@ -441,7 +583,8 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
|
|
|
441
583
|
if (num_contexts == 1) {
|
|
442
584
|
BlockSplitterAddSymbolLiteral(&lit_blocks.plain, literal);
|
|
443
585
|
} else {
|
|
444
|
-
size_t context =
|
|
586
|
+
size_t context =
|
|
587
|
+
BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut);
|
|
445
588
|
ContextBlockSplitterAddSymbol(&lit_blocks.ctx, m, literal,
|
|
446
589
|
static_context_map[context]);
|
|
447
590
|
if (BROTLI_IS_OOM(m)) return;
|
|
@@ -455,7 +598,7 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
|
|
|
455
598
|
prev_byte2 = ringbuffer[(pos - 2) & mask];
|
|
456
599
|
prev_byte = ringbuffer[(pos - 1) & mask];
|
|
457
600
|
if (cmd.cmd_prefix_ >= 128) {
|
|
458
|
-
BlockSplitterAddSymbolDistance(&dist_blocks, cmd.dist_prefix_);
|
|
601
|
+
BlockSplitterAddSymbolDistance(&dist_blocks, cmd.dist_prefix_ & 0x3FF);
|
|
459
602
|
}
|
|
460
603
|
}
|
|
461
604
|
}
|
|
@@ -482,7 +625,7 @@ void BrotliBuildMetaBlockGreedy(MemoryManager* m,
|
|
|
482
625
|
size_t mask,
|
|
483
626
|
uint8_t prev_byte,
|
|
484
627
|
uint8_t prev_byte2,
|
|
485
|
-
|
|
628
|
+
ContextLut literal_context_lut,
|
|
486
629
|
size_t num_contexts,
|
|
487
630
|
const uint32_t* static_context_map,
|
|
488
631
|
const Command* commands,
|
|
@@ -490,19 +633,17 @@ void BrotliBuildMetaBlockGreedy(MemoryManager* m,
|
|
|
490
633
|
MetaBlockSplit* mb) {
|
|
491
634
|
if (num_contexts == 1) {
|
|
492
635
|
BrotliBuildMetaBlockGreedyInternal(m, ringbuffer, pos, mask, prev_byte,
|
|
493
|
-
prev_byte2,
|
|
636
|
+
prev_byte2, literal_context_lut, 1, NULL, commands, n_commands, mb);
|
|
494
637
|
} else {
|
|
495
638
|
BrotliBuildMetaBlockGreedyInternal(m, ringbuffer, pos, mask, prev_byte,
|
|
496
|
-
prev_byte2,
|
|
639
|
+
prev_byte2, literal_context_lut, num_contexts, static_context_map,
|
|
497
640
|
commands, n_commands, mb);
|
|
498
641
|
}
|
|
499
642
|
}
|
|
500
643
|
|
|
501
|
-
void BrotliOptimizeHistograms(
|
|
502
|
-
size_t distance_postfix_bits,
|
|
644
|
+
void BrotliOptimizeHistograms(uint32_t num_distance_codes,
|
|
503
645
|
MetaBlockSplit* mb) {
|
|
504
646
|
uint8_t good_for_rle[BROTLI_NUM_COMMAND_SYMBOLS];
|
|
505
|
-
size_t num_distance_codes;
|
|
506
647
|
size_t i;
|
|
507
648
|
for (i = 0; i < mb->literal_histograms_size; ++i) {
|
|
508
649
|
BrotliOptimizeHuffmanCountsForRle(256, mb->literal_histograms[i].data_,
|
|
@@ -513,9 +654,6 @@ void BrotliOptimizeHistograms(size_t num_direct_distance_codes,
|
|
|
513
654
|
mb->command_histograms[i].data_,
|
|
514
655
|
good_for_rle);
|
|
515
656
|
}
|
|
516
|
-
num_distance_codes = BROTLI_NUM_DISTANCE_SHORT_CODES +
|
|
517
|
-
num_direct_distance_codes +
|
|
518
|
-
((2 * BROTLI_MAX_DISTANCE_BITS) << distance_postfix_bits);
|
|
519
657
|
for (i = 0; i < mb->distance_histograms_size; ++i) {
|
|
520
658
|
BrotliOptimizeHuffmanCountsForRle(num_distance_codes,
|
|
521
659
|
mb->distance_histograms[i].data_,
|