brotli 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +3 -0
- data/.travis.yml +11 -3
- data/Gemfile +2 -0
- data/ext/brotli/brotli.c +279 -0
- data/ext/brotli/brotli.h +2 -0
- data/ext/brotli/buffer.c +95 -0
- data/ext/brotli/buffer.h +19 -0
- data/ext/brotli/extconf.rb +21 -81
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/dec/bit_reader.c +5 -5
- data/vendor/brotli/dec/bit_reader.h +15 -15
- data/vendor/brotli/dec/context.h +1 -1
- data/vendor/brotli/dec/decode.c +433 -348
- data/vendor/brotli/dec/decode.h +74 -48
- data/vendor/brotli/dec/huffman.c +5 -4
- data/vendor/brotli/dec/huffman.h +4 -4
- data/vendor/brotli/dec/port.h +2 -95
- data/vendor/brotli/dec/prefix.h +5 -3
- data/vendor/brotli/dec/state.c +15 -27
- data/vendor/brotli/dec/state.h +21 -17
- data/vendor/brotli/dec/transform.h +1 -1
- data/vendor/brotli/enc/backward_references.c +892 -0
- data/vendor/brotli/enc/backward_references.h +85 -102
- data/vendor/brotli/enc/backward_references_inc.h +147 -0
- data/vendor/brotli/enc/bit_cost.c +35 -0
- data/vendor/brotli/enc/bit_cost.h +23 -121
- data/vendor/brotli/enc/bit_cost_inc.h +127 -0
- data/vendor/brotli/enc/block_encoder_inc.h +33 -0
- data/vendor/brotli/enc/block_splitter.c +197 -0
- data/vendor/brotli/enc/block_splitter.h +40 -50
- data/vendor/brotli/enc/block_splitter_inc.h +432 -0
- data/vendor/brotli/enc/brotli_bit_stream.c +1334 -0
- data/vendor/brotli/enc/brotli_bit_stream.h +95 -167
- data/vendor/brotli/enc/cluster.c +56 -0
- data/vendor/brotli/enc/cluster.h +23 -305
- data/vendor/brotli/enc/cluster_inc.h +315 -0
- data/vendor/brotli/enc/command.h +83 -76
- data/vendor/brotli/enc/compress_fragment.c +747 -0
- data/vendor/brotli/enc/compress_fragment.h +48 -37
- data/vendor/brotli/enc/compress_fragment_two_pass.c +557 -0
- data/vendor/brotli/enc/compress_fragment_two_pass.h +37 -26
- data/vendor/brotli/enc/compressor.cc +139 -0
- data/vendor/brotli/enc/compressor.h +146 -0
- data/vendor/brotli/enc/context.h +102 -96
- data/vendor/brotli/enc/dictionary_hash.h +9 -5
- data/vendor/brotli/enc/encode.c +1562 -0
- data/vendor/brotli/enc/encode.h +211 -199
- data/vendor/brotli/enc/encode_parallel.cc +161 -151
- data/vendor/brotli/enc/encode_parallel.h +7 -8
- data/vendor/brotli/enc/entropy_encode.c +501 -0
- data/vendor/brotli/enc/entropy_encode.h +107 -89
- data/vendor/brotli/enc/entropy_encode_static.h +29 -62
- data/vendor/brotli/enc/fast_log.h +26 -20
- data/vendor/brotli/enc/find_match_length.h +23 -20
- data/vendor/brotli/enc/hash.h +614 -871
- data/vendor/brotli/enc/hash_forgetful_chain_inc.h +249 -0
- data/vendor/brotli/enc/hash_longest_match_inc.h +241 -0
- data/vendor/brotli/enc/hash_longest_match_quickly_inc.h +230 -0
- data/vendor/brotli/enc/histogram.c +95 -0
- data/vendor/brotli/enc/histogram.h +49 -83
- data/vendor/brotli/enc/histogram_inc.h +51 -0
- data/vendor/brotli/enc/literal_cost.c +178 -0
- data/vendor/brotli/enc/literal_cost.h +16 -10
- data/vendor/brotli/enc/memory.c +181 -0
- data/vendor/brotli/enc/memory.h +62 -0
- data/vendor/brotli/enc/metablock.c +515 -0
- data/vendor/brotli/enc/metablock.h +87 -57
- data/vendor/brotli/enc/metablock_inc.h +183 -0
- data/vendor/brotli/enc/port.h +73 -47
- data/vendor/brotli/enc/prefix.h +34 -61
- data/vendor/brotli/enc/quality.h +130 -0
- data/vendor/brotli/enc/ringbuffer.h +137 -122
- data/vendor/brotli/enc/{static_dict.cc → static_dict.c} +162 -139
- data/vendor/brotli/enc/static_dict.h +23 -18
- data/vendor/brotli/enc/static_dict_lut.h +11223 -12037
- data/vendor/brotli/enc/streams.cc +7 -7
- data/vendor/brotli/enc/streams.h +32 -32
- data/vendor/brotli/enc/{utf8_util.cc → utf8_util.c} +22 -20
- data/vendor/brotli/enc/utf8_util.h +16 -9
- data/vendor/brotli/enc/write_bits.h +49 -43
- metadata +34 -25
- data/ext/brotli/brotli.cc +0 -181
- data/vendor/brotli/dec/Makefile +0 -12
- data/vendor/brotli/dec/dictionary.c +0 -9466
- data/vendor/brotli/dec/dictionary.h +0 -38
- data/vendor/brotli/dec/types.h +0 -38
- data/vendor/brotli/enc/Makefile +0 -14
- data/vendor/brotli/enc/backward_references.cc +0 -858
- data/vendor/brotli/enc/block_splitter.cc +0 -505
- data/vendor/brotli/enc/brotli_bit_stream.cc +0 -1181
- data/vendor/brotli/enc/compress_fragment.cc +0 -701
- data/vendor/brotli/enc/compress_fragment_two_pass.cc +0 -524
- data/vendor/brotli/enc/dictionary.cc +0 -9466
- data/vendor/brotli/enc/dictionary.h +0 -41
- data/vendor/brotli/enc/encode.cc +0 -1180
- data/vendor/brotli/enc/entropy_encode.cc +0 -480
- data/vendor/brotli/enc/histogram.cc +0 -67
- data/vendor/brotli/enc/literal_cost.cc +0 -165
- data/vendor/brotli/enc/metablock.cc +0 -539
- data/vendor/brotli/enc/transform.h +0 -248
- data/vendor/brotli/enc/types.h +0 -29
@@ -0,0 +1,183 @@
|
|
1
|
+
/* NOLINT(build/header_guard) */
|
2
|
+
/* Copyright 2015 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Distributed under MIT license.
|
5
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
6
|
+
*/
|
7
|
+
|
8
|
+
/* template parameters: FN */
|
9
|
+
|
10
|
+
#define HistogramType FN(Histogram)
|
11
|
+
|
12
|
+
/* Greedy block splitter for one block category (literal, command or distance).
|
13
|
+
*/
|
14
|
+
typedef struct FN(BlockSplitter) {
|
15
|
+
/* Alphabet size of particular block category. */
|
16
|
+
size_t alphabet_size_;
|
17
|
+
/* We collect at least this many symbols for each block. */
|
18
|
+
size_t min_block_size_;
|
19
|
+
/* We merge histograms A and B if
|
20
|
+
entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,
|
21
|
+
where A is the current histogram and B is the histogram of the last or the
|
22
|
+
second last block type. */
|
23
|
+
double split_threshold_;
|
24
|
+
|
25
|
+
size_t num_blocks_;
|
26
|
+
BlockSplit* split_; /* not owned */
|
27
|
+
HistogramType* histograms_; /* not owned */
|
28
|
+
size_t* histograms_size_; /* not owned */
|
29
|
+
|
30
|
+
/* The number of symbols that we want to collect before deciding on whether
|
31
|
+
or not to merge the block with a previous one or emit a new block. */
|
32
|
+
size_t target_block_size_;
|
33
|
+
/* The number of symbols in the current histogram. */
|
34
|
+
size_t block_size_;
|
35
|
+
/* Offset of the current histogram. */
|
36
|
+
size_t curr_histogram_ix_;
|
37
|
+
/* Offset of the histograms of the previous two block types. */
|
38
|
+
size_t last_histogram_ix_[2];
|
39
|
+
/* Entropy of the previous two block types. */
|
40
|
+
double last_entropy_[2];
|
41
|
+
/* The number of times we merged the current block with the last one. */
|
42
|
+
size_t merge_last_count_;
|
43
|
+
} FN(BlockSplitter);
|
44
|
+
|
45
|
+
static void FN(InitBlockSplitter)(
|
46
|
+
MemoryManager* m, FN(BlockSplitter)* self, size_t alphabet_size,
|
47
|
+
size_t min_block_size, double split_threshold, size_t num_symbols,
|
48
|
+
BlockSplit* split, HistogramType** histograms, size_t* histograms_size) {
|
49
|
+
size_t max_num_blocks = num_symbols / min_block_size + 1;
|
50
|
+
/* We have to allocate one more histogram than the maximum number of block
|
51
|
+
types for the current histogram when the meta-block is too big. */
|
52
|
+
size_t max_num_types =
|
53
|
+
BROTLI_MIN(size_t, max_num_blocks, BROTLI_MAX_NUMBER_OF_BLOCK_TYPES + 1);
|
54
|
+
self->alphabet_size_ = alphabet_size;
|
55
|
+
self->min_block_size_ = min_block_size;
|
56
|
+
self->split_threshold_ = split_threshold;
|
57
|
+
self->num_blocks_ = 0;
|
58
|
+
self->split_ = split;
|
59
|
+
self->histograms_size_ = histograms_size;
|
60
|
+
self->target_block_size_ = min_block_size;
|
61
|
+
self->block_size_ = 0;
|
62
|
+
self->curr_histogram_ix_ = 0;
|
63
|
+
self->merge_last_count_ = 0;
|
64
|
+
BROTLI_ENSURE_CAPACITY(m, uint8_t,
|
65
|
+
split->types, split->types_alloc_size, max_num_blocks);
|
66
|
+
BROTLI_ENSURE_CAPACITY(m, uint32_t,
|
67
|
+
split->lengths, split->lengths_alloc_size, max_num_blocks);
|
68
|
+
if (BROTLI_IS_OOM(m)) return;
|
69
|
+
self->split_->num_blocks = max_num_blocks;
|
70
|
+
assert(*histograms == 0);
|
71
|
+
*histograms_size = max_num_types;
|
72
|
+
*histograms = BROTLI_ALLOC(m, HistogramType, *histograms_size);
|
73
|
+
self->histograms_ = *histograms;
|
74
|
+
if (BROTLI_IS_OOM(m)) return;
|
75
|
+
/* Clear only current histogram. */
|
76
|
+
FN(HistogramClear)(&self->histograms_[0]);
|
77
|
+
self->last_histogram_ix_[0] = self->last_histogram_ix_[1] = 0;
|
78
|
+
}
|
79
|
+
|
80
|
+
/* Does either of three things:
|
81
|
+
(1) emits the current block with a new block type;
|
82
|
+
(2) emits the current block with the type of the second last block;
|
83
|
+
(3) merges the current block with the last block. */
|
84
|
+
static void FN(BlockSplitterFinishBlock)(
|
85
|
+
FN(BlockSplitter)* self, BROTLI_BOOL is_final) {
|
86
|
+
BlockSplit* split = self->split_;
|
87
|
+
double* last_entropy = self->last_entropy_;
|
88
|
+
HistogramType* histograms = self->histograms_;
|
89
|
+
self->block_size_ =
|
90
|
+
BROTLI_MAX(size_t, self->block_size_, self->min_block_size_);
|
91
|
+
if (self->num_blocks_ == 0) {
|
92
|
+
/* Create first block. */
|
93
|
+
split->lengths[0] = (uint32_t)self->block_size_;
|
94
|
+
split->types[0] = 0;
|
95
|
+
last_entropy[0] =
|
96
|
+
BitsEntropy(histograms[0].data_, self->alphabet_size_);
|
97
|
+
last_entropy[1] = last_entropy[0];
|
98
|
+
++self->num_blocks_;
|
99
|
+
++split->num_types;
|
100
|
+
++self->curr_histogram_ix_;
|
101
|
+
if (self->curr_histogram_ix_ < *self->histograms_size_)
|
102
|
+
FN(HistogramClear)(&histograms[self->curr_histogram_ix_]);
|
103
|
+
self->block_size_ = 0;
|
104
|
+
} else if (self->block_size_ > 0) {
|
105
|
+
double entropy = BitsEntropy(histograms[self->curr_histogram_ix_].data_,
|
106
|
+
self->alphabet_size_);
|
107
|
+
HistogramType combined_histo[2];
|
108
|
+
double combined_entropy[2];
|
109
|
+
double diff[2];
|
110
|
+
size_t j;
|
111
|
+
for (j = 0; j < 2; ++j) {
|
112
|
+
size_t last_histogram_ix = self->last_histogram_ix_[j];
|
113
|
+
combined_histo[j] = histograms[self->curr_histogram_ix_];
|
114
|
+
FN(HistogramAddHistogram)(&combined_histo[j],
|
115
|
+
&histograms[last_histogram_ix]);
|
116
|
+
combined_entropy[j] = BitsEntropy(
|
117
|
+
&combined_histo[j].data_[0], self->alphabet_size_);
|
118
|
+
diff[j] = combined_entropy[j] - entropy - last_entropy[j];
|
119
|
+
}
|
120
|
+
|
121
|
+
if (split->num_types < BROTLI_MAX_NUMBER_OF_BLOCK_TYPES &&
|
122
|
+
diff[0] > self->split_threshold_ &&
|
123
|
+
diff[1] > self->split_threshold_) {
|
124
|
+
/* Create new block. */
|
125
|
+
split->lengths[self->num_blocks_] = (uint32_t)self->block_size_;
|
126
|
+
split->types[self->num_blocks_] = (uint8_t)split->num_types;
|
127
|
+
self->last_histogram_ix_[1] = self->last_histogram_ix_[0];
|
128
|
+
self->last_histogram_ix_[0] = (uint8_t)split->num_types;
|
129
|
+
last_entropy[1] = last_entropy[0];
|
130
|
+
last_entropy[0] = entropy;
|
131
|
+
++self->num_blocks_;
|
132
|
+
++split->num_types;
|
133
|
+
++self->curr_histogram_ix_;
|
134
|
+
if (self->curr_histogram_ix_ < *self->histograms_size_)
|
135
|
+
FN(HistogramClear)(&histograms[self->curr_histogram_ix_]);
|
136
|
+
self->block_size_ = 0;
|
137
|
+
self->merge_last_count_ = 0;
|
138
|
+
self->target_block_size_ = self->min_block_size_;
|
139
|
+
} else if (diff[1] < diff[0] - 20.0) {
|
140
|
+
/* Combine this block with second last block. */
|
141
|
+
split->lengths[self->num_blocks_] = (uint32_t)self->block_size_;
|
142
|
+
split->types[self->num_blocks_] = split->types[self->num_blocks_ - 2];
|
143
|
+
BROTLI_SWAP(size_t, self->last_histogram_ix_, 0, 1);
|
144
|
+
histograms[self->last_histogram_ix_[0]] = combined_histo[1];
|
145
|
+
last_entropy[1] = last_entropy[0];
|
146
|
+
last_entropy[0] = combined_entropy[1];
|
147
|
+
++self->num_blocks_;
|
148
|
+
self->block_size_ = 0;
|
149
|
+
FN(HistogramClear)(&histograms[self->curr_histogram_ix_]);
|
150
|
+
self->merge_last_count_ = 0;
|
151
|
+
self->target_block_size_ = self->min_block_size_;
|
152
|
+
} else {
|
153
|
+
/* Combine this block with last block. */
|
154
|
+
split->lengths[self->num_blocks_ - 1] += (uint32_t)self->block_size_;
|
155
|
+
histograms[self->last_histogram_ix_[0]] = combined_histo[0];
|
156
|
+
last_entropy[0] = combined_entropy[0];
|
157
|
+
if (split->num_types == 1) {
|
158
|
+
last_entropy[1] = last_entropy[0];
|
159
|
+
}
|
160
|
+
self->block_size_ = 0;
|
161
|
+
FN(HistogramClear)(&histograms[self->curr_histogram_ix_]);
|
162
|
+
if (++self->merge_last_count_ > 1) {
|
163
|
+
self->target_block_size_ += self->min_block_size_;
|
164
|
+
}
|
165
|
+
}
|
166
|
+
}
|
167
|
+
if (is_final) {
|
168
|
+
*self->histograms_size_ = split->num_types;
|
169
|
+
split->num_blocks = self->num_blocks_;
|
170
|
+
}
|
171
|
+
}
|
172
|
+
|
173
|
+
/* Adds the next symbol to the current histogram. When the current histogram
|
174
|
+
reaches the target size, decides on merging the block. */
|
175
|
+
static void FN(BlockSplitterAddSymbol)(FN(BlockSplitter)* self, size_t symbol) {
|
176
|
+
FN(HistogramAdd)(&self->histograms_[self->curr_histogram_ix_], symbol);
|
177
|
+
++self->block_size_;
|
178
|
+
if (self->block_size_ == self->target_block_size_) {
|
179
|
+
FN(BlockSplitterFinishBlock)(self, /* is_final = */ BROTLI_FALSE);
|
180
|
+
}
|
181
|
+
}
|
182
|
+
|
183
|
+
#undef HistogramType
|
data/vendor/brotli/enc/port.h
CHANGED
@@ -4,14 +4,16 @@
|
|
4
4
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
5
|
*/
|
6
6
|
|
7
|
-
|
7
|
+
/* Macros for endianness, branch prediction and unaligned loads and stores. */
|
8
8
|
|
9
9
|
#ifndef BROTLI_ENC_PORT_H_
|
10
10
|
#define BROTLI_ENC_PORT_H_
|
11
11
|
|
12
12
|
#include <assert.h>
|
13
|
-
#include <string.h>
|
14
|
-
|
13
|
+
#include <string.h> /* memcpy */
|
14
|
+
|
15
|
+
#include "../common/port.h"
|
16
|
+
#include "../common/types.h"
|
15
17
|
|
16
18
|
#if defined OS_LINUX || defined OS_CYGWIN
|
17
19
|
#include <endian.h>
|
@@ -24,9 +26,9 @@
|
|
24
26
|
#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
25
27
|
#endif
|
26
28
|
|
27
|
-
|
28
|
-
|
29
|
-
|
29
|
+
/* define the macro IS_LITTLE_ENDIAN
|
30
|
+
using the above endian definitions from endian.h if
|
31
|
+
endian.h was included */
|
30
32
|
#ifdef __BYTE_ORDER
|
31
33
|
#if __BYTE_ORDER == __LITTLE_ENDIAN
|
32
34
|
#define IS_LITTLE_ENDIAN
|
@@ -37,49 +39,36 @@
|
|
37
39
|
#if defined(__LITTLE_ENDIAN__)
|
38
40
|
#define IS_LITTLE_ENDIAN
|
39
41
|
#endif
|
40
|
-
#endif
|
42
|
+
#endif /* __BYTE_ORDER */
|
41
43
|
|
42
44
|
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
43
45
|
#define IS_LITTLE_ENDIAN
|
44
46
|
#endif
|
45
47
|
|
46
|
-
|
48
|
+
/* Enable little-endian optimization for x64 architecture on Windows. */
|
47
49
|
#if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
|
48
50
|
#define IS_LITTLE_ENDIAN
|
49
51
|
#endif
|
50
52
|
|
51
|
-
/*
|
52
|
-
|
53
|
-
|
54
|
-
#endif
|
55
|
-
|
56
|
-
#if (__GNUC__ > 2) || (__GNUC__ == 2 && __GNUC_MINOR__ > 95) || \
|
57
|
-
(defined(__llvm__) && __has_builtin(__builtin_expect))
|
58
|
-
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
|
59
|
-
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
|
60
|
-
#else
|
61
|
-
#define PREDICT_FALSE(x) (x)
|
62
|
-
#define PREDICT_TRUE(x) (x)
|
63
|
-
#endif
|
64
|
-
|
65
|
-
// Portable handling of unaligned loads, stores, and copies.
|
66
|
-
// On some platforms, like ARM, the copy functions can be more efficient
|
67
|
-
// then a load and a store.
|
53
|
+
/* Portable handling of unaligned loads, stores, and copies.
|
54
|
+
On some platforms, like ARM, the copy functions can be more efficient
|
55
|
+
then a load and a store. */
|
68
56
|
|
69
57
|
#if defined(ARCH_PIII) || \
|
70
58
|
defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC)
|
71
59
|
|
72
|
-
|
73
|
-
|
74
|
-
|
60
|
+
/* x86 and x86-64 can perform unaligned loads/stores directly;
|
61
|
+
modern PowerPC hardware can also do unaligned integer loads and stores;
|
62
|
+
but note: the FPU still sends unaligned loads and stores to a trap handler!
|
63
|
+
*/
|
75
64
|
|
76
|
-
#define BROTLI_UNALIGNED_LOAD32(_p) (*
|
77
|
-
#define BROTLI_UNALIGNED_LOAD64(_p) (*
|
65
|
+
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
|
66
|
+
#define BROTLI_UNALIGNED_LOAD64(_p) (*(const uint64_t *)(_p))
|
78
67
|
|
79
68
|
#define BROTLI_UNALIGNED_STORE32(_p, _val) \
|
80
|
-
(*
|
69
|
+
(*(uint32_t *)(_p) = (_val))
|
81
70
|
#define BROTLI_UNALIGNED_STORE64(_p, _val) \
|
82
|
-
(*
|
71
|
+
(*(uint64_t *)(_p) = (_val))
|
83
72
|
|
84
73
|
#elif defined(__arm__) && \
|
85
74
|
!defined(__ARM_ARCH_5__) && \
|
@@ -93,50 +82,87 @@
|
|
93
82
|
!defined(__ARM_ARCH_6ZK__) && \
|
94
83
|
!defined(__ARM_ARCH_6T2__)
|
95
84
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
85
|
+
/* ARMv7 and newer support native unaligned accesses, but only of 16-bit
|
86
|
+
and 32-bit values (not 64-bit); older versions either raise a fatal signal,
|
87
|
+
do an unaligned read and rotate the words around a bit, or do the reads very
|
88
|
+
slowly (trip through kernel mode). */
|
100
89
|
|
101
|
-
#define BROTLI_UNALIGNED_LOAD32(_p) (*
|
90
|
+
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
|
102
91
|
#define BROTLI_UNALIGNED_STORE32(_p, _val) \
|
103
|
-
(*
|
92
|
+
(*(uint32_t *)(_p) = (_val))
|
104
93
|
|
105
|
-
|
94
|
+
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
|
106
95
|
uint64_t t;
|
107
96
|
memcpy(&t, p, sizeof t);
|
108
97
|
return t;
|
109
98
|
}
|
110
99
|
|
111
|
-
|
100
|
+
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
|
112
101
|
memcpy(p, &v, sizeof v);
|
113
102
|
}
|
114
103
|
|
115
104
|
#else
|
116
105
|
|
117
|
-
|
118
|
-
|
106
|
+
/* These functions are provided for architectures that don't support */
|
107
|
+
/* unaligned loads and stores. */
|
119
108
|
|
120
|
-
|
109
|
+
static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) {
|
121
110
|
uint32_t t;
|
122
111
|
memcpy(&t, p, sizeof t);
|
123
112
|
return t;
|
124
113
|
}
|
125
114
|
|
126
|
-
|
115
|
+
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
|
127
116
|
uint64_t t;
|
128
117
|
memcpy(&t, p, sizeof t);
|
129
118
|
return t;
|
130
119
|
}
|
131
120
|
|
132
|
-
|
121
|
+
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE32(void *p, uint32_t v) {
|
133
122
|
memcpy(p, &v, sizeof v);
|
134
123
|
}
|
135
124
|
|
136
|
-
|
125
|
+
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
|
137
126
|
memcpy(p, &v, sizeof v);
|
138
127
|
}
|
139
128
|
|
140
129
|
#endif
|
141
130
|
|
142
|
-
#
|
131
|
+
#if !defined(__cplusplus) && !defined(c_plusplus) && __STDC_VERSION__ >= 199901L
|
132
|
+
#define BROTLI_RESTRICT restrict
|
133
|
+
#elif BROTLI_GCC_VERSION > 295 || defined(__llvm__)
|
134
|
+
#define BROTLI_RESTRICT __restrict
|
135
|
+
#else
|
136
|
+
#define BROTLI_RESTRICT
|
137
|
+
#endif
|
138
|
+
|
139
|
+
#define _TEMPLATE(T) \
|
140
|
+
static BROTLI_INLINE T brotli_min_ ## T (T a, T b) { return a < b ? a : b; } \
|
141
|
+
static BROTLI_INLINE T brotli_max_ ## T (T a, T b) { return a > b ? a : b; }
|
142
|
+
_TEMPLATE(double) _TEMPLATE(float) _TEMPLATE(int)
|
143
|
+
_TEMPLATE(size_t) _TEMPLATE(uint32_t) _TEMPLATE(uint8_t)
|
144
|
+
#undef _TEMPLATE
|
145
|
+
#define BROTLI_MIN(T, A, B) (brotli_min_ ## T((A), (B)))
|
146
|
+
#define BROTLI_MAX(T, A, B) (brotli_max_ ## T((A), (B)))
|
147
|
+
|
148
|
+
#define BROTLI_SWAP(T, A, I, J) { \
|
149
|
+
T __brotli_swap_tmp = (A)[(I)]; \
|
150
|
+
(A)[(I)] = (A)[(J)]; \
|
151
|
+
(A)[(J)] = __brotli_swap_tmp; \
|
152
|
+
}
|
153
|
+
|
154
|
+
#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \
|
155
|
+
if (C < (R)) { \
|
156
|
+
size_t _new_size = (C == 0) ? (R) : C; \
|
157
|
+
T* new_array; \
|
158
|
+
while (_new_size < (R)) _new_size *= 2; \
|
159
|
+
new_array = BROTLI_ALLOC((M), T, _new_size); \
|
160
|
+
if (!BROTLI_IS_OOM(m) && C != 0) \
|
161
|
+
memcpy(new_array, A, C * sizeof(T)); \
|
162
|
+
BROTLI_FREE((M), A); \
|
163
|
+
A = new_array; \
|
164
|
+
C = _new_size; \
|
165
|
+
} \
|
166
|
+
}
|
167
|
+
|
168
|
+
#endif /* BROTLI_ENC_PORT_H_ */
|
data/vendor/brotli/enc/prefix.h
CHANGED
@@ -4,76 +4,49 @@
|
|
4
4
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
5
|
*/
|
6
6
|
|
7
|
-
|
8
|
-
|
7
|
+
/* Functions for encoding of integers into prefix codes the amount of extra
|
8
|
+
bits, and the actual values of the extra bits. */
|
9
9
|
|
10
10
|
#ifndef BROTLI_ENC_PREFIX_H_
|
11
11
|
#define BROTLI_ENC_PREFIX_H_
|
12
12
|
|
13
|
+
#include "../common/constants.h"
|
14
|
+
#include "../common/port.h"
|
15
|
+
#include "../common/types.h"
|
13
16
|
#include "./fast_log.h"
|
14
|
-
#include "./types.h"
|
15
17
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
static
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
struct PrefixCodeRange {
|
28
|
-
uint32_t offset;
|
29
|
-
uint32_t nbits;
|
30
|
-
};
|
31
|
-
|
32
|
-
static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
|
33
|
-
{ 1, 2}, { 5, 2}, { 9, 2}, { 13, 2},
|
34
|
-
{ 17, 3}, { 25, 3}, { 33, 3}, { 41, 3},
|
35
|
-
{ 49, 4}, { 65, 4}, { 81, 4}, { 97, 4},
|
36
|
-
{ 113, 5}, { 145, 5}, { 177, 5}, { 209, 5},
|
37
|
-
{ 241, 6}, { 305, 6}, { 369, 7}, { 497, 8},
|
38
|
-
{ 753, 9}, { 1265, 10}, {2289, 11}, {4337, 12},
|
39
|
-
{8433, 13}, {16625, 24}
|
40
|
-
};
|
41
|
-
|
42
|
-
inline void GetBlockLengthPrefixCode(uint32_t len, uint32_t* code,
|
43
|
-
uint32_t* n_extra, uint32_t* extra) {
|
44
|
-
*code = 0;
|
45
|
-
while (*code < 25 && len >= kBlockLengthPrefixCode[*code + 1].offset) {
|
46
|
-
++(*code);
|
47
|
-
}
|
48
|
-
*n_extra = kBlockLengthPrefixCode[*code].nbits;
|
49
|
-
*extra = len - kBlockLengthPrefixCode[*code].offset;
|
50
|
-
}
|
51
|
-
|
52
|
-
inline void PrefixEncodeCopyDistance(size_t distance_code,
|
53
|
-
size_t num_direct_codes,
|
54
|
-
size_t postfix_bits,
|
55
|
-
uint16_t* code,
|
56
|
-
uint32_t* extra_bits) {
|
57
|
-
if (distance_code < kNumDistanceShortCodes + num_direct_codes) {
|
58
|
-
*code = static_cast<uint16_t>(distance_code);
|
18
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
19
|
+
extern "C" {
|
20
|
+
#endif
|
21
|
+
|
22
|
+
static BROTLI_INLINE void PrefixEncodeCopyDistance(size_t distance_code,
|
23
|
+
size_t num_direct_codes,
|
24
|
+
size_t postfix_bits,
|
25
|
+
uint16_t* code,
|
26
|
+
uint32_t* extra_bits) {
|
27
|
+
if (distance_code < BROTLI_NUM_DISTANCE_SHORT_CODES + num_direct_codes) {
|
28
|
+
*code = (uint16_t)distance_code;
|
59
29
|
*extra_bits = 0;
|
60
30
|
return;
|
31
|
+
} else {
|
32
|
+
size_t dist = ((size_t)1 << (postfix_bits + 2u)) +
|
33
|
+
(distance_code - BROTLI_NUM_DISTANCE_SHORT_CODES - num_direct_codes);
|
34
|
+
size_t bucket = Log2FloorNonZero(dist) - 1;
|
35
|
+
size_t postfix_mask = (1u << postfix_bits) - 1;
|
36
|
+
size_t postfix = dist & postfix_mask;
|
37
|
+
size_t prefix = (dist >> bucket) & 1;
|
38
|
+
size_t offset = (2 + prefix) << bucket;
|
39
|
+
size_t nbits = bucket - postfix_bits;
|
40
|
+
*code = (uint16_t)(
|
41
|
+
(BROTLI_NUM_DISTANCE_SHORT_CODES + num_direct_codes +
|
42
|
+
((2 * (nbits - 1) + prefix) << postfix_bits) + postfix));
|
43
|
+
*extra_bits = (uint32_t)(
|
44
|
+
(nbits << 24) | ((dist - offset) >> postfix_bits));
|
61
45
|
}
|
62
|
-
distance_code -= kNumDistanceShortCodes + num_direct_codes; /* >= 0 */
|
63
|
-
distance_code += (1u << (postfix_bits + 2u)); /* > 0 */
|
64
|
-
size_t bucket = Log2FloorNonZero(distance_code) - 1;
|
65
|
-
size_t postfix_mask = (1 << postfix_bits) - 1;
|
66
|
-
size_t postfix = distance_code & postfix_mask;
|
67
|
-
size_t prefix = (distance_code >> bucket) & 1;
|
68
|
-
size_t offset = (2 + prefix) << bucket;
|
69
|
-
size_t nbits = bucket - postfix_bits;
|
70
|
-
*code = static_cast<uint16_t>(
|
71
|
-
(kNumDistanceShortCodes + num_direct_codes +
|
72
|
-
((2 * (nbits - 1) + prefix) << postfix_bits) + postfix));
|
73
|
-
*extra_bits = static_cast<uint32_t>(
|
74
|
-
(nbits << 24) | ((distance_code - offset) >> postfix_bits));
|
75
46
|
}
|
76
47
|
|
77
|
-
|
48
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
49
|
+
} /* extern "C" */
|
50
|
+
#endif
|
78
51
|
|
79
|
-
#endif
|
52
|
+
#endif /* BROTLI_ENC_PREFIX_H_ */
|
@@ -0,0 +1,130 @@
|
|
1
|
+
/* Copyright 2016 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
/* Constants and formulas that affect speed-ratio trade-offs and thus define
|
8
|
+
quality levels. */
|
9
|
+
|
10
|
+
#ifndef BROTLI_ENC_QUALITY_H_
|
11
|
+
#define BROTLI_ENC_QUALITY_H_
|
12
|
+
|
13
|
+
#include "./encode.h"
|
14
|
+
|
15
|
+
#define FAST_ONE_PASS_COMPRESSION_QUALITY 0
|
16
|
+
#define FAST_TWO_PASS_COMPRESSION_QUALITY 1
|
17
|
+
#define ZOPFLIFICATION_QUALITY 10
|
18
|
+
#define HQ_ZOPFLIFICATION_QUALITY 11
|
19
|
+
|
20
|
+
#define MAX_QUALITY_FOR_STATIC_ENRTOPY_CODES 2
|
21
|
+
#define MIN_QUALITY_FOR_BLOCK_SPLIT 4
|
22
|
+
#define MIN_QUALITY_FOR_OPTIMIZE_HISTOGRAMS 4
|
23
|
+
#define MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH 5
|
24
|
+
#define MIN_QUALITY_FOR_CONTEXT_MODELING 5
|
25
|
+
#define MIN_QUALITY_FOR_HQ_CONTEXT_MODELING 7
|
26
|
+
#define MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING 10
|
27
|
+
/* Only for "font" mode. */
|
28
|
+
#define MIN_QUALITY_FOR_RECOMPUTE_DISTANCE_PREFIXES 10
|
29
|
+
|
30
|
+
/* For quality below MIN_QUALITY_FOR_BLOCK_SPLIT there is no block splitting,
|
31
|
+
so we buffer at most this much literals and commands. */
|
32
|
+
#define MAX_NUM_DELAYED_SYMBOLS 0x2fff
|
33
|
+
|
34
|
+
/* Encoding parameters */
|
35
|
+
typedef struct BrotliEncoderParams {
|
36
|
+
BrotliEncoderMode mode;
|
37
|
+
int quality;
|
38
|
+
int lgwin;
|
39
|
+
int lgblock;
|
40
|
+
} BrotliEncoderParams;
|
41
|
+
|
42
|
+
/* Returns hashtable size for quality levels 0 and 1. */
|
43
|
+
static BROTLI_INLINE size_t MaxHashTableSize(int quality) {
|
44
|
+
return quality == FAST_ONE_PASS_COMPRESSION_QUALITY ? 1 << 15 : 1 << 17;
|
45
|
+
}
|
46
|
+
|
47
|
+
/* The maximum length for which the zopflification uses distinct distances. */
|
48
|
+
#define MAX_ZOPFLI_LEN_QUALITY_10 150
|
49
|
+
#define MAX_ZOPFLI_LEN_QUALITY_11 325
|
50
|
+
|
51
|
+
static BROTLI_INLINE size_t MaxZopfliLen(const BrotliEncoderParams* params) {
|
52
|
+
return params->quality <= 10 ?
|
53
|
+
MAX_ZOPFLI_LEN_QUALITY_10 :
|
54
|
+
MAX_ZOPFLI_LEN_QUALITY_11;
|
55
|
+
}
|
56
|
+
|
57
|
+
/* Number of best candidates to evaluate to expand zopfli chain. */
|
58
|
+
static BROTLI_INLINE size_t MaxZopfliCandidates(
|
59
|
+
const BrotliEncoderParams* params) {
|
60
|
+
return params->quality <= 10 ? 1 : 5;
|
61
|
+
}
|
62
|
+
|
63
|
+
static BROTLI_INLINE void SanitizeParams(BrotliEncoderParams* params) {
|
64
|
+
params->quality = BROTLI_MIN(int, BROTLI_MAX_QUALITY,
|
65
|
+
BROTLI_MAX(int, BROTLI_MIN_QUALITY, params->quality));
|
66
|
+
if (params->lgwin < kBrotliMinWindowBits) {
|
67
|
+
params->lgwin = kBrotliMinWindowBits;
|
68
|
+
} else if (params->lgwin > kBrotliMaxWindowBits) {
|
69
|
+
params->lgwin = kBrotliMaxWindowBits;
|
70
|
+
}
|
71
|
+
}
|
72
|
+
|
73
|
+
/* Returns optimized lg_block value. */
|
74
|
+
static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
|
75
|
+
int lgblock = params->lgblock;
|
76
|
+
if (params->quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
|
77
|
+
params->quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
|
78
|
+
lgblock = params->lgwin;
|
79
|
+
} else if (params->quality < MIN_QUALITY_FOR_BLOCK_SPLIT) {
|
80
|
+
lgblock = 14;
|
81
|
+
} else if (lgblock == 0) {
|
82
|
+
lgblock = 16;
|
83
|
+
if (params->quality >= 9 && params->lgwin > lgblock) {
|
84
|
+
lgblock = BROTLI_MIN(int, 18, params->lgwin);
|
85
|
+
}
|
86
|
+
} else {
|
87
|
+
lgblock = BROTLI_MIN(int, kBrotliMaxInputBlockBits,
|
88
|
+
BROTLI_MAX(int, kBrotliMinInputBlockBits, lgblock));
|
89
|
+
}
|
90
|
+
return lgblock;
|
91
|
+
}
|
92
|
+
|
93
|
+
/* Returns log2 of the size of main ring buffer area.
|
94
|
+
Allocate at least lgwin + 1 bits for the ring buffer so that the newly
|
95
|
+
added block fits there completely and we still get lgwin bits and at least
|
96
|
+
read_block_size_bits + 1 bits because the copy tail length needs to be
|
97
|
+
smaller than ringbuffer size. */
|
98
|
+
static BROTLI_INLINE int ComputeRbBits(const BrotliEncoderParams* params) {
|
99
|
+
return 1 + BROTLI_MAX(int, params->lgwin, params->lgblock);
|
100
|
+
}
|
101
|
+
|
102
|
+
static BROTLI_INLINE size_t MaxMetablockSize(
|
103
|
+
const BrotliEncoderParams* params) {
|
104
|
+
int bits = BROTLI_MIN(int, ComputeRbBits(params), kBrotliMaxInputBlockBits);
|
105
|
+
return (size_t)1 << bits;
|
106
|
+
}
|
107
|
+
|
108
|
+
/* When searching for backward references and have not seen matches for a long
|
109
|
+
time, we can skip some match lookups. Unsuccessful match lookups are very
|
110
|
+
expensive and this kind of a heuristic speeds up compression quite a lot.
|
111
|
+
At first 8 byte strides are taken and every second byte is put to hasher.
|
112
|
+
After 4x more literals stride by 16 bytes, every put 4-th byte to hasher.
|
113
|
+
Applied only to qualities 2 to 9. */
|
114
|
+
static BROTLI_INLINE size_t LiteralSpreeLengthForSparseSearch(
|
115
|
+
const BrotliEncoderParams* params) {
|
116
|
+
return params->quality < 9 ? 64 : 512;
|
117
|
+
}
|
118
|
+
|
119
|
+
static BROTLI_INLINE int ChooseHasher(const BrotliEncoderParams* params) {
|
120
|
+
if (params->quality > 9) {
|
121
|
+
return 10;
|
122
|
+
} else if (params->quality < 5) {
|
123
|
+
return params->quality;
|
124
|
+
} else if (params->lgwin <= 16) {
|
125
|
+
return params->quality < 7 ? 40 : params->quality < 9 ? 41 : 42;
|
126
|
+
}
|
127
|
+
return params->quality;
|
128
|
+
}
|
129
|
+
|
130
|
+
#endif /* BROTLI_ENC_QUALITY_H_ */
|