brotli 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitmodules +1 -1
- data/.travis.yml +2 -1
- data/README.md +1 -1
- data/Rakefile +1 -1
- data/ext/brotli/brotli.cc +1 -1
- data/ext/brotli/extconf.rb +72 -14
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/LICENSE +19 -202
- data/vendor/brotli/dec/Makefile +1 -1
- data/vendor/brotli/dec/bit_reader.c +23 -30
- data/vendor/brotli/dec/bit_reader.h +270 -141
- data/vendor/brotli/dec/context.h +3 -12
- data/vendor/brotli/dec/decode.c +1813 -1048
- data/vendor/brotli/dec/decode.h +22 -16
- data/vendor/brotli/dec/dictionary.c +9466 -0
- data/vendor/brotli/dec/dictionary.h +6 -9461
- data/vendor/brotli/dec/huffman.c +104 -71
- data/vendor/brotli/dec/huffman.h +19 -28
- data/vendor/brotli/dec/port.h +124 -32
- data/vendor/brotli/dec/prefix.h +4 -13
- data/vendor/brotli/dec/state.c +93 -56
- data/vendor/brotli/dec/state.h +124 -53
- data/vendor/brotli/dec/streams.c +14 -11
- data/vendor/brotli/dec/streams.h +6 -11
- data/vendor/brotli/dec/transform.h +2 -11
- data/vendor/brotli/dec/types.h +21 -19
- data/vendor/brotli/enc/Makefile +4 -1
- data/vendor/brotli/enc/backward_references.cc +87 -94
- data/vendor/brotli/enc/backward_references.h +8 -18
- data/vendor/brotli/enc/bit_cost.h +11 -19
- data/vendor/brotli/enc/block_splitter.cc +43 -48
- data/vendor/brotli/enc/block_splitter.h +7 -16
- data/vendor/brotli/enc/brotli_bit_stream.cc +48 -50
- data/vendor/brotli/enc/brotli_bit_stream.h +7 -16
- data/vendor/brotli/enc/cluster.h +24 -25
- data/vendor/brotli/enc/command.h +34 -41
- data/vendor/brotli/enc/context.h +11 -18
- data/vendor/brotli/enc/dictionary.cc +9466 -0
- data/vendor/brotli/enc/dictionary.h +20 -9464
- data/vendor/brotli/enc/dictionary_hash.h +7 -15
- data/vendor/brotli/enc/encode.cc +80 -148
- data/vendor/brotli/enc/encode.h +19 -29
- data/vendor/brotli/enc/encode_parallel.cc +35 -108
- data/vendor/brotli/enc/encode_parallel.h +7 -16
- data/vendor/brotli/enc/entropy_encode.cc +33 -42
- data/vendor/brotli/enc/entropy_encode.h +8 -16
- data/vendor/brotli/enc/fast_log.h +8 -15
- data/vendor/brotli/enc/find_match_length.h +7 -17
- data/vendor/brotli/enc/hash.h +130 -150
- data/vendor/brotli/enc/histogram.cc +7 -16
- data/vendor/brotli/enc/histogram.h +11 -17
- data/vendor/brotli/enc/literal_cost.cc +28 -35
- data/vendor/brotli/enc/literal_cost.h +9 -23
- data/vendor/brotli/enc/metablock.cc +18 -26
- data/vendor/brotli/enc/metablock.h +6 -14
- data/vendor/brotli/enc/port.h +14 -14
- data/vendor/brotli/enc/prefix.h +11 -18
- data/vendor/brotli/enc/ringbuffer.h +18 -27
- data/vendor/brotli/enc/static_dict.cc +7 -1
- data/vendor/brotli/enc/static_dict.h +7 -15
- data/vendor/brotli/enc/static_dict_lut.h +7 -15
- data/vendor/brotli/enc/streams.cc +15 -28
- data/vendor/brotli/enc/streams.h +27 -35
- data/vendor/brotli/enc/transform.h +9 -16
- data/vendor/brotli/enc/types.h +27 -0
- data/vendor/brotli/enc/utf8_util.cc +82 -0
- data/vendor/brotli/enc/utf8_util.h +25 -0
- data/vendor/brotli/enc/write_bits.h +11 -18
- metadata +7 -2
@@ -1,26 +1,19 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Literal cost model to allow backward reference replacement to be efficient.
|
16
8
|
|
17
9
|
#include "./literal_cost.h"
|
18
10
|
|
19
11
|
#include <math.h>
|
20
|
-
#include <stdint.h>
|
21
12
|
#include <algorithm>
|
22
13
|
|
23
14
|
#include "./fast_log.h"
|
15
|
+
#include "./types.h"
|
16
|
+
#include "./utf8_util.h"
|
24
17
|
|
25
18
|
namespace brotli {
|
26
19
|
|
@@ -45,7 +38,7 @@ static int DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
|
|
45
38
|
int max_utf8 = 1; // should be 2, but 1 compresses better.
|
46
39
|
int last_c = 0;
|
47
40
|
int utf8_pos = 0;
|
48
|
-
for (
|
41
|
+
for (size_t i = 0; i < len; ++i) {
|
49
42
|
int c = data[(pos + i) & mask];
|
50
43
|
utf8_pos = UTF8Position(last_c, c, 2);
|
51
44
|
++counts[utf8_pos];
|
@@ -61,15 +54,14 @@ static int DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
|
|
61
54
|
}
|
62
55
|
|
63
56
|
void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
64
|
-
|
65
|
-
float *cost) {
|
57
|
+
const uint8_t *data, float *cost) {
|
66
58
|
|
67
59
|
// max_utf8 is 0 (normal ascii single byte modeling),
|
68
60
|
// 1 (for 2-byte utf-8 modeling), or 2 (for 3-byte utf-8 modeling).
|
69
61
|
const int max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
|
70
62
|
int histogram[3][256] = { { 0 } };
|
71
63
|
int window_half = 495;
|
72
|
-
int in_window = std::min(static_cast<
|
64
|
+
int in_window = std::min(window_half, static_cast<int>(len));
|
73
65
|
int in_window_utf8[3] = { 0 };
|
74
66
|
|
75
67
|
// Bootstrap histograms.
|
@@ -84,7 +76,7 @@ void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
|
84
76
|
}
|
85
77
|
|
86
78
|
// Compute bit costs with sliding window.
|
87
|
-
for (int i = 0; i < len; ++i) {
|
79
|
+
for (int i = 0; i < static_cast<int>(len); ++i) {
|
88
80
|
if (i - window_half >= 0) {
|
89
81
|
// Remove a byte in the past.
|
90
82
|
int c = (i - window_half - 1) < 0 ?
|
@@ -95,12 +87,10 @@ void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
|
95
87
|
--histogram[utf8_pos2][data[(pos + i - window_half) & mask]];
|
96
88
|
--in_window_utf8[utf8_pos2];
|
97
89
|
}
|
98
|
-
if (i + window_half < len) {
|
90
|
+
if (i + window_half < static_cast<int>(len)) {
|
99
91
|
// Add a byte in the future.
|
100
|
-
int c = (i + window_half - 1)
|
101
|
-
|
102
|
-
int last_c = (i + window_half - 2) < 0 ?
|
103
|
-
0 : data[(pos + i + window_half - 2) & mask];
|
92
|
+
int c = data[(pos + i + window_half - 1) & mask];
|
93
|
+
int last_c = data[(pos + i + window_half - 2) & mask];
|
104
94
|
int utf8_pos2 = UTF8Position(last_c, c, max_utf8);
|
105
95
|
++histogram[utf8_pos2][data[(pos + i + window_half) & mask]];
|
106
96
|
++in_window_utf8[utf8_pos2];
|
@@ -108,12 +98,12 @@ void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
|
108
98
|
int c = i < 1 ? 0 : data[(pos + i - 1) & mask];
|
109
99
|
int last_c = i < 2 ? 0 : data[(pos + i - 2) & mask];
|
110
100
|
int utf8_pos = UTF8Position(last_c, c, max_utf8);
|
111
|
-
|
101
|
+
size_t masked_pos = (pos + i) & mask;
|
112
102
|
int histo = histogram[utf8_pos][data[masked_pos]];
|
113
103
|
if (histo == 0) {
|
114
104
|
histo = 1;
|
115
105
|
}
|
116
|
-
|
106
|
+
double lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo);
|
117
107
|
lit_cost += 0.02905;
|
118
108
|
if (lit_cost < 1.0) {
|
119
109
|
lit_cost *= 0.5;
|
@@ -126,16 +116,19 @@ void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
|
126
116
|
if (i < 2000) {
|
127
117
|
lit_cost += 0.7 - ((2000 - i) / 2000.0 * 0.35);
|
128
118
|
}
|
129
|
-
cost[
|
119
|
+
cost[i] = static_cast<float>(lit_cost);
|
130
120
|
}
|
131
121
|
}
|
132
122
|
|
133
123
|
void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
|
134
|
-
|
135
|
-
|
124
|
+
const uint8_t *data, float *cost) {
|
125
|
+
if (IsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
|
126
|
+
EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost);
|
127
|
+
return;
|
128
|
+
}
|
136
129
|
int histogram[256] = { 0 };
|
137
130
|
int window_half = 2000;
|
138
|
-
int in_window = std::min(static_cast<
|
131
|
+
int in_window = std::min(window_half, static_cast<int>(len));
|
139
132
|
|
140
133
|
// Bootstrap histogram.
|
141
134
|
for (int i = 0; i < in_window; ++i) {
|
@@ -143,13 +136,13 @@ void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
|
|
143
136
|
}
|
144
137
|
|
145
138
|
// Compute bit costs with sliding window.
|
146
|
-
for (int i = 0; i < len; ++i) {
|
139
|
+
for (int i = 0; i < static_cast<int>(len); ++i) {
|
147
140
|
if (i - window_half >= 0) {
|
148
141
|
// Remove a byte in the past.
|
149
142
|
--histogram[data[(pos + i - window_half) & mask]];
|
150
143
|
--in_window;
|
151
144
|
}
|
152
|
-
if (i + window_half < len) {
|
145
|
+
if (i + window_half < static_cast<int>(len)) {
|
153
146
|
// Add a byte in the future.
|
154
147
|
++histogram[data[(pos + i + window_half) & mask]];
|
155
148
|
++in_window;
|
@@ -158,13 +151,13 @@ void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
|
|
158
151
|
if (histo == 0) {
|
159
152
|
histo = 1;
|
160
153
|
}
|
161
|
-
|
154
|
+
double lit_cost = FastLog2(in_window) - FastLog2(histo);
|
162
155
|
lit_cost += 0.029;
|
163
156
|
if (lit_cost < 1.0) {
|
164
157
|
lit_cost *= 0.5;
|
165
158
|
lit_cost += 0.5;
|
166
159
|
}
|
167
|
-
cost[
|
160
|
+
cost[i] = static_cast<float>(lit_cost);
|
168
161
|
}
|
169
162
|
}
|
170
163
|
|
@@ -1,37 +1,23 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Literal cost model to allow backward reference replacement to be efficient.
|
16
8
|
|
17
9
|
#ifndef BROTLI_ENC_LITERAL_COST_H_
|
18
10
|
#define BROTLI_ENC_LITERAL_COST_H_
|
19
11
|
|
20
|
-
#include
|
21
|
-
#include <stdint.h>
|
12
|
+
#include "./types.h"
|
22
13
|
|
23
14
|
namespace brotli {
|
24
15
|
|
25
16
|
// Estimates how many bits the literals in the interval [pos, pos + len) in the
|
26
17
|
// ringbuffer (data, mask) will take entropy coded and writes these estimates
|
27
|
-
// to the
|
18
|
+
// to the cost[0..len) array.
|
28
19
|
void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
|
29
|
-
|
30
|
-
float *cost);
|
31
|
-
|
32
|
-
void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
33
|
-
size_t cost_mask, const uint8_t *data,
|
34
|
-
float *cost);
|
20
|
+
const uint8_t *data, float *cost);
|
35
21
|
|
36
22
|
} // namespace brotli
|
37
23
|
|
@@ -1,17 +1,9 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2015 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Algorithms for distributing the literals and commands of a metablock between
|
16
8
|
// block types and contexts.
|
17
9
|
|
@@ -64,7 +56,7 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
|
|
64
56
|
&distance_histograms);
|
65
57
|
|
66
58
|
// Histogram ids need to fit in one byte.
|
67
|
-
static const
|
59
|
+
static const size_t kMaxNumberOfHistograms = 256;
|
68
60
|
|
69
61
|
mb->literal_histograms = literal_histograms;
|
70
62
|
ClusterHistograms(literal_histograms,
|
@@ -247,7 +239,7 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
|
|
247
239
|
size_t n_commands,
|
248
240
|
MetaBlockSplit* mb) {
|
249
241
|
int num_literals = 0;
|
250
|
-
for (
|
242
|
+
for (size_t i = 0; i < n_commands; ++i) {
|
251
243
|
num_literals += commands[i].insert_len_;
|
252
244
|
}
|
253
245
|
|
@@ -255,13 +247,13 @@ void BuildMetaBlockGreedy(const uint8_t* ringbuffer,
|
|
255
247
|
256, 512, 400.0, num_literals,
|
256
248
|
&mb->literal_split, &mb->literal_histograms);
|
257
249
|
BlockSplitter<HistogramCommand> cmd_blocks(
|
258
|
-
kNumCommandPrefixes, 1024, 500.0, n_commands,
|
250
|
+
kNumCommandPrefixes, 1024, 500.0, static_cast<int>(n_commands),
|
259
251
|
&mb->command_split, &mb->command_histograms);
|
260
252
|
BlockSplitter<HistogramDistance> dist_blocks(
|
261
|
-
64, 512, 100.0, n_commands,
|
253
|
+
64, 512, 100.0, static_cast<int>(n_commands),
|
262
254
|
&mb->distance_split, &mb->distance_histograms);
|
263
255
|
|
264
|
-
for (
|
256
|
+
for (size_t i = 0; i < n_commands; ++i) {
|
265
257
|
const Command cmd = commands[i];
|
266
258
|
cmd_blocks.AddSymbol(cmd.cmd_prefix_);
|
267
259
|
for (int j = 0; j < cmd.insert_len_; ++j) {
|
@@ -473,7 +465,7 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
473
465
|
size_t n_commands,
|
474
466
|
MetaBlockSplit* mb) {
|
475
467
|
int num_literals = 0;
|
476
|
-
for (
|
468
|
+
for (size_t i = 0; i < n_commands; ++i) {
|
477
469
|
num_literals += commands[i].insert_len_;
|
478
470
|
}
|
479
471
|
|
@@ -481,13 +473,13 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
481
473
|
256, num_contexts, 512, 400.0, num_literals,
|
482
474
|
&mb->literal_split, &mb->literal_histograms);
|
483
475
|
BlockSplitter<HistogramCommand> cmd_blocks(
|
484
|
-
kNumCommandPrefixes, 1024, 500.0, n_commands,
|
476
|
+
kNumCommandPrefixes, 1024, 500.0, static_cast<int>(n_commands),
|
485
477
|
&mb->command_split, &mb->command_histograms);
|
486
478
|
BlockSplitter<HistogramDistance> dist_blocks(
|
487
|
-
64, 512, 100.0, n_commands,
|
479
|
+
64, 512, 100.0, static_cast<int>(n_commands),
|
488
480
|
&mb->distance_split, &mb->distance_histograms);
|
489
481
|
|
490
|
-
for (
|
482
|
+
for (size_t i = 0; i < n_commands; ++i) {
|
491
483
|
const Command cmd = commands[i];
|
492
484
|
cmd_blocks.AddSymbol(cmd.cmd_prefix_);
|
493
485
|
for (int j = 0; j < cmd.insert_len_; ++j) {
|
@@ -525,17 +517,17 @@ void BuildMetaBlockGreedyWithContexts(const uint8_t* ringbuffer,
|
|
525
517
|
void OptimizeHistograms(int num_direct_distance_codes,
|
526
518
|
int distance_postfix_bits,
|
527
519
|
MetaBlockSplit* mb) {
|
528
|
-
for (
|
520
|
+
for (size_t i = 0; i < mb->literal_histograms.size(); ++i) {
|
529
521
|
OptimizeHuffmanCountsForRle(256, &mb->literal_histograms[i].data_[0]);
|
530
522
|
}
|
531
|
-
for (
|
523
|
+
for (size_t i = 0; i < mb->command_histograms.size(); ++i) {
|
532
524
|
OptimizeHuffmanCountsForRle(kNumCommandPrefixes,
|
533
525
|
&mb->command_histograms[i].data_[0]);
|
534
526
|
}
|
535
527
|
int num_distance_codes =
|
536
528
|
kNumDistanceShortCodes + num_direct_distance_codes +
|
537
529
|
(48 << distance_postfix_bits);
|
538
|
-
for (
|
530
|
+
for (size_t i = 0; i < mb->distance_histograms.size(); ++i) {
|
539
531
|
OptimizeHuffmanCountsForRle(num_distance_codes,
|
540
532
|
&mb->distance_histograms[i].data_[0]);
|
541
533
|
}
|
@@ -1,17 +1,9 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2015 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Algorithms for distributing the literals and commands of a metablock between
|
16
8
|
// block types and contexts.
|
17
9
|
|
data/vendor/brotli/enc/port.h
CHANGED
@@ -1,23 +1,17 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Macros for endianness, branch prediction and unaligned loads and stores.
|
16
8
|
|
17
9
|
#ifndef BROTLI_ENC_PORT_H_
|
18
10
|
#define BROTLI_ENC_PORT_H_
|
19
11
|
|
12
|
+
#include <assert.h>
|
20
13
|
#include <string.h>
|
14
|
+
#include "./types.h"
|
21
15
|
|
22
16
|
#if defined OS_LINUX || defined OS_CYGWIN
|
23
17
|
#include <endian.h>
|
@@ -148,4 +142,10 @@ inline void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
|
|
148
142
|
|
149
143
|
#endif
|
150
144
|
|
145
|
+
#ifdef BROTLI_ENCODE_DEBUG
|
146
|
+
#define BROTLI_DCHECK(x) assert(x)
|
147
|
+
#else
|
148
|
+
#define BROTLI_DCHECK(x)
|
149
|
+
#endif
|
150
|
+
|
151
151
|
#endif // BROTLI_ENC_PORT_H_
|
data/vendor/brotli/enc/prefix.h
CHANGED
@@ -1,25 +1,17 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Functions for encoding of integers into prefix codes the amount of extra
|
16
8
|
// bits, and the actual values of the extra bits.
|
17
9
|
|
18
10
|
#ifndef BROTLI_ENC_PREFIX_H_
|
19
11
|
#define BROTLI_ENC_PREFIX_H_
|
20
12
|
|
21
|
-
#include <stdint.h>
|
22
13
|
#include "./fast_log.h"
|
14
|
+
#include "./types.h"
|
23
15
|
|
24
16
|
namespace brotli {
|
25
17
|
|
@@ -63,7 +55,7 @@ inline void PrefixEncodeCopyDistance(int distance_code,
|
|
63
55
|
uint16_t* code,
|
64
56
|
uint32_t* extra_bits) {
|
65
57
|
if (distance_code < kNumDistanceShortCodes + num_direct_codes) {
|
66
|
-
*code = distance_code;
|
58
|
+
*code = static_cast<uint16_t>(distance_code);
|
67
59
|
*extra_bits = 0;
|
68
60
|
return;
|
69
61
|
}
|
@@ -75,8 +67,9 @@ inline void PrefixEncodeCopyDistance(int distance_code,
|
|
75
67
|
int prefix = (distance_code >> bucket) & 1;
|
76
68
|
int offset = (2 + prefix) << bucket;
|
77
69
|
int nbits = bucket - postfix_bits;
|
78
|
-
*code =
|
79
|
-
(
|
70
|
+
*code = static_cast<uint16_t>(
|
71
|
+
(kNumDistanceShortCodes + num_direct_codes +
|
72
|
+
((2 * (nbits - 1) + prefix) << postfix_bits) + postfix));
|
80
73
|
*extra_bits = (nbits << 24) | ((distance_code - offset) >> postfix_bits);
|
81
74
|
}
|
82
75
|
|
@@ -1,26 +1,17 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Sliding window over the input data.
|
16
8
|
|
17
9
|
#ifndef BROTLI_ENC_RINGBUFFER_H_
|
18
10
|
#define BROTLI_ENC_RINGBUFFER_H_
|
19
11
|
|
20
|
-
#include <stddef.h>
|
21
|
-
#include <stdint.h>
|
22
12
|
|
23
13
|
#include "./port.h"
|
14
|
+
#include "./types.h"
|
24
15
|
|
25
16
|
namespace brotli {
|
26
17
|
|
@@ -32,12 +23,12 @@ namespace brotli {
|
|
32
23
|
class RingBuffer {
|
33
24
|
public:
|
34
25
|
RingBuffer(int window_bits, int tail_bits)
|
35
|
-
:
|
36
|
-
mask_((1 << window_bits) - 1),
|
37
|
-
tail_size_(1 << tail_bits),
|
26
|
+
: size_((size_t(1) << window_bits)),
|
27
|
+
mask_((size_t(1) << window_bits) - 1),
|
28
|
+
tail_size_(size_t(1) << tail_bits),
|
38
29
|
pos_(0) {
|
39
30
|
static const int kSlackForEightByteHashingEverywhere = 7;
|
40
|
-
const
|
31
|
+
const size_t buflen = size_ + tail_size_;
|
41
32
|
buffer_ = new uint8_t[buflen + kSlackForEightByteHashingEverywhere];
|
42
33
|
for (int i = 0; i < kSlackForEightByteHashingEverywhere; ++i) {
|
43
34
|
buffer_[buflen + i] = 0;
|
@@ -53,17 +44,17 @@ class RingBuffer {
|
|
53
44
|
// The length of the writes is limited so that we do not need to worry
|
54
45
|
// about a write
|
55
46
|
WriteTail(bytes, n);
|
56
|
-
if (PREDICT_TRUE(masked_pos + n <=
|
47
|
+
if (PREDICT_TRUE(masked_pos + n <= size_)) {
|
57
48
|
// A single write fits.
|
58
49
|
memcpy(&buffer_[masked_pos], bytes, n);
|
59
50
|
} else {
|
60
51
|
// Split into two writes.
|
61
52
|
// Copy into the end of the buffer, including the tail buffer.
|
62
53
|
memcpy(&buffer_[masked_pos], bytes,
|
63
|
-
std::min(n, (
|
64
|
-
// Copy into the
|
65
|
-
memcpy(&buffer_[0], bytes + (
|
66
|
-
n - (
|
54
|
+
std::min(n, (size_ + tail_size_) - masked_pos));
|
55
|
+
// Copy into the beginning of the buffer
|
56
|
+
memcpy(&buffer_[0], bytes + (size_ - masked_pos),
|
57
|
+
n - (size_ - masked_pos));
|
67
58
|
}
|
68
59
|
pos_ += n;
|
69
60
|
}
|
@@ -86,13 +77,13 @@ class RingBuffer {
|
|
86
77
|
const size_t masked_pos = pos_ & mask_;
|
87
78
|
if (PREDICT_FALSE(masked_pos < tail_size_)) {
|
88
79
|
// Just fill the tail buffer with the beginning data.
|
89
|
-
const size_t p =
|
80
|
+
const size_t p = size_ + masked_pos;
|
90
81
|
memcpy(&buffer_[p], bytes, std::min(n, tail_size_ - masked_pos));
|
91
82
|
}
|
92
83
|
}
|
93
84
|
|
94
85
|
// Size of the ringbuffer is (1 << window_bits) + tail_size_.
|
95
|
-
const
|
86
|
+
const size_t size_;
|
96
87
|
const size_t mask_;
|
97
88
|
const size_t tail_size_;
|
98
89
|
|
@@ -1,3 +1,9 @@
|
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
1
7
|
#include "./static_dict.h"
|
2
8
|
|
3
9
|
#include <algorithm>
|
@@ -76,7 +82,7 @@ bool FindAllStaticDictionaryMatches(const uint8_t* data,
|
|
76
82
|
AddMatch(id, l, l, matches);
|
77
83
|
found_match = true;
|
78
84
|
}
|
79
|
-
//
|
85
|
+
// Transforms "" + kOmitLast1 + "" and "" + kOmitLast1 + "ing "
|
80
86
|
if (matchlen >= l - 1) {
|
81
87
|
AddMatch(id + 12 * n, l - 1, l, matches);
|
82
88
|
if (l + 2 < max_length &&
|
@@ -1,23 +1,15 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Class to model the static dictionary.
|
16
8
|
|
17
9
|
#ifndef BROTLI_ENC_STATIC_DICT_H_
|
18
10
|
#define BROTLI_ENC_STATIC_DICT_H_
|
19
11
|
|
20
|
-
#include
|
12
|
+
#include "./types.h"
|
21
13
|
|
22
14
|
namespace brotli {
|
23
15
|
|
@@ -1,23 +1,15 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2015 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Lookup table for static dictionary and transforms.
|
16
8
|
|
17
9
|
#ifndef BROTLI_ENC_DICTIONARY_LUT_H_
|
18
10
|
#define BROTLI_ENC_DICTIONARY_LUT_H_
|
19
11
|
|
20
|
-
#include
|
12
|
+
#include "./types.h"
|
21
13
|
|
22
14
|
namespace brotli {
|
23
15
|
|