brotli 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitmodules +1 -1
- data/.travis.yml +2 -1
- data/README.md +1 -1
- data/Rakefile +1 -1
- data/ext/brotli/brotli.cc +1 -1
- data/ext/brotli/extconf.rb +72 -14
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/LICENSE +19 -202
- data/vendor/brotli/dec/Makefile +1 -1
- data/vendor/brotli/dec/bit_reader.c +23 -30
- data/vendor/brotli/dec/bit_reader.h +270 -141
- data/vendor/brotli/dec/context.h +3 -12
- data/vendor/brotli/dec/decode.c +1813 -1048
- data/vendor/brotli/dec/decode.h +22 -16
- data/vendor/brotli/dec/dictionary.c +9466 -0
- data/vendor/brotli/dec/dictionary.h +6 -9461
- data/vendor/brotli/dec/huffman.c +104 -71
- data/vendor/brotli/dec/huffman.h +19 -28
- data/vendor/brotli/dec/port.h +124 -32
- data/vendor/brotli/dec/prefix.h +4 -13
- data/vendor/brotli/dec/state.c +93 -56
- data/vendor/brotli/dec/state.h +124 -53
- data/vendor/brotli/dec/streams.c +14 -11
- data/vendor/brotli/dec/streams.h +6 -11
- data/vendor/brotli/dec/transform.h +2 -11
- data/vendor/brotli/dec/types.h +21 -19
- data/vendor/brotli/enc/Makefile +4 -1
- data/vendor/brotli/enc/backward_references.cc +87 -94
- data/vendor/brotli/enc/backward_references.h +8 -18
- data/vendor/brotli/enc/bit_cost.h +11 -19
- data/vendor/brotli/enc/block_splitter.cc +43 -48
- data/vendor/brotli/enc/block_splitter.h +7 -16
- data/vendor/brotli/enc/brotli_bit_stream.cc +48 -50
- data/vendor/brotli/enc/brotli_bit_stream.h +7 -16
- data/vendor/brotli/enc/cluster.h +24 -25
- data/vendor/brotli/enc/command.h +34 -41
- data/vendor/brotli/enc/context.h +11 -18
- data/vendor/brotli/enc/dictionary.cc +9466 -0
- data/vendor/brotli/enc/dictionary.h +20 -9464
- data/vendor/brotli/enc/dictionary_hash.h +7 -15
- data/vendor/brotli/enc/encode.cc +80 -148
- data/vendor/brotli/enc/encode.h +19 -29
- data/vendor/brotli/enc/encode_parallel.cc +35 -108
- data/vendor/brotli/enc/encode_parallel.h +7 -16
- data/vendor/brotli/enc/entropy_encode.cc +33 -42
- data/vendor/brotli/enc/entropy_encode.h +8 -16
- data/vendor/brotli/enc/fast_log.h +8 -15
- data/vendor/brotli/enc/find_match_length.h +7 -17
- data/vendor/brotli/enc/hash.h +130 -150
- data/vendor/brotli/enc/histogram.cc +7 -16
- data/vendor/brotli/enc/histogram.h +11 -17
- data/vendor/brotli/enc/literal_cost.cc +28 -35
- data/vendor/brotli/enc/literal_cost.h +9 -23
- data/vendor/brotli/enc/metablock.cc +18 -26
- data/vendor/brotli/enc/metablock.h +6 -14
- data/vendor/brotli/enc/port.h +14 -14
- data/vendor/brotli/enc/prefix.h +11 -18
- data/vendor/brotli/enc/ringbuffer.h +18 -27
- data/vendor/brotli/enc/static_dict.cc +7 -1
- data/vendor/brotli/enc/static_dict.h +7 -15
- data/vendor/brotli/enc/static_dict_lut.h +7 -15
- data/vendor/brotli/enc/streams.cc +15 -28
- data/vendor/brotli/enc/streams.h +27 -35
- data/vendor/brotli/enc/transform.h +9 -16
- data/vendor/brotli/enc/types.h +27 -0
- data/vendor/brotli/enc/utf8_util.cc +82 -0
- data/vendor/brotli/enc/utf8_util.h +25 -0
- data/vendor/brotli/enc/write_bits.h +11 -18
- metadata +7 -2
@@ -1,21 +1,14 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Block split point selection utilities.
|
16
8
|
|
17
9
|
#include "./block_splitter.h"
|
18
10
|
|
11
|
+
#include <assert.h>
|
19
12
|
#include <math.h>
|
20
13
|
#include <stdio.h>
|
21
14
|
#include <stdlib.h>
|
@@ -53,7 +46,7 @@ void CopyLiteralsToByteArray(const Command* cmds,
|
|
53
46
|
std::vector<uint8_t>* literals) {
|
54
47
|
// Count how many we have.
|
55
48
|
size_t total_length = 0;
|
56
|
-
for (
|
49
|
+
for (size_t i = 0; i < num_commands; ++i) {
|
57
50
|
total_length += cmds[i].insert_len_;
|
58
51
|
}
|
59
52
|
if (total_length == 0) {
|
@@ -66,7 +59,7 @@ void CopyLiteralsToByteArray(const Command* cmds,
|
|
66
59
|
// Loop again, and copy this time.
|
67
60
|
size_t pos = 0;
|
68
61
|
size_t from_pos = offset & mask;
|
69
|
-
for (
|
62
|
+
for (size_t i = 0; i < num_commands && pos < total_length; ++i) {
|
70
63
|
size_t insert_len = cmds[i].insert_len_;
|
71
64
|
if (from_pos + insert_len > mask) {
|
72
65
|
size_t head_size = mask + 1 - from_pos;
|
@@ -87,7 +80,7 @@ void CopyCommandsToByteArray(const Command* cmds,
|
|
87
80
|
const size_t num_commands,
|
88
81
|
std::vector<uint16_t>* insert_and_copy_codes,
|
89
82
|
std::vector<uint16_t>* distance_prefixes) {
|
90
|
-
for (
|
83
|
+
for (size_t i = 0; i < num_commands; ++i) {
|
91
84
|
const Command& cmd = cmds[i];
|
92
85
|
insert_and_copy_codes->push_back(cmd.cmd_prefix_);
|
93
86
|
if (cmd.copy_len_ > 0 && cmd.cmd_prefix_ >= 128) {
|
@@ -110,14 +103,14 @@ void InitialEntropyCodes(const DataType* data, size_t length,
|
|
110
103
|
int max_histograms,
|
111
104
|
size_t stride,
|
112
105
|
std::vector<HistogramType>* vec) {
|
113
|
-
int total_histograms = length / literals_per_histogram + 1;
|
106
|
+
int total_histograms = static_cast<int>(length) / literals_per_histogram + 1;
|
114
107
|
if (total_histograms > max_histograms) {
|
115
108
|
total_histograms = max_histograms;
|
116
109
|
}
|
117
110
|
unsigned int seed = 7;
|
118
|
-
|
111
|
+
size_t block_length = length / total_histograms;
|
119
112
|
for (int i = 0; i < total_histograms; ++i) {
|
120
|
-
|
113
|
+
size_t pos = length * i / total_histograms;
|
121
114
|
if (i != 0) {
|
122
115
|
pos += MyRand(&seed) % block_length;
|
123
116
|
}
|
@@ -150,19 +143,19 @@ template<typename HistogramType, typename DataType>
|
|
150
143
|
void RefineEntropyCodes(const DataType* data, size_t length,
|
151
144
|
size_t stride,
|
152
145
|
std::vector<HistogramType>* vec) {
|
153
|
-
|
146
|
+
size_t iters =
|
154
147
|
kIterMulForRefining * length / stride + kMinItersForRefining;
|
155
148
|
unsigned int seed = 7;
|
156
149
|
iters = ((iters + vec->size() - 1) / vec->size()) * vec->size();
|
157
|
-
for (
|
150
|
+
for (size_t iter = 0; iter < iters; ++iter) {
|
158
151
|
HistogramType sample;
|
159
152
|
RandomSample(&seed, data, length, stride, &sample);
|
160
|
-
|
153
|
+
size_t ix = iter % vec->size();
|
161
154
|
(*vec)[ix].AddHistogram(sample);
|
162
155
|
}
|
163
156
|
}
|
164
157
|
|
165
|
-
inline static
|
158
|
+
inline static double BitCost(int count) {
|
166
159
|
return count == 0 ? -2 : FastLog2(count);
|
167
160
|
}
|
168
161
|
|
@@ -172,12 +165,13 @@ void FindBlocks(const DataType* data, const size_t length,
|
|
172
165
|
const std::vector<Histogram<kSize> > &vec,
|
173
166
|
uint8_t *block_id) {
|
174
167
|
if (vec.size() <= 1) {
|
175
|
-
for (
|
168
|
+
for (size_t i = 0; i < length; ++i) {
|
176
169
|
block_id[i] = 0;
|
177
170
|
}
|
178
171
|
return;
|
179
172
|
}
|
180
|
-
int vecsize = vec.size();
|
173
|
+
int vecsize = static_cast<int>(vec.size());
|
174
|
+
assert(vecsize <= 256);
|
181
175
|
double* insert_cost = new double[kSize * vecsize];
|
182
176
|
memset(insert_cost, 0, sizeof(insert_cost[0]) * kSize * vecsize);
|
183
177
|
for (int j = 0; j < vecsize; ++j) {
|
@@ -199,7 +193,7 @@ void FindBlocks(const DataType* data, const size_t length,
|
|
199
193
|
// reaches block switch cost, it means that when we trace back from the last
|
200
194
|
// position, we need to switch here.
|
201
195
|
for (size_t byte_ix = 0; byte_ix < length; ++byte_ix) {
|
202
|
-
|
196
|
+
size_t ix = byte_ix * vecsize;
|
203
197
|
int insert_cost_ix = data[byte_ix] * vecsize;
|
204
198
|
double min_cost = 1e99;
|
205
199
|
for (int k = 0; k < vecsize; ++k) {
|
@@ -207,7 +201,7 @@ void FindBlocks(const DataType* data, const size_t length,
|
|
207
201
|
cost[k] += insert_cost[insert_cost_ix + k];
|
208
202
|
if (cost[k] < min_cost) {
|
209
203
|
min_cost = cost[k];
|
210
|
-
block_id[byte_ix] = k;
|
204
|
+
block_id[byte_ix] = static_cast<uint8_t>(k);
|
211
205
|
}
|
212
206
|
}
|
213
207
|
double block_switch_cost = block_switch_bitcost;
|
@@ -224,9 +218,9 @@ void FindBlocks(const DataType* data, const size_t length,
|
|
224
218
|
}
|
225
219
|
}
|
226
220
|
// Now trace back from the last position and switch at the marked places.
|
227
|
-
|
228
|
-
|
229
|
-
|
221
|
+
size_t byte_ix = length - 1;
|
222
|
+
size_t ix = byte_ix * vecsize;
|
223
|
+
uint8_t cur_id = block_id[byte_ix];
|
230
224
|
while (byte_ix > 0) {
|
231
225
|
--byte_ix;
|
232
226
|
ix -= vecsize;
|
@@ -243,13 +237,13 @@ void FindBlocks(const DataType* data, const size_t length,
|
|
243
237
|
int RemapBlockIds(uint8_t* block_ids, const size_t length) {
|
244
238
|
std::map<uint8_t, uint8_t> new_id;
|
245
239
|
int next_id = 0;
|
246
|
-
for (
|
240
|
+
for (size_t i = 0; i < length; ++i) {
|
247
241
|
if (new_id.find(block_ids[i]) == new_id.end()) {
|
248
|
-
new_id[block_ids[i]] = next_id;
|
242
|
+
new_id[block_ids[i]] = static_cast<uint8_t>(next_id);
|
249
243
|
++next_id;
|
250
244
|
}
|
251
245
|
}
|
252
|
-
for (
|
246
|
+
for (size_t i = 0; i < length; ++i) {
|
253
247
|
block_ids[i] = new_id[block_ids[i]];
|
254
248
|
}
|
255
249
|
return next_id;
|
@@ -260,9 +254,10 @@ void BuildBlockHistograms(const DataType* data, const size_t length,
|
|
260
254
|
uint8_t* block_ids,
|
261
255
|
std::vector<HistogramType>* histograms) {
|
262
256
|
int num_types = RemapBlockIds(block_ids, length);
|
257
|
+
assert(num_types <= 256);
|
263
258
|
histograms->clear();
|
264
259
|
histograms->resize(num_types);
|
265
|
-
for (
|
260
|
+
for (size_t i = 0; i < length; ++i) {
|
266
261
|
(*histograms)[block_ids[i]].Add(data[i]);
|
267
262
|
}
|
268
263
|
}
|
@@ -274,7 +269,7 @@ void ClusterBlocks(const DataType* data, const size_t length,
|
|
274
269
|
std::vector<int> block_index(length);
|
275
270
|
int cur_idx = 0;
|
276
271
|
HistogramType cur_histogram;
|
277
|
-
for (
|
272
|
+
for (size_t i = 0; i < length; ++i) {
|
278
273
|
bool block_boundary = (i + 1 == length || block_ids[i] != block_ids[i + 1]);
|
279
274
|
block_index[i] = cur_idx;
|
280
275
|
cur_histogram.Add(data[i]);
|
@@ -287,13 +282,13 @@ void ClusterBlocks(const DataType* data, const size_t length,
|
|
287
282
|
std::vector<HistogramType> clustered_histograms;
|
288
283
|
std::vector<int> histogram_symbols;
|
289
284
|
// Block ids need to fit in one byte.
|
290
|
-
static const
|
291
|
-
ClusterHistograms(histograms, 1, histograms.size(),
|
285
|
+
static const size_t kMaxNumberOfBlockTypes = 256;
|
286
|
+
ClusterHistograms(histograms, 1, static_cast<int>(histograms.size()),
|
292
287
|
kMaxNumberOfBlockTypes,
|
293
288
|
&clustered_histograms,
|
294
289
|
&histogram_symbols);
|
295
|
-
for (
|
296
|
-
block_ids[i] = histogram_symbols[block_index[i]];
|
290
|
+
for (size_t i = 0; i < length; ++i) {
|
291
|
+
block_ids[i] = static_cast<uint8_t>(histogram_symbols[block_index[i]]);
|
297
292
|
}
|
298
293
|
}
|
299
294
|
|
@@ -301,7 +296,7 @@ void BuildBlockSplit(const std::vector<uint8_t>& block_ids, BlockSplit* split) {
|
|
301
296
|
int cur_id = block_ids[0];
|
302
297
|
int cur_length = 1;
|
303
298
|
split->num_types = -1;
|
304
|
-
for (
|
299
|
+
for (size_t i = 1; i < block_ids.size(); ++i) {
|
305
300
|
if (block_ids[i] != cur_id) {
|
306
301
|
split->types.push_back(cur_id);
|
307
302
|
split->lengths.push_back(cur_length);
|
@@ -330,29 +325,29 @@ void SplitByteVector(const std::vector<DataType>& data,
|
|
330
325
|
} else if (data.size() < kMinLengthForBlockSplitting) {
|
331
326
|
split->num_types = 1;
|
332
327
|
split->types.push_back(0);
|
333
|
-
split->lengths.push_back(data.size());
|
328
|
+
split->lengths.push_back(static_cast<int>(data.size()));
|
334
329
|
return;
|
335
330
|
}
|
336
331
|
std::vector<HistogramType> histograms;
|
337
332
|
// Find good entropy codes.
|
338
|
-
InitialEntropyCodes(data
|
333
|
+
InitialEntropyCodes(&data[0], data.size(),
|
339
334
|
literals_per_histogram,
|
340
335
|
max_histograms,
|
341
336
|
sampling_stride_length,
|
342
337
|
&histograms);
|
343
|
-
RefineEntropyCodes(data
|
338
|
+
RefineEntropyCodes(&data[0], data.size(),
|
344
339
|
sampling_stride_length,
|
345
340
|
&histograms);
|
346
341
|
// Find a good path through literals with the good entropy codes.
|
347
342
|
std::vector<uint8_t> block_ids(data.size());
|
348
343
|
for (int i = 0; i < 10; ++i) {
|
349
|
-
FindBlocks(data
|
344
|
+
FindBlocks(&data[0], data.size(),
|
350
345
|
block_switch_cost,
|
351
346
|
histograms,
|
352
347
|
&block_ids[0]);
|
353
|
-
BuildBlockHistograms(data
|
348
|
+
BuildBlockHistograms(&data[0], data.size(), &block_ids[0], &histograms);
|
354
349
|
}
|
355
|
-
ClusterBlocks<HistogramType>(data
|
350
|
+
ClusterBlocks<HistogramType>(&data[0], data.size(), &block_ids[0]);
|
356
351
|
BuildBlockSplit(block_ids, split);
|
357
352
|
}
|
358
353
|
|
@@ -401,7 +396,7 @@ void SplitBlockByTotalLength(const Command* all_commands,
|
|
401
396
|
int length_limit = input_size / num_blocks + 1;
|
402
397
|
int total_length = 0;
|
403
398
|
std::vector<Command> cur_block;
|
404
|
-
for (
|
399
|
+
for (size_t i = 0; i < num_commands; ++i) {
|
405
400
|
const Command& cmd = all_commands[i];
|
406
401
|
int cmd_length = cmd.insert_len_ + cmd.copy_len_;
|
407
402
|
if (total_length > length_limit) {
|
@@ -1,30 +1,21 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Block split point selection utilities.
|
16
8
|
|
17
9
|
#ifndef BROTLI_ENC_BLOCK_SPLITTER_H_
|
18
10
|
#define BROTLI_ENC_BLOCK_SPLITTER_H_
|
19
11
|
|
20
|
-
#include <stddef.h>
|
21
|
-
#include <stdint.h>
|
22
12
|
#include <string.h>
|
23
13
|
#include <vector>
|
24
14
|
#include <utility>
|
25
15
|
|
26
16
|
#include "./command.h"
|
27
17
|
#include "./metablock.h"
|
18
|
+
#include "./types.h"
|
28
19
|
|
29
20
|
namespace brotli {
|
30
21
|
|
@@ -1,17 +1,9 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2014 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Brotli bit stream functions to support the low level format. There are no
|
16
8
|
// compression algorithms here, just the right ordering of bits to match the
|
17
9
|
// specs.
|
@@ -28,19 +20,21 @@
|
|
28
20
|
#include "./fast_log.h"
|
29
21
|
#include "./prefix.h"
|
30
22
|
#include "./write_bits.h"
|
31
|
-
|
32
23
|
namespace brotli {
|
33
24
|
|
34
25
|
// returns false if fail
|
35
26
|
// nibblesbits represents the 2 bits to encode MNIBBLES (0-3)
|
36
27
|
bool EncodeMlen(size_t length, int* bits, int* numbits, int* nibblesbits) {
|
28
|
+
if (length > (1 << 24)) {
|
29
|
+
return false;
|
30
|
+
}
|
37
31
|
length--; // MLEN - 1 is encoded
|
38
|
-
int lg = length == 0 ? 1 : Log2Floor(length) + 1;
|
39
|
-
|
32
|
+
int lg = length == 0 ? 1 : Log2Floor(static_cast<uint32_t>(length)) + 1;
|
33
|
+
assert(lg <= 24);
|
40
34
|
int mnibbles = (lg < 16 ? 16 : (lg + 3)) / 4;
|
41
35
|
*nibblesbits = mnibbles - 4;
|
42
36
|
*numbits = mnibbles * 4;
|
43
|
-
*bits = length;
|
37
|
+
*bits = static_cast<int>(length);
|
44
38
|
return true;
|
45
39
|
}
|
46
40
|
|
@@ -166,7 +160,7 @@ void StoreHuffmanTreeToBitMask(
|
|
166
160
|
const std::vector<uint16_t> &code_length_bitdepth_symbols,
|
167
161
|
int * __restrict storage_ix,
|
168
162
|
uint8_t * __restrict storage) {
|
169
|
-
for (
|
163
|
+
for (size_t i = 0; i < huffman_tree.size(); ++i) {
|
170
164
|
int ix = huffman_tree[i];
|
171
165
|
WriteBits(code_length_bitdepth[ix], code_length_bitdepth_symbols[ix],
|
172
166
|
storage_ix, storage);
|
@@ -219,7 +213,7 @@ void StoreSimpleHuffmanTree(const uint8_t* depths,
|
|
219
213
|
|
220
214
|
// num = alphabet size
|
221
215
|
// depths = symbol depths
|
222
|
-
void StoreHuffmanTree(const uint8_t* depths,
|
216
|
+
void StoreHuffmanTree(const uint8_t* depths, int num,
|
223
217
|
int *storage_ix, uint8_t *storage) {
|
224
218
|
// Write the Huffman tree into the brotli-representation.
|
225
219
|
std::vector<uint8_t> huffman_tree;
|
@@ -231,7 +225,7 @@ void StoreHuffmanTree(const uint8_t* depths, size_t num,
|
|
231
225
|
|
232
226
|
// Calculate the statistics of the Huffman tree in brotli-representation.
|
233
227
|
int huffman_tree_histogram[kCodeLengthCodes] = { 0 };
|
234
|
-
for (
|
228
|
+
for (size_t i = 0; i < huffman_tree.size(); ++i) {
|
235
229
|
++huffman_tree_histogram[huffman_tree[i]];
|
236
230
|
}
|
237
231
|
|
@@ -257,7 +251,7 @@ void StoreHuffmanTree(const uint8_t* depths, size_t num,
|
|
257
251
|
CreateHuffmanTree(&huffman_tree_histogram[0], kCodeLengthCodes,
|
258
252
|
5, &code_length_bitdepth[0]);
|
259
253
|
ConvertBitDepthsToSymbols(code_length_bitdepth, kCodeLengthCodes,
|
260
|
-
code_length_bitdepth_symbols
|
254
|
+
&code_length_bitdepth_symbols[0]);
|
261
255
|
|
262
256
|
// Now, we have all the data, let's start storing it
|
263
257
|
StoreHuffmanTreeOfHuffmanTreeToBitMask(num_codes, code_length_bitdepth,
|
@@ -275,6 +269,7 @@ void StoreHuffmanTree(const uint8_t* depths, size_t num,
|
|
275
269
|
storage_ix, storage);
|
276
270
|
}
|
277
271
|
|
272
|
+
|
278
273
|
void BuildAndStoreHuffmanTree(const int *histogram,
|
279
274
|
const int length,
|
280
275
|
uint8_t* depth,
|
@@ -283,7 +278,7 @@ void BuildAndStoreHuffmanTree(const int *histogram,
|
|
283
278
|
uint8_t* storage) {
|
284
279
|
int count = 0;
|
285
280
|
int s4[4] = { 0 };
|
286
|
-
for (
|
281
|
+
for (int i = 0; i < length; i++) {
|
287
282
|
if (histogram[i]) {
|
288
283
|
if (count < 4) {
|
289
284
|
s4[count] = i;
|
@@ -318,7 +313,7 @@ void BuildAndStoreHuffmanTree(const int *histogram,
|
|
318
313
|
}
|
319
314
|
|
320
315
|
int IndexOf(const std::vector<int>& v, int value) {
|
321
|
-
for (int i = 0; i < v.size(); ++i) {
|
316
|
+
for (int i = 0; i < static_cast<int>(v.size()); ++i) {
|
322
317
|
if (v[i] == value) return i;
|
323
318
|
}
|
324
319
|
return -1;
|
@@ -335,10 +330,11 @@ void MoveToFront(std::vector<int>* v, int index) {
|
|
335
330
|
std::vector<int> MoveToFrontTransform(const std::vector<int>& v) {
|
336
331
|
if (v.empty()) return v;
|
337
332
|
std::vector<int> mtf(*std::max_element(v.begin(), v.end()) + 1);
|
338
|
-
for (int i = 0; i < mtf.size(); ++i) mtf[i] = i;
|
333
|
+
for (int i = 0; i < static_cast<int>(mtf.size()); ++i) mtf[i] = i;
|
339
334
|
std::vector<int> result(v.size());
|
340
|
-
for (
|
335
|
+
for (size_t i = 0; i < v.size(); ++i) {
|
341
336
|
int index = IndexOf(mtf, v[i]);
|
337
|
+
assert(index >= 0);
|
342
338
|
result[i] = index;
|
343
339
|
MoveToFront(&mtf, index);
|
344
340
|
}
|
@@ -356,7 +352,7 @@ void RunLengthCodeZeros(const std::vector<int>& v_in,
|
|
356
352
|
std::vector<int>* v_out,
|
357
353
|
std::vector<int>* extra_bits) {
|
358
354
|
int max_reps = 0;
|
359
|
-
for (
|
355
|
+
for (size_t i = 0; i < v_in.size();) {
|
360
356
|
for (; i < v_in.size() && v_in[i] != 0; ++i) ;
|
361
357
|
int reps = 0;
|
362
358
|
for (; i < v_in.size() && v_in[i] == 0; ++i) {
|
@@ -366,14 +362,14 @@ void RunLengthCodeZeros(const std::vector<int>& v_in,
|
|
366
362
|
}
|
367
363
|
int max_prefix = max_reps > 0 ? Log2Floor(max_reps) : 0;
|
368
364
|
*max_run_length_prefix = std::min(max_prefix, *max_run_length_prefix);
|
369
|
-
for (
|
365
|
+
for (size_t i = 0; i < v_in.size();) {
|
370
366
|
if (v_in[i] != 0) {
|
371
367
|
v_out->push_back(v_in[i] + *max_run_length_prefix);
|
372
368
|
extra_bits->push_back(0);
|
373
369
|
++i;
|
374
370
|
} else {
|
375
371
|
int reps = 1;
|
376
|
-
for (
|
372
|
+
for (size_t k = i + 1; k < v_in.size() && v_in[k] == 0; ++k) {
|
377
373
|
++reps;
|
378
374
|
}
|
379
375
|
i += reps;
|
@@ -409,7 +405,7 @@ void EncodeContextMap(const std::vector<int>& context_map,
|
|
409
405
|
RunLengthCodeZeros(transformed_symbols, &max_run_length_prefix,
|
410
406
|
&rle_symbols, &extra_bits);
|
411
407
|
HistogramContextMap symbol_histogram;
|
412
|
-
for (
|
408
|
+
for (size_t i = 0; i < rle_symbols.size(); ++i) {
|
413
409
|
symbol_histogram.Add(rle_symbols[i]);
|
414
410
|
}
|
415
411
|
bool use_rle = max_run_length_prefix > 0;
|
@@ -424,7 +420,7 @@ void EncodeContextMap(const std::vector<int>& context_map,
|
|
424
420
|
num_clusters + max_run_length_prefix,
|
425
421
|
symbol_code.depth_, symbol_code.bits_,
|
426
422
|
storage_ix, storage);
|
427
|
-
for (
|
423
|
+
for (size_t i = 0; i < rle_symbols.size(); ++i) {
|
428
424
|
WriteBits(symbol_code.depth_[rle_symbols[i]],
|
429
425
|
symbol_code.bits_[rle_symbols[i]],
|
430
426
|
storage_ix, storage);
|
@@ -457,7 +453,7 @@ void BuildAndStoreBlockSplitCode(const std::vector<int>& types,
|
|
457
453
|
BlockSplitCode* code,
|
458
454
|
int* storage_ix,
|
459
455
|
uint8_t* storage) {
|
460
|
-
const int num_blocks = types.size();
|
456
|
+
const int num_blocks = static_cast<int>(types.size());
|
461
457
|
std::vector<int> type_histo(num_types + 2);
|
462
458
|
std::vector<int> length_histo(26);
|
463
459
|
int last_type = 1;
|
@@ -562,8 +558,8 @@ class BlockEncoder {
|
|
562
558
|
int* storage_ix, uint8_t* storage) {
|
563
559
|
depths_.resize(histograms.size() * alphabet_size_);
|
564
560
|
bits_.resize(histograms.size() * alphabet_size_);
|
565
|
-
for (
|
566
|
-
|
561
|
+
for (size_t i = 0; i < histograms.size(); ++i) {
|
562
|
+
size_t ix = i * alphabet_size_;
|
567
563
|
BuildAndStoreHuffmanTree(&histograms[i].data_[0], alphabet_size_,
|
568
564
|
&depths_[ix], &bits_[ix],
|
569
565
|
storage_ix, storage);
|
@@ -674,19 +670,21 @@ bool StoreMetaBlock(const uint8_t* input,
|
|
674
670
|
WriteBits(2, literal_context_mode, storage_ix, storage);
|
675
671
|
}
|
676
672
|
|
673
|
+
int num_literal_histograms = static_cast<int>(mb.literal_histograms.size());
|
677
674
|
if (mb.literal_context_map.empty()) {
|
678
|
-
StoreTrivialContextMap(
|
675
|
+
StoreTrivialContextMap(num_literal_histograms, kLiteralContextBits,
|
679
676
|
storage_ix, storage);
|
680
677
|
} else {
|
681
|
-
EncodeContextMap(mb.literal_context_map,
|
678
|
+
EncodeContextMap(mb.literal_context_map, num_literal_histograms,
|
682
679
|
storage_ix, storage);
|
683
680
|
}
|
684
681
|
|
682
|
+
int num_dist_histograms = static_cast<int>(mb.distance_histograms.size());
|
685
683
|
if (mb.distance_context_map.empty()) {
|
686
|
-
StoreTrivialContextMap(
|
684
|
+
StoreTrivialContextMap(num_dist_histograms, kDistanceContextBits,
|
687
685
|
storage_ix, storage);
|
688
686
|
} else {
|
689
|
-
EncodeContextMap(mb.distance_context_map,
|
687
|
+
EncodeContextMap(mb.distance_context_map, num_dist_histograms,
|
690
688
|
storage_ix, storage);
|
691
689
|
}
|
692
690
|
|
@@ -698,11 +696,11 @@ bool StoreMetaBlock(const uint8_t* input,
|
|
698
696
|
storage_ix, storage);
|
699
697
|
|
700
698
|
size_t pos = start_pos;
|
701
|
-
for (
|
699
|
+
for (size_t i = 0; i < n_commands; ++i) {
|
702
700
|
const Command cmd = commands[i];
|
703
701
|
int cmd_code = cmd.cmd_prefix_;
|
704
|
-
int lennumextra = cmd.cmd_extra_ >> 48;
|
705
|
-
uint64_t lenextra = cmd.cmd_extra_ &
|
702
|
+
int lennumextra = static_cast<int>(cmd.cmd_extra_ >> 48);
|
703
|
+
uint64_t lenextra = cmd.cmd_extra_ & 0xffffffffffffUL;
|
706
704
|
command_enc.StoreSymbol(cmd_code, storage_ix, storage);
|
707
705
|
WriteBits(lennumextra, lenextra, storage_ix, storage);
|
708
706
|
if (mb.literal_context_map.empty()) {
|
@@ -714,7 +712,7 @@ bool StoreMetaBlock(const uint8_t* input,
|
|
714
712
|
for (int j = 0; j < cmd.insert_len_; ++j) {
|
715
713
|
int context = Context(prev_byte, prev_byte2,
|
716
714
|
literal_context_mode);
|
717
|
-
|
715
|
+
uint8_t literal = input[pos & mask];
|
718
716
|
literal_enc.StoreSymbolWithContext<kLiteralContextBits>(
|
719
717
|
literal, context, mb.literal_context_map, storage_ix, storage);
|
720
718
|
prev_byte2 = prev_byte;
|
@@ -771,7 +769,7 @@ bool StoreMetaBlockTrivial(const uint8_t* input,
|
|
771
769
|
HistogramDistance dist_histo;
|
772
770
|
|
773
771
|
size_t pos = start_pos;
|
774
|
-
for (
|
772
|
+
for (size_t i = 0; i < n_commands; ++i) {
|
775
773
|
const Command cmd = commands[i];
|
776
774
|
cmd_histo.Add(cmd.cmd_prefix_);
|
777
775
|
for (int j = 0; j < cmd.insert_len_; ++j) {
|
@@ -804,11 +802,11 @@ bool StoreMetaBlockTrivial(const uint8_t* input,
|
|
804
802
|
storage_ix, storage);
|
805
803
|
|
806
804
|
pos = start_pos;
|
807
|
-
for (
|
805
|
+
for (size_t i = 0; i < n_commands; ++i) {
|
808
806
|
const Command cmd = commands[i];
|
809
807
|
const int cmd_code = cmd.cmd_prefix_;
|
810
|
-
const int lennumextra = cmd.cmd_extra_ >> 48;
|
811
|
-
const uint64_t lenextra = cmd.cmd_extra_ &
|
808
|
+
const int lennumextra = static_cast<int>(cmd.cmd_extra_ >> 48);
|
809
|
+
const uint64_t lenextra = cmd.cmd_extra_ & 0xffffffffffffUL;
|
812
810
|
WriteBits(cmd_depth[cmd_code], cmd_bits[cmd_code], storage_ix, storage);
|
813
811
|
WriteBits(lennumextra, lenextra, storage_ix, storage);
|
814
812
|
for (int j = 0; j < cmd.insert_len_; j++) {
|
@@ -849,19 +847,19 @@ bool StoreUncompressedMetaBlock(bool final_block,
|
|
849
847
|
if (masked_pos + len > mask + 1) {
|
850
848
|
size_t len1 = mask + 1 - masked_pos;
|
851
849
|
memcpy(&storage[*storage_ix >> 3], &input[masked_pos], len1);
|
852
|
-
*storage_ix += len1 << 3;
|
850
|
+
*storage_ix += static_cast<int>(len1 << 3);
|
853
851
|
len -= len1;
|
854
852
|
masked_pos = 0;
|
855
853
|
}
|
856
854
|
memcpy(&storage[*storage_ix >> 3], &input[masked_pos], len);
|
857
|
-
*storage_ix += len << 3;
|
855
|
+
*storage_ix += static_cast<int>(len << 3);
|
858
856
|
|
859
857
|
// We need to clear the next 4 bytes to continue to be
|
860
858
|
// compatible with WriteBits.
|
861
859
|
brotli::WriteBitsPrepareStorage(*storage_ix, storage);
|
862
860
|
|
863
|
-
// Since the
|
864
|
-
// one after this.
|
861
|
+
// Since the uncompressed block itself may not be the final block, add an
|
862
|
+
// empty one after this.
|
865
863
|
if (final_block) {
|
866
864
|
brotli::WriteBits(1, 1, storage_ix, storage); // islast
|
867
865
|
brotli::WriteBits(1, 1, storage_ix, storage); // isempty
|
@@ -1,17 +1,9 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2014 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Functions to convert brotli-related data structures into the
|
16
8
|
// brotli bit stream. The functions here operate under
|
17
9
|
// assumption that there is enough space in the storage, i.e., there are
|
@@ -24,11 +16,10 @@
|
|
24
16
|
#ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
25
17
|
#define BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
26
18
|
|
27
|
-
#include <stddef.h>
|
28
|
-
#include <stdint.h>
|
29
19
|
#include <vector>
|
30
20
|
|
31
21
|
#include "./metablock.h"
|
22
|
+
#include "./types.h"
|
32
23
|
|
33
24
|
namespace brotli {
|
34
25
|
|