brotli 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitmodules +1 -1
- data/.travis.yml +2 -1
- data/README.md +1 -1
- data/Rakefile +1 -1
- data/ext/brotli/brotli.cc +1 -1
- data/ext/brotli/extconf.rb +72 -14
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/LICENSE +19 -202
- data/vendor/brotli/dec/Makefile +1 -1
- data/vendor/brotli/dec/bit_reader.c +23 -30
- data/vendor/brotli/dec/bit_reader.h +270 -141
- data/vendor/brotli/dec/context.h +3 -12
- data/vendor/brotli/dec/decode.c +1813 -1048
- data/vendor/brotli/dec/decode.h +22 -16
- data/vendor/brotli/dec/dictionary.c +9466 -0
- data/vendor/brotli/dec/dictionary.h +6 -9461
- data/vendor/brotli/dec/huffman.c +104 -71
- data/vendor/brotli/dec/huffman.h +19 -28
- data/vendor/brotli/dec/port.h +124 -32
- data/vendor/brotli/dec/prefix.h +4 -13
- data/vendor/brotli/dec/state.c +93 -56
- data/vendor/brotli/dec/state.h +124 -53
- data/vendor/brotli/dec/streams.c +14 -11
- data/vendor/brotli/dec/streams.h +6 -11
- data/vendor/brotli/dec/transform.h +2 -11
- data/vendor/brotli/dec/types.h +21 -19
- data/vendor/brotli/enc/Makefile +4 -1
- data/vendor/brotli/enc/backward_references.cc +87 -94
- data/vendor/brotli/enc/backward_references.h +8 -18
- data/vendor/brotli/enc/bit_cost.h +11 -19
- data/vendor/brotli/enc/block_splitter.cc +43 -48
- data/vendor/brotli/enc/block_splitter.h +7 -16
- data/vendor/brotli/enc/brotli_bit_stream.cc +48 -50
- data/vendor/brotli/enc/brotli_bit_stream.h +7 -16
- data/vendor/brotli/enc/cluster.h +24 -25
- data/vendor/brotli/enc/command.h +34 -41
- data/vendor/brotli/enc/context.h +11 -18
- data/vendor/brotli/enc/dictionary.cc +9466 -0
- data/vendor/brotli/enc/dictionary.h +20 -9464
- data/vendor/brotli/enc/dictionary_hash.h +7 -15
- data/vendor/brotli/enc/encode.cc +80 -148
- data/vendor/brotli/enc/encode.h +19 -29
- data/vendor/brotli/enc/encode_parallel.cc +35 -108
- data/vendor/brotli/enc/encode_parallel.h +7 -16
- data/vendor/brotli/enc/entropy_encode.cc +33 -42
- data/vendor/brotli/enc/entropy_encode.h +8 -16
- data/vendor/brotli/enc/fast_log.h +8 -15
- data/vendor/brotli/enc/find_match_length.h +7 -17
- data/vendor/brotli/enc/hash.h +130 -150
- data/vendor/brotli/enc/histogram.cc +7 -16
- data/vendor/brotli/enc/histogram.h +11 -17
- data/vendor/brotli/enc/literal_cost.cc +28 -35
- data/vendor/brotli/enc/literal_cost.h +9 -23
- data/vendor/brotli/enc/metablock.cc +18 -26
- data/vendor/brotli/enc/metablock.h +6 -14
- data/vendor/brotli/enc/port.h +14 -14
- data/vendor/brotli/enc/prefix.h +11 -18
- data/vendor/brotli/enc/ringbuffer.h +18 -27
- data/vendor/brotli/enc/static_dict.cc +7 -1
- data/vendor/brotli/enc/static_dict.h +7 -15
- data/vendor/brotli/enc/static_dict_lut.h +7 -15
- data/vendor/brotli/enc/streams.cc +15 -28
- data/vendor/brotli/enc/streams.h +27 -35
- data/vendor/brotli/enc/transform.h +9 -16
- data/vendor/brotli/enc/types.h +27 -0
- data/vendor/brotli/enc/utf8_util.cc +82 -0
- data/vendor/brotli/enc/utf8_util.h +25 -0
- data/vendor/brotli/enc/write_bits.h +11 -18
- metadata +7 -2
@@ -1,17 +1,9 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Function to find backward reference copies.
|
16
8
|
|
17
9
|
#include "./backward_references.h"
|
@@ -22,6 +14,7 @@
|
|
22
14
|
|
23
15
|
#include "./command.h"
|
24
16
|
#include "./fast_log.h"
|
17
|
+
#include "./literal_cost.h"
|
25
18
|
|
26
19
|
namespace brotli {
|
27
20
|
|
@@ -30,19 +23,21 @@ static const double kInfinity = std::numeric_limits<double>::infinity();
|
|
30
23
|
// Histogram based cost model for zopflification.
|
31
24
|
class ZopfliCostModel {
|
32
25
|
public:
|
26
|
+
ZopfliCostModel() : min_cost_cmd_(kInfinity) {}
|
27
|
+
|
33
28
|
void SetFromCommands(size_t num_bytes,
|
34
29
|
size_t position,
|
35
30
|
const uint8_t* ringbuffer,
|
36
31
|
size_t ringbuffer_mask,
|
37
32
|
const Command* commands,
|
38
|
-
|
33
|
+
size_t num_commands,
|
39
34
|
int last_insert_len) {
|
40
35
|
std::vector<int> histogram_literal(256, 0);
|
41
36
|
std::vector<int> histogram_cmd(kNumCommandPrefixes, 0);
|
42
37
|
std::vector<int> histogram_dist(kNumDistancePrefixes, 0);
|
43
38
|
|
44
39
|
size_t pos = position - last_insert_len;
|
45
|
-
for (
|
40
|
+
for (size_t i = 0; i < num_commands; i++) {
|
46
41
|
int inslength = commands[i].insert_len_;
|
47
42
|
int copylength = commands[i].copy_len_;
|
48
43
|
int distcode = commands[i].dist_prefix_;
|
@@ -63,14 +58,13 @@ class ZopfliCostModel {
|
|
63
58
|
Set(histogram_cmd, &cost_cmd_);
|
64
59
|
Set(histogram_dist, &cost_dist_);
|
65
60
|
|
66
|
-
min_cost_cmd_ = kInfinity;
|
67
61
|
for (int i = 0; i < kNumCommandPrefixes; ++i) {
|
68
62
|
min_cost_cmd_ = std::min(min_cost_cmd_, cost_cmd_[i]);
|
69
63
|
}
|
70
64
|
|
71
65
|
literal_costs_.resize(num_bytes + 1);
|
72
66
|
literal_costs_[0] = 0.0;
|
73
|
-
for (
|
67
|
+
for (size_t i = 0; i < num_bytes; ++i) {
|
74
68
|
literal_costs_[i + 1] = literal_costs_[i] +
|
75
69
|
cost_literal[ringbuffer[(position + i) & ringbuffer_mask]];
|
76
70
|
}
|
@@ -78,19 +72,15 @@ class ZopfliCostModel {
|
|
78
72
|
|
79
73
|
void SetFromLiteralCosts(size_t num_bytes,
|
80
74
|
size_t position,
|
81
|
-
const
|
82
|
-
size_t
|
75
|
+
const uint8_t* ringbuffer,
|
76
|
+
size_t ringbuffer_mask) {
|
77
|
+
std::vector<float> literal_cost(num_bytes + 1);
|
78
|
+
EstimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask,
|
79
|
+
ringbuffer, &literal_cost[0]);
|
83
80
|
literal_costs_.resize(num_bytes + 1);
|
84
81
|
literal_costs_[0] = 0.0;
|
85
|
-
|
86
|
-
|
87
|
-
literal_costs_[i + 1] = literal_costs_[i] +
|
88
|
-
literal_cost[(position + i) & literal_cost_mask];
|
89
|
-
}
|
90
|
-
} else {
|
91
|
-
for (int i = 1; i <= num_bytes; ++i) {
|
92
|
-
literal_costs_[i] = i * 5.4;
|
93
|
-
}
|
82
|
+
for (size_t i = 0; i < num_bytes; ++i) {
|
83
|
+
literal_costs_[i + 1] = literal_costs_[i] + literal_cost[i];
|
94
84
|
}
|
95
85
|
cost_cmd_.resize(kNumCommandPrefixes);
|
96
86
|
cost_dist_.resize(kNumDistancePrefixes);
|
@@ -105,23 +95,21 @@ class ZopfliCostModel {
|
|
105
95
|
|
106
96
|
double GetCommandCost(
|
107
97
|
int dist_code, int length_code, int insert_length) const {
|
108
|
-
|
109
|
-
|
110
|
-
uint16_t cmdcode = CombineLengthCodes(inscode, copycode, dist_code);
|
111
|
-
uint64_t insnumextra = insextra[inscode];
|
112
|
-
uint64_t copynumextra = copyextra[copycode];
|
98
|
+
uint16_t inscode = GetInsertLengthCode(insert_length);
|
99
|
+
uint16_t copycode = GetCopyLengthCode(length_code);
|
100
|
+
uint16_t cmdcode = CombineLengthCodes(inscode, copycode, dist_code == 0);
|
113
101
|
uint16_t dist_symbol;
|
114
102
|
uint32_t distextra;
|
115
103
|
PrefixEncodeCopyDistance(dist_code, 0, 0, &dist_symbol, &distextra);
|
116
104
|
uint32_t distnumextra = distextra >> 24;
|
117
105
|
|
118
|
-
double result =
|
106
|
+
double result = insextra[inscode] + copyextra[copycode] + distnumextra;
|
119
107
|
result += cost_cmd_[cmdcode];
|
120
108
|
if (cmdcode >= 128) result += cost_dist_[dist_symbol];
|
121
109
|
return result;
|
122
110
|
}
|
123
111
|
|
124
|
-
double GetLiteralCosts(
|
112
|
+
double GetLiteralCosts(size_t from, size_t to) const {
|
125
113
|
return literal_costs_[to] - literal_costs_[from];
|
126
114
|
}
|
127
115
|
|
@@ -233,7 +221,7 @@ inline void UpdateZopfliNode(ZopfliNode* nodes, size_t pos, size_t start_pos,
|
|
233
221
|
next.length_code = len_code;
|
234
222
|
next.distance = dist;
|
235
223
|
next.distance_code = dist_code;
|
236
|
-
next.insert_length = pos - start_pos;
|
224
|
+
next.insert_length = static_cast<int>(pos - start_pos);
|
237
225
|
next.cost = cost;
|
238
226
|
SetDistanceCache(dist, dist_code, max_dist, dist_cache,
|
239
227
|
&next.distance_cache[0]);
|
@@ -266,7 +254,7 @@ class StartPosQueue {
|
|
266
254
|
++idx_;
|
267
255
|
}
|
268
256
|
|
269
|
-
int size() const { return std::min
|
257
|
+
int size() const { return std::min(idx_, mask_ + 1); }
|
270
258
|
|
271
259
|
size_t GetStartPos(int k) const {
|
272
260
|
return q_[(idx_ - k - 1) & mask_].first;
|
@@ -320,7 +308,7 @@ void ZopfliIterate(size_t num_bytes,
|
|
320
308
|
int* dist_cache,
|
321
309
|
int* last_insert_len,
|
322
310
|
Command* commands,
|
323
|
-
|
311
|
+
size_t* num_commands,
|
324
312
|
int* num_literals) {
|
325
313
|
const Command * const orig_commands = commands;
|
326
314
|
|
@@ -336,8 +324,8 @@ void ZopfliIterate(size_t num_bytes,
|
|
336
324
|
for (size_t i = 0; i + 3 < num_bytes; i++) {
|
337
325
|
size_t cur_ix = position + i;
|
338
326
|
size_t cur_ix_masked = cur_ix & ringbuffer_mask;
|
339
|
-
|
340
|
-
int max_length = num_bytes - i;
|
327
|
+
int max_distance = static_cast<int>(std::min(cur_ix, max_backward_limit));
|
328
|
+
int max_length = static_cast<int>(num_bytes - i);
|
341
329
|
|
342
330
|
queue.Push(i, nodes[i].cost - model.GetLiteralCosts(0, i));
|
343
331
|
|
@@ -346,7 +334,7 @@ void ZopfliIterate(size_t num_bytes,
|
|
346
334
|
|
347
335
|
// Go over the command starting positions in order of increasing cost
|
348
336
|
// difference.
|
349
|
-
for (
|
337
|
+
for (int k = 0; k < 5 && k < queue.size(); ++k) {
|
350
338
|
const size_t start = queue.GetStartPos(k);
|
351
339
|
const double start_costdiff =
|
352
340
|
nodes[start].cost - model.GetLiteralCosts(0, start);
|
@@ -373,12 +361,13 @@ void ZopfliIterate(size_t num_bytes,
|
|
373
361
|
ringbuffer[prev_ix + best_len]) {
|
374
362
|
continue;
|
375
363
|
}
|
376
|
-
const
|
364
|
+
const int len =
|
377
365
|
FindMatchLengthWithLimit(&ringbuffer[prev_ix],
|
378
366
|
&ringbuffer[cur_ix_masked],
|
379
367
|
max_length);
|
380
368
|
for (int l = best_len + 1; l <= len; ++l) {
|
381
|
-
|
369
|
+
const int inslen = static_cast<int>(i - start);
|
370
|
+
double cmd_cost = model.GetCommandCost(j, l, inslen);
|
382
371
|
double cost = start_costdiff + cmd_cost + model.GetLiteralCosts(0, i);
|
383
372
|
if (cost < nodes[i + l].cost) {
|
384
373
|
UpdateZopfliNode(&nodes[0], i, start, l, l, backward, j,
|
@@ -411,8 +400,8 @@ void ZopfliIterate(size_t num_bytes,
|
|
411
400
|
}
|
412
401
|
for (; len <= max_len; ++len) {
|
413
402
|
int len_code = is_dictionary_match ? match.length_code() : len;
|
414
|
-
|
415
|
-
|
403
|
+
const int inslen = static_cast<int>(i - start);
|
404
|
+
double cmd_cost = model.GetCommandCost(dist_code, len_code, inslen);
|
416
405
|
double cost = start_costdiff + cmd_cost + model.GetLiteralCosts(0, i);
|
417
406
|
if (cost < nodes[i + len].cost) {
|
418
407
|
UpdateZopfliNode(&nodes[0], i, start, len, len_code, dist,
|
@@ -459,7 +448,8 @@ void ZopfliIterate(size_t num_bytes,
|
|
459
448
|
}
|
460
449
|
int distance = next.distance;
|
461
450
|
int len_code = next.length_code;
|
462
|
-
|
451
|
+
int max_distance =
|
452
|
+
static_cast<int>(std::min(position + pos, max_backward_limit));
|
463
453
|
bool is_dictionary = (distance > max_distance);
|
464
454
|
int dist_code = next.distance_code;
|
465
455
|
|
@@ -477,7 +467,7 @@ void ZopfliIterate(size_t num_bytes,
|
|
477
467
|
insert_length = 0;
|
478
468
|
pos += copy_length;
|
479
469
|
}
|
480
|
-
*last_insert_len += num_bytes - pos;
|
470
|
+
*last_insert_len += static_cast<int>(num_bytes - pos);
|
481
471
|
*num_commands += (commands - orig_commands);
|
482
472
|
}
|
483
473
|
|
@@ -492,42 +482,43 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
492
482
|
int* dist_cache,
|
493
483
|
int* last_insert_len,
|
494
484
|
Command* commands,
|
495
|
-
|
485
|
+
size_t* num_commands,
|
496
486
|
int* num_literals) {
|
497
487
|
if (num_bytes >= 3 && position >= 3) {
|
498
488
|
// Prepare the hashes for three last bytes of the last write.
|
499
489
|
// These could not be calculated before, since they require knowledge
|
500
490
|
// of both the previous and the current block.
|
501
491
|
hasher->Store(&ringbuffer[(position - 3) & ringbuffer_mask],
|
502
|
-
position - 3);
|
492
|
+
static_cast<uint32_t>(position - 3));
|
503
493
|
hasher->Store(&ringbuffer[(position - 2) & ringbuffer_mask],
|
504
|
-
position - 2);
|
494
|
+
static_cast<uint32_t>(position - 2));
|
505
495
|
hasher->Store(&ringbuffer[(position - 1) & ringbuffer_mask],
|
506
|
-
position - 1);
|
496
|
+
static_cast<uint32_t>(position - 1));
|
507
497
|
}
|
508
498
|
const Command * const orig_commands = commands;
|
509
499
|
int insert_length = *last_insert_len;
|
510
500
|
size_t i = position & ringbuffer_mask;
|
511
|
-
const
|
501
|
+
const size_t i_diff = position - i;
|
512
502
|
const size_t i_end = i + num_bytes;
|
513
503
|
|
514
504
|
// For speed up heuristics for random data.
|
515
|
-
const
|
516
|
-
|
505
|
+
const size_t random_heuristics_window_size = quality < 9 ? 64 : 512;
|
506
|
+
size_t apply_random_heuristics = i + random_heuristics_window_size;
|
517
507
|
|
518
508
|
// Minimum score to accept a backward reference.
|
519
509
|
const int kMinScore = 4.0;
|
520
510
|
|
521
511
|
while (i + Hasher::kHashTypeLength - 1 < i_end) {
|
522
|
-
int max_length = i_end - i;
|
523
|
-
|
512
|
+
int max_length = static_cast<int>(i_end - i);
|
513
|
+
int max_distance =
|
514
|
+
static_cast<int>(std::min(i + i_diff, max_backward_limit));
|
524
515
|
int best_len = 0;
|
525
516
|
int best_len_code = 0;
|
526
517
|
int best_dist = 0;
|
527
518
|
double best_score = kMinScore;
|
528
519
|
bool match_found = hasher->FindLongestMatch(
|
529
520
|
ringbuffer, ringbuffer_mask,
|
530
|
-
dist_cache, i + i_diff, max_length, max_distance,
|
521
|
+
dist_cache, static_cast<uint32_t>(i + i_diff), max_length, max_distance,
|
531
522
|
&best_len, &best_len_code, &best_dist, &best_score);
|
532
523
|
if (match_found) {
|
533
524
|
// Found a match. Let's look for something even better ahead.
|
@@ -538,11 +529,13 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
538
529
|
int best_len_code_2 = 0;
|
539
530
|
int best_dist_2 = 0;
|
540
531
|
double best_score_2 = kMinScore;
|
541
|
-
max_distance =
|
542
|
-
|
532
|
+
max_distance =
|
533
|
+
static_cast<int>(std::min(i + i_diff + 1, max_backward_limit));
|
534
|
+
hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
|
543
535
|
match_found = hasher->FindLongestMatch(
|
544
536
|
ringbuffer, ringbuffer_mask,
|
545
|
-
dist_cache, i + i_diff + 1,
|
537
|
+
dist_cache, static_cast<uint32_t>(i + i_diff + 1),
|
538
|
+
max_length, max_distance,
|
546
539
|
&best_len_2, &best_len_code_2, &best_dist_2, &best_score_2);
|
547
540
|
double cost_diff_lazy = 7.0;
|
548
541
|
if (match_found && best_score_2 >= best_score + cost_diff_lazy) {
|
@@ -562,7 +555,7 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
562
555
|
}
|
563
556
|
apply_random_heuristics =
|
564
557
|
i + 2 * best_len + random_heuristics_window_size;
|
565
|
-
max_distance = std::min(i + i_diff, max_backward_limit);
|
558
|
+
max_distance = static_cast<int>(std::min(i + i_diff, max_backward_limit));
|
566
559
|
// The first 16 codes are special shortcodes, and the minimum offset is 1.
|
567
560
|
int distance_code =
|
568
561
|
ComputeDistanceCode(best_dist, max_distance, quality, dist_cache);
|
@@ -579,12 +572,13 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
579
572
|
// Put the hash keys into the table, if there are enough
|
580
573
|
// bytes left.
|
581
574
|
for (int j = 1; j < best_len; ++j) {
|
582
|
-
hasher->Store(&ringbuffer[i + j],
|
575
|
+
hasher->Store(&ringbuffer[i + j],
|
576
|
+
static_cast<uint32_t>(i + i_diff + j));
|
583
577
|
}
|
584
578
|
i += best_len;
|
585
579
|
} else {
|
586
580
|
++insert_length;
|
587
|
-
hasher->Store(ringbuffer + i, i + i_diff);
|
581
|
+
hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
|
588
582
|
++i;
|
589
583
|
// If we have not seen matches for a long time, we can skip some
|
590
584
|
// match lookups. Unsuccessful match lookups are very very expensive
|
@@ -599,32 +593,30 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
599
593
|
// turn out to be useful in the future, too, so we store less of
|
600
594
|
// them to not to flood out the hash table of good compressible
|
601
595
|
// data.
|
602
|
-
|
596
|
+
size_t i_jump = std::min(i + 16, i_end - 4);
|
603
597
|
for (; i < i_jump; i += 4) {
|
604
|
-
hasher->Store(ringbuffer + i, i + i_diff);
|
598
|
+
hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
|
605
599
|
insert_length += 4;
|
606
600
|
}
|
607
601
|
} else {
|
608
|
-
|
602
|
+
size_t i_jump = std::min(i + 8, i_end - 3);
|
609
603
|
for (; i < i_jump; i += 2) {
|
610
|
-
hasher->Store(ringbuffer + i, i + i_diff);
|
604
|
+
hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
|
611
605
|
insert_length += 2;
|
612
606
|
}
|
613
607
|
}
|
614
608
|
}
|
615
609
|
}
|
616
610
|
}
|
617
|
-
insert_length += (i_end - i);
|
611
|
+
insert_length += static_cast<int>(i_end - i);
|
618
612
|
*last_insert_len = insert_length;
|
619
|
-
*num_commands +=
|
613
|
+
*num_commands += commands - orig_commands;
|
620
614
|
}
|
621
615
|
|
622
616
|
void CreateBackwardReferences(size_t num_bytes,
|
623
617
|
size_t position,
|
624
618
|
const uint8_t* ringbuffer,
|
625
619
|
size_t ringbuffer_mask,
|
626
|
-
const float* literal_cost,
|
627
|
-
size_t literal_cost_mask,
|
628
620
|
const size_t max_backward_limit,
|
629
621
|
const int quality,
|
630
622
|
Hashers* hashers,
|
@@ -632,46 +624,47 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
632
624
|
int* dist_cache,
|
633
625
|
int* last_insert_len,
|
634
626
|
Command* commands,
|
635
|
-
|
627
|
+
size_t* num_commands,
|
636
628
|
int* num_literals) {
|
637
629
|
bool zopflify = quality > 9;
|
638
630
|
if (zopflify) {
|
639
|
-
Hashers::H9* hasher = hashers->hash_h9
|
631
|
+
Hashers::H9* hasher = hashers->hash_h9;
|
640
632
|
if (num_bytes >= 3 && position >= 3) {
|
641
633
|
// Prepare the hashes for three last bytes of the last write.
|
642
634
|
// These could not be calculated before, since they require knowledge
|
643
635
|
// of both the previous and the current block.
|
644
636
|
hasher->Store(&ringbuffer[(position - 3) & ringbuffer_mask],
|
645
|
-
position - 3);
|
637
|
+
static_cast<uint32_t>(position - 3));
|
646
638
|
hasher->Store(&ringbuffer[(position - 2) & ringbuffer_mask],
|
647
|
-
position - 2);
|
639
|
+
static_cast<uint32_t>(position - 2));
|
648
640
|
hasher->Store(&ringbuffer[(position - 1) & ringbuffer_mask],
|
649
|
-
position - 1);
|
641
|
+
static_cast<uint32_t>(position - 1));
|
650
642
|
}
|
651
643
|
std::vector<int> num_matches(num_bytes);
|
652
644
|
std::vector<BackwardMatch> matches(3 * num_bytes);
|
653
645
|
size_t cur_match_pos = 0;
|
654
646
|
for (size_t i = 0; i + 3 < num_bytes; ++i) {
|
655
|
-
|
656
|
-
|
647
|
+
int max_distance =
|
648
|
+
static_cast<int>(std::min(position + i, max_backward_limit));
|
649
|
+
int max_length = static_cast<int>(num_bytes - i);
|
657
650
|
// Ensure that we have at least kMaxZopfliLen free slots.
|
658
651
|
if (matches.size() < cur_match_pos + kMaxZopfliLen) {
|
659
652
|
matches.resize(cur_match_pos + kMaxZopfliLen);
|
660
653
|
}
|
661
654
|
hasher->FindAllMatches(
|
662
655
|
ringbuffer, ringbuffer_mask,
|
663
|
-
position + i, max_length, max_distance,
|
656
|
+
static_cast<uint32_t>(position + i), max_length, max_distance,
|
664
657
|
&num_matches[i], &matches[cur_match_pos]);
|
665
658
|
hasher->Store(&ringbuffer[(position + i) & ringbuffer_mask],
|
666
|
-
position + i);
|
659
|
+
static_cast<uint32_t>(position + i));
|
667
660
|
cur_match_pos += num_matches[i];
|
668
661
|
if (num_matches[i] == 1) {
|
669
662
|
const int match_len = matches[cur_match_pos - 1].length();
|
670
663
|
if (match_len > kMaxZopfliLen) {
|
671
664
|
for (int j = 1; j < match_len; ++j) {
|
672
665
|
++i;
|
673
|
-
hasher->Store(
|
674
|
-
|
666
|
+
hasher->Store(&ringbuffer[(position + i) & ringbuffer_mask],
|
667
|
+
static_cast<uint32_t>(position + i));
|
675
668
|
num_matches[i] = 0;
|
676
669
|
}
|
677
670
|
}
|
@@ -682,13 +675,13 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
682
675
|
int orig_dist_cache[4] = {
|
683
676
|
dist_cache[0], dist_cache[1], dist_cache[2], dist_cache[3]
|
684
677
|
};
|
685
|
-
|
678
|
+
size_t orig_num_commands = *num_commands;
|
686
679
|
static const int kIterations = 2;
|
687
680
|
for (int i = 0; i < kIterations; i++) {
|
688
681
|
ZopfliCostModel model;
|
689
682
|
if (i == 0) {
|
690
683
|
model.SetFromLiteralCosts(num_bytes, position,
|
691
|
-
|
684
|
+
ringbuffer, ringbuffer_mask);
|
692
685
|
} else {
|
693
686
|
model.SetFromCommands(num_bytes, position,
|
694
687
|
ringbuffer, ringbuffer_mask,
|
@@ -710,55 +703,55 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
710
703
|
case 1:
|
711
704
|
CreateBackwardReferences<Hashers::H1>(
|
712
705
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
713
|
-
quality, hashers->hash_h1
|
706
|
+
quality, hashers->hash_h1, dist_cache, last_insert_len,
|
714
707
|
commands, num_commands, num_literals);
|
715
708
|
break;
|
716
709
|
case 2:
|
717
710
|
CreateBackwardReferences<Hashers::H2>(
|
718
711
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
719
|
-
quality, hashers->hash_h2
|
712
|
+
quality, hashers->hash_h2, dist_cache, last_insert_len,
|
720
713
|
commands, num_commands, num_literals);
|
721
714
|
break;
|
722
715
|
case 3:
|
723
716
|
CreateBackwardReferences<Hashers::H3>(
|
724
717
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
725
|
-
quality, hashers->hash_h3
|
718
|
+
quality, hashers->hash_h3, dist_cache, last_insert_len,
|
726
719
|
commands, num_commands, num_literals);
|
727
720
|
break;
|
728
721
|
case 4:
|
729
722
|
CreateBackwardReferences<Hashers::H4>(
|
730
723
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
731
|
-
quality, hashers->hash_h4
|
724
|
+
quality, hashers->hash_h4, dist_cache, last_insert_len,
|
732
725
|
commands, num_commands, num_literals);
|
733
726
|
break;
|
734
727
|
case 5:
|
735
728
|
CreateBackwardReferences<Hashers::H5>(
|
736
729
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
737
|
-
quality, hashers->hash_h5
|
730
|
+
quality, hashers->hash_h5, dist_cache, last_insert_len,
|
738
731
|
commands, num_commands, num_literals);
|
739
732
|
break;
|
740
733
|
case 6:
|
741
734
|
CreateBackwardReferences<Hashers::H6>(
|
742
735
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
743
|
-
quality, hashers->hash_h6
|
736
|
+
quality, hashers->hash_h6, dist_cache, last_insert_len,
|
744
737
|
commands, num_commands, num_literals);
|
745
738
|
break;
|
746
739
|
case 7:
|
747
740
|
CreateBackwardReferences<Hashers::H7>(
|
748
741
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
749
|
-
quality, hashers->hash_h7
|
742
|
+
quality, hashers->hash_h7, dist_cache, last_insert_len,
|
750
743
|
commands, num_commands, num_literals);
|
751
744
|
break;
|
752
745
|
case 8:
|
753
746
|
CreateBackwardReferences<Hashers::H8>(
|
754
747
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
755
|
-
quality, hashers->hash_h8
|
748
|
+
quality, hashers->hash_h8, dist_cache, last_insert_len,
|
756
749
|
commands, num_commands, num_literals);
|
757
750
|
break;
|
758
751
|
case 9:
|
759
752
|
CreateBackwardReferences<Hashers::H9>(
|
760
753
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
761
|
-
quality, hashers->hash_h9
|
754
|
+
quality, hashers->hash_h9, dist_cache, last_insert_len,
|
762
755
|
commands, num_commands, num_literals);
|
763
756
|
break;
|
764
757
|
default:
|
@@ -1,27 +1,19 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Function to find backward reference copies.
|
16
8
|
|
17
9
|
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_
|
18
10
|
#define BROTLI_ENC_BACKWARD_REFERENCES_H_
|
19
11
|
|
20
|
-
#include <stdint.h>
|
21
12
|
#include <vector>
|
22
13
|
|
23
14
|
#include "./hash.h"
|
24
15
|
#include "./command.h"
|
16
|
+
#include "./types.h"
|
25
17
|
|
26
18
|
namespace brotli {
|
27
19
|
|
@@ -33,8 +25,6 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
33
25
|
size_t position,
|
34
26
|
const uint8_t* ringbuffer,
|
35
27
|
size_t ringbuffer_mask,
|
36
|
-
const float* literal_cost,
|
37
|
-
size_t literal_cost_mask,
|
38
28
|
const size_t max_backward_limit,
|
39
29
|
const int quality,
|
40
30
|
Hashers* hashers,
|
@@ -42,7 +32,7 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
42
32
|
int* dist_cache,
|
43
33
|
int* last_insert_len,
|
44
34
|
Command* commands,
|
45
|
-
|
35
|
+
size_t* num_commands,
|
46
36
|
int* num_literals);
|
47
37
|
|
48
38
|
} // namespace brotli
|
@@ -1,27 +1,19 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Functions to estimate the bit cost of Huffman trees.
|
16
8
|
|
17
9
|
#ifndef BROTLI_ENC_BIT_COST_H_
|
18
10
|
#define BROTLI_ENC_BIT_COST_H_
|
19
11
|
|
20
12
|
|
21
|
-
#include <stdint.h>
|
22
13
|
|
23
14
|
#include "./entropy_encode.h"
|
24
15
|
#include "./fast_log.h"
|
16
|
+
#include "./types.h"
|
25
17
|
|
26
18
|
namespace brotli {
|
27
19
|
|
@@ -77,12 +69,12 @@ double PopulationCost(const Histogram<kSize>& histogram) {
|
|
77
69
|
return 20 + histogram.total_count_;
|
78
70
|
}
|
79
71
|
double bits = 0;
|
80
|
-
uint8_t
|
72
|
+
uint8_t depth_array[kSize] = { 0 };
|
81
73
|
if (count <= 4) {
|
82
74
|
// For very low symbol count we build the Huffman tree.
|
83
|
-
CreateHuffmanTree(&histogram.data_[0], kSize, 15,
|
75
|
+
CreateHuffmanTree(&histogram.data_[0], kSize, 15, depth_array);
|
84
76
|
for (int i = 0; i < kSize; ++i) {
|
85
|
-
bits += histogram.data_[i] *
|
77
|
+
bits += histogram.data_[i] * depth_array[i];
|
86
78
|
}
|
87
79
|
return count == 3 ? bits + 28 : bits + 37;
|
88
80
|
}
|
@@ -110,7 +102,7 @@ double PopulationCost(const Histogram<kSize>& histogram) {
|
|
110
102
|
++depth_histo[depth];
|
111
103
|
++i;
|
112
104
|
} else {
|
113
|
-
// Compute the run length of zeros and add the
|
105
|
+
// Compute the run length of zeros and add the appropriate number of 0 and
|
114
106
|
// 17 code length codes to the code length code histogram.
|
115
107
|
int reps = 1;
|
116
108
|
for (int k = i + 1; k < kSize && histogram.data_[k] == 0; ++k) {
|