brotli 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitmodules +1 -1
- data/.travis.yml +2 -1
- data/README.md +1 -1
- data/Rakefile +1 -1
- data/ext/brotli/brotli.cc +1 -1
- data/ext/brotli/extconf.rb +72 -14
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/LICENSE +19 -202
- data/vendor/brotli/dec/Makefile +1 -1
- data/vendor/brotli/dec/bit_reader.c +23 -30
- data/vendor/brotli/dec/bit_reader.h +270 -141
- data/vendor/brotli/dec/context.h +3 -12
- data/vendor/brotli/dec/decode.c +1813 -1048
- data/vendor/brotli/dec/decode.h +22 -16
- data/vendor/brotli/dec/dictionary.c +9466 -0
- data/vendor/brotli/dec/dictionary.h +6 -9461
- data/vendor/brotli/dec/huffman.c +104 -71
- data/vendor/brotli/dec/huffman.h +19 -28
- data/vendor/brotli/dec/port.h +124 -32
- data/vendor/brotli/dec/prefix.h +4 -13
- data/vendor/brotli/dec/state.c +93 -56
- data/vendor/brotli/dec/state.h +124 -53
- data/vendor/brotli/dec/streams.c +14 -11
- data/vendor/brotli/dec/streams.h +6 -11
- data/vendor/brotli/dec/transform.h +2 -11
- data/vendor/brotli/dec/types.h +21 -19
- data/vendor/brotli/enc/Makefile +4 -1
- data/vendor/brotli/enc/backward_references.cc +87 -94
- data/vendor/brotli/enc/backward_references.h +8 -18
- data/vendor/brotli/enc/bit_cost.h +11 -19
- data/vendor/brotli/enc/block_splitter.cc +43 -48
- data/vendor/brotli/enc/block_splitter.h +7 -16
- data/vendor/brotli/enc/brotli_bit_stream.cc +48 -50
- data/vendor/brotli/enc/brotli_bit_stream.h +7 -16
- data/vendor/brotli/enc/cluster.h +24 -25
- data/vendor/brotli/enc/command.h +34 -41
- data/vendor/brotli/enc/context.h +11 -18
- data/vendor/brotli/enc/dictionary.cc +9466 -0
- data/vendor/brotli/enc/dictionary.h +20 -9464
- data/vendor/brotli/enc/dictionary_hash.h +7 -15
- data/vendor/brotli/enc/encode.cc +80 -148
- data/vendor/brotli/enc/encode.h +19 -29
- data/vendor/brotli/enc/encode_parallel.cc +35 -108
- data/vendor/brotli/enc/encode_parallel.h +7 -16
- data/vendor/brotli/enc/entropy_encode.cc +33 -42
- data/vendor/brotli/enc/entropy_encode.h +8 -16
- data/vendor/brotli/enc/fast_log.h +8 -15
- data/vendor/brotli/enc/find_match_length.h +7 -17
- data/vendor/brotli/enc/hash.h +130 -150
- data/vendor/brotli/enc/histogram.cc +7 -16
- data/vendor/brotli/enc/histogram.h +11 -17
- data/vendor/brotli/enc/literal_cost.cc +28 -35
- data/vendor/brotli/enc/literal_cost.h +9 -23
- data/vendor/brotli/enc/metablock.cc +18 -26
- data/vendor/brotli/enc/metablock.h +6 -14
- data/vendor/brotli/enc/port.h +14 -14
- data/vendor/brotli/enc/prefix.h +11 -18
- data/vendor/brotli/enc/ringbuffer.h +18 -27
- data/vendor/brotli/enc/static_dict.cc +7 -1
- data/vendor/brotli/enc/static_dict.h +7 -15
- data/vendor/brotli/enc/static_dict_lut.h +7 -15
- data/vendor/brotli/enc/streams.cc +15 -28
- data/vendor/brotli/enc/streams.h +27 -35
- data/vendor/brotli/enc/transform.h +9 -16
- data/vendor/brotli/enc/types.h +27 -0
- data/vendor/brotli/enc/utf8_util.cc +82 -0
- data/vendor/brotli/enc/utf8_util.h +25 -0
- data/vendor/brotli/enc/write_bits.h +11 -18
- metadata +7 -2
@@ -1,17 +1,9 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Function to find backward reference copies.
|
16
8
|
|
17
9
|
#include "./backward_references.h"
|
@@ -22,6 +14,7 @@
|
|
22
14
|
|
23
15
|
#include "./command.h"
|
24
16
|
#include "./fast_log.h"
|
17
|
+
#include "./literal_cost.h"
|
25
18
|
|
26
19
|
namespace brotli {
|
27
20
|
|
@@ -30,19 +23,21 @@ static const double kInfinity = std::numeric_limits<double>::infinity();
|
|
30
23
|
// Histogram based cost model for zopflification.
|
31
24
|
class ZopfliCostModel {
|
32
25
|
public:
|
26
|
+
ZopfliCostModel() : min_cost_cmd_(kInfinity) {}
|
27
|
+
|
33
28
|
void SetFromCommands(size_t num_bytes,
|
34
29
|
size_t position,
|
35
30
|
const uint8_t* ringbuffer,
|
36
31
|
size_t ringbuffer_mask,
|
37
32
|
const Command* commands,
|
38
|
-
|
33
|
+
size_t num_commands,
|
39
34
|
int last_insert_len) {
|
40
35
|
std::vector<int> histogram_literal(256, 0);
|
41
36
|
std::vector<int> histogram_cmd(kNumCommandPrefixes, 0);
|
42
37
|
std::vector<int> histogram_dist(kNumDistancePrefixes, 0);
|
43
38
|
|
44
39
|
size_t pos = position - last_insert_len;
|
45
|
-
for (
|
40
|
+
for (size_t i = 0; i < num_commands; i++) {
|
46
41
|
int inslength = commands[i].insert_len_;
|
47
42
|
int copylength = commands[i].copy_len_;
|
48
43
|
int distcode = commands[i].dist_prefix_;
|
@@ -63,14 +58,13 @@ class ZopfliCostModel {
|
|
63
58
|
Set(histogram_cmd, &cost_cmd_);
|
64
59
|
Set(histogram_dist, &cost_dist_);
|
65
60
|
|
66
|
-
min_cost_cmd_ = kInfinity;
|
67
61
|
for (int i = 0; i < kNumCommandPrefixes; ++i) {
|
68
62
|
min_cost_cmd_ = std::min(min_cost_cmd_, cost_cmd_[i]);
|
69
63
|
}
|
70
64
|
|
71
65
|
literal_costs_.resize(num_bytes + 1);
|
72
66
|
literal_costs_[0] = 0.0;
|
73
|
-
for (
|
67
|
+
for (size_t i = 0; i < num_bytes; ++i) {
|
74
68
|
literal_costs_[i + 1] = literal_costs_[i] +
|
75
69
|
cost_literal[ringbuffer[(position + i) & ringbuffer_mask]];
|
76
70
|
}
|
@@ -78,19 +72,15 @@ class ZopfliCostModel {
|
|
78
72
|
|
79
73
|
void SetFromLiteralCosts(size_t num_bytes,
|
80
74
|
size_t position,
|
81
|
-
const
|
82
|
-
size_t
|
75
|
+
const uint8_t* ringbuffer,
|
76
|
+
size_t ringbuffer_mask) {
|
77
|
+
std::vector<float> literal_cost(num_bytes + 1);
|
78
|
+
EstimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask,
|
79
|
+
ringbuffer, &literal_cost[0]);
|
83
80
|
literal_costs_.resize(num_bytes + 1);
|
84
81
|
literal_costs_[0] = 0.0;
|
85
|
-
|
86
|
-
|
87
|
-
literal_costs_[i + 1] = literal_costs_[i] +
|
88
|
-
literal_cost[(position + i) & literal_cost_mask];
|
89
|
-
}
|
90
|
-
} else {
|
91
|
-
for (int i = 1; i <= num_bytes; ++i) {
|
92
|
-
literal_costs_[i] = i * 5.4;
|
93
|
-
}
|
82
|
+
for (size_t i = 0; i < num_bytes; ++i) {
|
83
|
+
literal_costs_[i + 1] = literal_costs_[i] + literal_cost[i];
|
94
84
|
}
|
95
85
|
cost_cmd_.resize(kNumCommandPrefixes);
|
96
86
|
cost_dist_.resize(kNumDistancePrefixes);
|
@@ -105,23 +95,21 @@ class ZopfliCostModel {
|
|
105
95
|
|
106
96
|
double GetCommandCost(
|
107
97
|
int dist_code, int length_code, int insert_length) const {
|
108
|
-
|
109
|
-
|
110
|
-
uint16_t cmdcode = CombineLengthCodes(inscode, copycode, dist_code);
|
111
|
-
uint64_t insnumextra = insextra[inscode];
|
112
|
-
uint64_t copynumextra = copyextra[copycode];
|
98
|
+
uint16_t inscode = GetInsertLengthCode(insert_length);
|
99
|
+
uint16_t copycode = GetCopyLengthCode(length_code);
|
100
|
+
uint16_t cmdcode = CombineLengthCodes(inscode, copycode, dist_code == 0);
|
113
101
|
uint16_t dist_symbol;
|
114
102
|
uint32_t distextra;
|
115
103
|
PrefixEncodeCopyDistance(dist_code, 0, 0, &dist_symbol, &distextra);
|
116
104
|
uint32_t distnumextra = distextra >> 24;
|
117
105
|
|
118
|
-
double result =
|
106
|
+
double result = insextra[inscode] + copyextra[copycode] + distnumextra;
|
119
107
|
result += cost_cmd_[cmdcode];
|
120
108
|
if (cmdcode >= 128) result += cost_dist_[dist_symbol];
|
121
109
|
return result;
|
122
110
|
}
|
123
111
|
|
124
|
-
double GetLiteralCosts(
|
112
|
+
double GetLiteralCosts(size_t from, size_t to) const {
|
125
113
|
return literal_costs_[to] - literal_costs_[from];
|
126
114
|
}
|
127
115
|
|
@@ -233,7 +221,7 @@ inline void UpdateZopfliNode(ZopfliNode* nodes, size_t pos, size_t start_pos,
|
|
233
221
|
next.length_code = len_code;
|
234
222
|
next.distance = dist;
|
235
223
|
next.distance_code = dist_code;
|
236
|
-
next.insert_length = pos - start_pos;
|
224
|
+
next.insert_length = static_cast<int>(pos - start_pos);
|
237
225
|
next.cost = cost;
|
238
226
|
SetDistanceCache(dist, dist_code, max_dist, dist_cache,
|
239
227
|
&next.distance_cache[0]);
|
@@ -266,7 +254,7 @@ class StartPosQueue {
|
|
266
254
|
++idx_;
|
267
255
|
}
|
268
256
|
|
269
|
-
int size() const { return std::min
|
257
|
+
int size() const { return std::min(idx_, mask_ + 1); }
|
270
258
|
|
271
259
|
size_t GetStartPos(int k) const {
|
272
260
|
return q_[(idx_ - k - 1) & mask_].first;
|
@@ -320,7 +308,7 @@ void ZopfliIterate(size_t num_bytes,
|
|
320
308
|
int* dist_cache,
|
321
309
|
int* last_insert_len,
|
322
310
|
Command* commands,
|
323
|
-
|
311
|
+
size_t* num_commands,
|
324
312
|
int* num_literals) {
|
325
313
|
const Command * const orig_commands = commands;
|
326
314
|
|
@@ -336,8 +324,8 @@ void ZopfliIterate(size_t num_bytes,
|
|
336
324
|
for (size_t i = 0; i + 3 < num_bytes; i++) {
|
337
325
|
size_t cur_ix = position + i;
|
338
326
|
size_t cur_ix_masked = cur_ix & ringbuffer_mask;
|
339
|
-
|
340
|
-
int max_length = num_bytes - i;
|
327
|
+
int max_distance = static_cast<int>(std::min(cur_ix, max_backward_limit));
|
328
|
+
int max_length = static_cast<int>(num_bytes - i);
|
341
329
|
|
342
330
|
queue.Push(i, nodes[i].cost - model.GetLiteralCosts(0, i));
|
343
331
|
|
@@ -346,7 +334,7 @@ void ZopfliIterate(size_t num_bytes,
|
|
346
334
|
|
347
335
|
// Go over the command starting positions in order of increasing cost
|
348
336
|
// difference.
|
349
|
-
for (
|
337
|
+
for (int k = 0; k < 5 && k < queue.size(); ++k) {
|
350
338
|
const size_t start = queue.GetStartPos(k);
|
351
339
|
const double start_costdiff =
|
352
340
|
nodes[start].cost - model.GetLiteralCosts(0, start);
|
@@ -373,12 +361,13 @@ void ZopfliIterate(size_t num_bytes,
|
|
373
361
|
ringbuffer[prev_ix + best_len]) {
|
374
362
|
continue;
|
375
363
|
}
|
376
|
-
const
|
364
|
+
const int len =
|
377
365
|
FindMatchLengthWithLimit(&ringbuffer[prev_ix],
|
378
366
|
&ringbuffer[cur_ix_masked],
|
379
367
|
max_length);
|
380
368
|
for (int l = best_len + 1; l <= len; ++l) {
|
381
|
-
|
369
|
+
const int inslen = static_cast<int>(i - start);
|
370
|
+
double cmd_cost = model.GetCommandCost(j, l, inslen);
|
382
371
|
double cost = start_costdiff + cmd_cost + model.GetLiteralCosts(0, i);
|
383
372
|
if (cost < nodes[i + l].cost) {
|
384
373
|
UpdateZopfliNode(&nodes[0], i, start, l, l, backward, j,
|
@@ -411,8 +400,8 @@ void ZopfliIterate(size_t num_bytes,
|
|
411
400
|
}
|
412
401
|
for (; len <= max_len; ++len) {
|
413
402
|
int len_code = is_dictionary_match ? match.length_code() : len;
|
414
|
-
|
415
|
-
|
403
|
+
const int inslen = static_cast<int>(i - start);
|
404
|
+
double cmd_cost = model.GetCommandCost(dist_code, len_code, inslen);
|
416
405
|
double cost = start_costdiff + cmd_cost + model.GetLiteralCosts(0, i);
|
417
406
|
if (cost < nodes[i + len].cost) {
|
418
407
|
UpdateZopfliNode(&nodes[0], i, start, len, len_code, dist,
|
@@ -459,7 +448,8 @@ void ZopfliIterate(size_t num_bytes,
|
|
459
448
|
}
|
460
449
|
int distance = next.distance;
|
461
450
|
int len_code = next.length_code;
|
462
|
-
|
451
|
+
int max_distance =
|
452
|
+
static_cast<int>(std::min(position + pos, max_backward_limit));
|
463
453
|
bool is_dictionary = (distance > max_distance);
|
464
454
|
int dist_code = next.distance_code;
|
465
455
|
|
@@ -477,7 +467,7 @@ void ZopfliIterate(size_t num_bytes,
|
|
477
467
|
insert_length = 0;
|
478
468
|
pos += copy_length;
|
479
469
|
}
|
480
|
-
*last_insert_len += num_bytes - pos;
|
470
|
+
*last_insert_len += static_cast<int>(num_bytes - pos);
|
481
471
|
*num_commands += (commands - orig_commands);
|
482
472
|
}
|
483
473
|
|
@@ -492,42 +482,43 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
492
482
|
int* dist_cache,
|
493
483
|
int* last_insert_len,
|
494
484
|
Command* commands,
|
495
|
-
|
485
|
+
size_t* num_commands,
|
496
486
|
int* num_literals) {
|
497
487
|
if (num_bytes >= 3 && position >= 3) {
|
498
488
|
// Prepare the hashes for three last bytes of the last write.
|
499
489
|
// These could not be calculated before, since they require knowledge
|
500
490
|
// of both the previous and the current block.
|
501
491
|
hasher->Store(&ringbuffer[(position - 3) & ringbuffer_mask],
|
502
|
-
position - 3);
|
492
|
+
static_cast<uint32_t>(position - 3));
|
503
493
|
hasher->Store(&ringbuffer[(position - 2) & ringbuffer_mask],
|
504
|
-
position - 2);
|
494
|
+
static_cast<uint32_t>(position - 2));
|
505
495
|
hasher->Store(&ringbuffer[(position - 1) & ringbuffer_mask],
|
506
|
-
position - 1);
|
496
|
+
static_cast<uint32_t>(position - 1));
|
507
497
|
}
|
508
498
|
const Command * const orig_commands = commands;
|
509
499
|
int insert_length = *last_insert_len;
|
510
500
|
size_t i = position & ringbuffer_mask;
|
511
|
-
const
|
501
|
+
const size_t i_diff = position - i;
|
512
502
|
const size_t i_end = i + num_bytes;
|
513
503
|
|
514
504
|
// For speed up heuristics for random data.
|
515
|
-
const
|
516
|
-
|
505
|
+
const size_t random_heuristics_window_size = quality < 9 ? 64 : 512;
|
506
|
+
size_t apply_random_heuristics = i + random_heuristics_window_size;
|
517
507
|
|
518
508
|
// Minimum score to accept a backward reference.
|
519
509
|
const int kMinScore = 4.0;
|
520
510
|
|
521
511
|
while (i + Hasher::kHashTypeLength - 1 < i_end) {
|
522
|
-
int max_length = i_end - i;
|
523
|
-
|
512
|
+
int max_length = static_cast<int>(i_end - i);
|
513
|
+
int max_distance =
|
514
|
+
static_cast<int>(std::min(i + i_diff, max_backward_limit));
|
524
515
|
int best_len = 0;
|
525
516
|
int best_len_code = 0;
|
526
517
|
int best_dist = 0;
|
527
518
|
double best_score = kMinScore;
|
528
519
|
bool match_found = hasher->FindLongestMatch(
|
529
520
|
ringbuffer, ringbuffer_mask,
|
530
|
-
dist_cache, i + i_diff, max_length, max_distance,
|
521
|
+
dist_cache, static_cast<uint32_t>(i + i_diff), max_length, max_distance,
|
531
522
|
&best_len, &best_len_code, &best_dist, &best_score);
|
532
523
|
if (match_found) {
|
533
524
|
// Found a match. Let's look for something even better ahead.
|
@@ -538,11 +529,13 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
538
529
|
int best_len_code_2 = 0;
|
539
530
|
int best_dist_2 = 0;
|
540
531
|
double best_score_2 = kMinScore;
|
541
|
-
max_distance =
|
542
|
-
|
532
|
+
max_distance =
|
533
|
+
static_cast<int>(std::min(i + i_diff + 1, max_backward_limit));
|
534
|
+
hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
|
543
535
|
match_found = hasher->FindLongestMatch(
|
544
536
|
ringbuffer, ringbuffer_mask,
|
545
|
-
dist_cache, i + i_diff + 1,
|
537
|
+
dist_cache, static_cast<uint32_t>(i + i_diff + 1),
|
538
|
+
max_length, max_distance,
|
546
539
|
&best_len_2, &best_len_code_2, &best_dist_2, &best_score_2);
|
547
540
|
double cost_diff_lazy = 7.0;
|
548
541
|
if (match_found && best_score_2 >= best_score + cost_diff_lazy) {
|
@@ -562,7 +555,7 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
562
555
|
}
|
563
556
|
apply_random_heuristics =
|
564
557
|
i + 2 * best_len + random_heuristics_window_size;
|
565
|
-
max_distance = std::min(i + i_diff, max_backward_limit);
|
558
|
+
max_distance = static_cast<int>(std::min(i + i_diff, max_backward_limit));
|
566
559
|
// The first 16 codes are special shortcodes, and the minimum offset is 1.
|
567
560
|
int distance_code =
|
568
561
|
ComputeDistanceCode(best_dist, max_distance, quality, dist_cache);
|
@@ -579,12 +572,13 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
579
572
|
// Put the hash keys into the table, if there are enough
|
580
573
|
// bytes left.
|
581
574
|
for (int j = 1; j < best_len; ++j) {
|
582
|
-
hasher->Store(&ringbuffer[i + j],
|
575
|
+
hasher->Store(&ringbuffer[i + j],
|
576
|
+
static_cast<uint32_t>(i + i_diff + j));
|
583
577
|
}
|
584
578
|
i += best_len;
|
585
579
|
} else {
|
586
580
|
++insert_length;
|
587
|
-
hasher->Store(ringbuffer + i, i + i_diff);
|
581
|
+
hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
|
588
582
|
++i;
|
589
583
|
// If we have not seen matches for a long time, we can skip some
|
590
584
|
// match lookups. Unsuccessful match lookups are very very expensive
|
@@ -599,32 +593,30 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
599
593
|
// turn out to be useful in the future, too, so we store less of
|
600
594
|
// them to not to flood out the hash table of good compressible
|
601
595
|
// data.
|
602
|
-
|
596
|
+
size_t i_jump = std::min(i + 16, i_end - 4);
|
603
597
|
for (; i < i_jump; i += 4) {
|
604
|
-
hasher->Store(ringbuffer + i, i + i_diff);
|
598
|
+
hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
|
605
599
|
insert_length += 4;
|
606
600
|
}
|
607
601
|
} else {
|
608
|
-
|
602
|
+
size_t i_jump = std::min(i + 8, i_end - 3);
|
609
603
|
for (; i < i_jump; i += 2) {
|
610
|
-
hasher->Store(ringbuffer + i, i + i_diff);
|
604
|
+
hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
|
611
605
|
insert_length += 2;
|
612
606
|
}
|
613
607
|
}
|
614
608
|
}
|
615
609
|
}
|
616
610
|
}
|
617
|
-
insert_length += (i_end - i);
|
611
|
+
insert_length += static_cast<int>(i_end - i);
|
618
612
|
*last_insert_len = insert_length;
|
619
|
-
*num_commands +=
|
613
|
+
*num_commands += commands - orig_commands;
|
620
614
|
}
|
621
615
|
|
622
616
|
void CreateBackwardReferences(size_t num_bytes,
|
623
617
|
size_t position,
|
624
618
|
const uint8_t* ringbuffer,
|
625
619
|
size_t ringbuffer_mask,
|
626
|
-
const float* literal_cost,
|
627
|
-
size_t literal_cost_mask,
|
628
620
|
const size_t max_backward_limit,
|
629
621
|
const int quality,
|
630
622
|
Hashers* hashers,
|
@@ -632,46 +624,47 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
632
624
|
int* dist_cache,
|
633
625
|
int* last_insert_len,
|
634
626
|
Command* commands,
|
635
|
-
|
627
|
+
size_t* num_commands,
|
636
628
|
int* num_literals) {
|
637
629
|
bool zopflify = quality > 9;
|
638
630
|
if (zopflify) {
|
639
|
-
Hashers::H9* hasher = hashers->hash_h9
|
631
|
+
Hashers::H9* hasher = hashers->hash_h9;
|
640
632
|
if (num_bytes >= 3 && position >= 3) {
|
641
633
|
// Prepare the hashes for three last bytes of the last write.
|
642
634
|
// These could not be calculated before, since they require knowledge
|
643
635
|
// of both the previous and the current block.
|
644
636
|
hasher->Store(&ringbuffer[(position - 3) & ringbuffer_mask],
|
645
|
-
position - 3);
|
637
|
+
static_cast<uint32_t>(position - 3));
|
646
638
|
hasher->Store(&ringbuffer[(position - 2) & ringbuffer_mask],
|
647
|
-
position - 2);
|
639
|
+
static_cast<uint32_t>(position - 2));
|
648
640
|
hasher->Store(&ringbuffer[(position - 1) & ringbuffer_mask],
|
649
|
-
position - 1);
|
641
|
+
static_cast<uint32_t>(position - 1));
|
650
642
|
}
|
651
643
|
std::vector<int> num_matches(num_bytes);
|
652
644
|
std::vector<BackwardMatch> matches(3 * num_bytes);
|
653
645
|
size_t cur_match_pos = 0;
|
654
646
|
for (size_t i = 0; i + 3 < num_bytes; ++i) {
|
655
|
-
|
656
|
-
|
647
|
+
int max_distance =
|
648
|
+
static_cast<int>(std::min(position + i, max_backward_limit));
|
649
|
+
int max_length = static_cast<int>(num_bytes - i);
|
657
650
|
// Ensure that we have at least kMaxZopfliLen free slots.
|
658
651
|
if (matches.size() < cur_match_pos + kMaxZopfliLen) {
|
659
652
|
matches.resize(cur_match_pos + kMaxZopfliLen);
|
660
653
|
}
|
661
654
|
hasher->FindAllMatches(
|
662
655
|
ringbuffer, ringbuffer_mask,
|
663
|
-
position + i, max_length, max_distance,
|
656
|
+
static_cast<uint32_t>(position + i), max_length, max_distance,
|
664
657
|
&num_matches[i], &matches[cur_match_pos]);
|
665
658
|
hasher->Store(&ringbuffer[(position + i) & ringbuffer_mask],
|
666
|
-
position + i);
|
659
|
+
static_cast<uint32_t>(position + i));
|
667
660
|
cur_match_pos += num_matches[i];
|
668
661
|
if (num_matches[i] == 1) {
|
669
662
|
const int match_len = matches[cur_match_pos - 1].length();
|
670
663
|
if (match_len > kMaxZopfliLen) {
|
671
664
|
for (int j = 1; j < match_len; ++j) {
|
672
665
|
++i;
|
673
|
-
hasher->Store(
|
674
|
-
|
666
|
+
hasher->Store(&ringbuffer[(position + i) & ringbuffer_mask],
|
667
|
+
static_cast<uint32_t>(position + i));
|
675
668
|
num_matches[i] = 0;
|
676
669
|
}
|
677
670
|
}
|
@@ -682,13 +675,13 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
682
675
|
int orig_dist_cache[4] = {
|
683
676
|
dist_cache[0], dist_cache[1], dist_cache[2], dist_cache[3]
|
684
677
|
};
|
685
|
-
|
678
|
+
size_t orig_num_commands = *num_commands;
|
686
679
|
static const int kIterations = 2;
|
687
680
|
for (int i = 0; i < kIterations; i++) {
|
688
681
|
ZopfliCostModel model;
|
689
682
|
if (i == 0) {
|
690
683
|
model.SetFromLiteralCosts(num_bytes, position,
|
691
|
-
|
684
|
+
ringbuffer, ringbuffer_mask);
|
692
685
|
} else {
|
693
686
|
model.SetFromCommands(num_bytes, position,
|
694
687
|
ringbuffer, ringbuffer_mask,
|
@@ -710,55 +703,55 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
710
703
|
case 1:
|
711
704
|
CreateBackwardReferences<Hashers::H1>(
|
712
705
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
713
|
-
quality, hashers->hash_h1
|
706
|
+
quality, hashers->hash_h1, dist_cache, last_insert_len,
|
714
707
|
commands, num_commands, num_literals);
|
715
708
|
break;
|
716
709
|
case 2:
|
717
710
|
CreateBackwardReferences<Hashers::H2>(
|
718
711
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
719
|
-
quality, hashers->hash_h2
|
712
|
+
quality, hashers->hash_h2, dist_cache, last_insert_len,
|
720
713
|
commands, num_commands, num_literals);
|
721
714
|
break;
|
722
715
|
case 3:
|
723
716
|
CreateBackwardReferences<Hashers::H3>(
|
724
717
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
725
|
-
quality, hashers->hash_h3
|
718
|
+
quality, hashers->hash_h3, dist_cache, last_insert_len,
|
726
719
|
commands, num_commands, num_literals);
|
727
720
|
break;
|
728
721
|
case 4:
|
729
722
|
CreateBackwardReferences<Hashers::H4>(
|
730
723
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
731
|
-
quality, hashers->hash_h4
|
724
|
+
quality, hashers->hash_h4, dist_cache, last_insert_len,
|
732
725
|
commands, num_commands, num_literals);
|
733
726
|
break;
|
734
727
|
case 5:
|
735
728
|
CreateBackwardReferences<Hashers::H5>(
|
736
729
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
737
|
-
quality, hashers->hash_h5
|
730
|
+
quality, hashers->hash_h5, dist_cache, last_insert_len,
|
738
731
|
commands, num_commands, num_literals);
|
739
732
|
break;
|
740
733
|
case 6:
|
741
734
|
CreateBackwardReferences<Hashers::H6>(
|
742
735
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
743
|
-
quality, hashers->hash_h6
|
736
|
+
quality, hashers->hash_h6, dist_cache, last_insert_len,
|
744
737
|
commands, num_commands, num_literals);
|
745
738
|
break;
|
746
739
|
case 7:
|
747
740
|
CreateBackwardReferences<Hashers::H7>(
|
748
741
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
749
|
-
quality, hashers->hash_h7
|
742
|
+
quality, hashers->hash_h7, dist_cache, last_insert_len,
|
750
743
|
commands, num_commands, num_literals);
|
751
744
|
break;
|
752
745
|
case 8:
|
753
746
|
CreateBackwardReferences<Hashers::H8>(
|
754
747
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
755
|
-
quality, hashers->hash_h8
|
748
|
+
quality, hashers->hash_h8, dist_cache, last_insert_len,
|
756
749
|
commands, num_commands, num_literals);
|
757
750
|
break;
|
758
751
|
case 9:
|
759
752
|
CreateBackwardReferences<Hashers::H9>(
|
760
753
|
num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
|
761
|
-
quality, hashers->hash_h9
|
754
|
+
quality, hashers->hash_h9, dist_cache, last_insert_len,
|
762
755
|
commands, num_commands, num_literals);
|
763
756
|
break;
|
764
757
|
default:
|
@@ -1,27 +1,19 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Function to find backward reference copies.
|
16
8
|
|
17
9
|
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_
|
18
10
|
#define BROTLI_ENC_BACKWARD_REFERENCES_H_
|
19
11
|
|
20
|
-
#include <stdint.h>
|
21
12
|
#include <vector>
|
22
13
|
|
23
14
|
#include "./hash.h"
|
24
15
|
#include "./command.h"
|
16
|
+
#include "./types.h"
|
25
17
|
|
26
18
|
namespace brotli {
|
27
19
|
|
@@ -33,8 +25,6 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
33
25
|
size_t position,
|
34
26
|
const uint8_t* ringbuffer,
|
35
27
|
size_t ringbuffer_mask,
|
36
|
-
const float* literal_cost,
|
37
|
-
size_t literal_cost_mask,
|
38
28
|
const size_t max_backward_limit,
|
39
29
|
const int quality,
|
40
30
|
Hashers* hashers,
|
@@ -42,7 +32,7 @@ void CreateBackwardReferences(size_t num_bytes,
|
|
42
32
|
int* dist_cache,
|
43
33
|
int* last_insert_len,
|
44
34
|
Command* commands,
|
45
|
-
|
35
|
+
size_t* num_commands,
|
46
36
|
int* num_literals);
|
47
37
|
|
48
38
|
} // namespace brotli
|
@@ -1,27 +1,19 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
//
|
9
|
-
// Unless required by applicable law or agreed to in writing, software
|
10
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
// See the License for the specific language governing permissions and
|
13
|
-
// limitations under the License.
|
14
|
-
//
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
15
7
|
// Functions to estimate the bit cost of Huffman trees.
|
16
8
|
|
17
9
|
#ifndef BROTLI_ENC_BIT_COST_H_
|
18
10
|
#define BROTLI_ENC_BIT_COST_H_
|
19
11
|
|
20
12
|
|
21
|
-
#include <stdint.h>
|
22
13
|
|
23
14
|
#include "./entropy_encode.h"
|
24
15
|
#include "./fast_log.h"
|
16
|
+
#include "./types.h"
|
25
17
|
|
26
18
|
namespace brotli {
|
27
19
|
|
@@ -77,12 +69,12 @@ double PopulationCost(const Histogram<kSize>& histogram) {
|
|
77
69
|
return 20 + histogram.total_count_;
|
78
70
|
}
|
79
71
|
double bits = 0;
|
80
|
-
uint8_t
|
72
|
+
uint8_t depth_array[kSize] = { 0 };
|
81
73
|
if (count <= 4) {
|
82
74
|
// For very low symbol count we build the Huffman tree.
|
83
|
-
CreateHuffmanTree(&histogram.data_[0], kSize, 15,
|
75
|
+
CreateHuffmanTree(&histogram.data_[0], kSize, 15, depth_array);
|
84
76
|
for (int i = 0; i < kSize; ++i) {
|
85
|
-
bits += histogram.data_[i] *
|
77
|
+
bits += histogram.data_[i] * depth_array[i];
|
86
78
|
}
|
87
79
|
return count == 3 ? bits + 28 : bits + 37;
|
88
80
|
}
|
@@ -110,7 +102,7 @@ double PopulationCost(const Histogram<kSize>& histogram) {
|
|
110
102
|
++depth_histo[depth];
|
111
103
|
++i;
|
112
104
|
} else {
|
113
|
-
// Compute the run length of zeros and add the
|
105
|
+
// Compute the run length of zeros and add the appropriate number of 0 and
|
114
106
|
// 17 code length codes to the code length code histogram.
|
115
107
|
int reps = 1;
|
116
108
|
for (int k = i + 1; k < kSize && histogram.data_[k] == 0; ++k) {
|