brotli 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +6 -3
- data/.github/workflows/publish.yml +7 -17
- data/.gitmodules +1 -1
- data/README.md +2 -2
- data/ext/brotli/brotli.c +1 -0
- data/ext/brotli/extconf.rb +6 -0
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +4 -1
- data/vendor/brotli/c/common/constants.c +1 -1
- data/vendor/brotli/c/common/constants.h +2 -1
- data/vendor/brotli/c/common/context.c +1 -1
- data/vendor/brotli/c/common/dictionary.c +5 -3
- data/vendor/brotli/c/common/platform.c +2 -1
- data/vendor/brotli/c/common/platform.h +60 -113
- data/vendor/brotli/c/common/shared_dictionary.c +521 -0
- data/vendor/brotli/c/common/shared_dictionary_internal.h +75 -0
- data/vendor/brotli/c/common/transform.c +1 -1
- data/vendor/brotli/c/common/version.h +31 -6
- data/vendor/brotli/c/dec/bit_reader.c +10 -8
- data/vendor/brotli/c/dec/bit_reader.h +172 -100
- data/vendor/brotli/c/dec/decode.c +467 -200
- data/vendor/brotli/c/dec/huffman.c +7 -4
- data/vendor/brotli/c/dec/huffman.h +2 -1
- data/vendor/brotli/c/dec/prefix.h +2 -1
- data/vendor/brotli/c/dec/state.c +33 -9
- data/vendor/brotli/c/dec/state.h +70 -35
- data/vendor/brotli/c/enc/backward_references.c +81 -19
- data/vendor/brotli/c/enc/backward_references.h +5 -4
- data/vendor/brotli/c/enc/backward_references_hq.c +148 -52
- data/vendor/brotli/c/enc/backward_references_hq.h +6 -5
- data/vendor/brotli/c/enc/backward_references_inc.h +31 -5
- data/vendor/brotli/c/enc/bit_cost.c +8 -7
- data/vendor/brotli/c/enc/bit_cost.h +5 -4
- data/vendor/brotli/c/enc/block_splitter.c +37 -14
- data/vendor/brotli/c/enc/block_splitter.h +5 -4
- data/vendor/brotli/c/enc/block_splitter_inc.h +86 -45
- data/vendor/brotli/c/enc/brotli_bit_stream.c +132 -110
- data/vendor/brotli/c/enc/brotli_bit_stream.h +11 -6
- data/vendor/brotli/c/enc/cluster.c +10 -9
- data/vendor/brotli/c/enc/cluster.h +7 -6
- data/vendor/brotli/c/enc/cluster_inc.h +25 -20
- data/vendor/brotli/c/enc/command.c +1 -1
- data/vendor/brotli/c/enc/command.h +5 -4
- data/vendor/brotli/c/enc/compound_dictionary.c +207 -0
- data/vendor/brotli/c/enc/compound_dictionary.h +74 -0
- data/vendor/brotli/c/enc/compress_fragment.c +93 -83
- data/vendor/brotli/c/enc/compress_fragment.h +32 -7
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +99 -87
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +21 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +3 -1
- data/vendor/brotli/c/enc/encode.c +473 -404
- data/vendor/brotli/c/enc/encoder_dict.c +611 -4
- data/vendor/brotli/c/enc/encoder_dict.h +117 -3
- data/vendor/brotli/c/enc/entropy_encode.c +3 -2
- data/vendor/brotli/c/enc/entropy_encode.h +2 -1
- data/vendor/brotli/c/enc/entropy_encode_static.h +5 -2
- data/vendor/brotli/c/enc/fast_log.c +1 -1
- data/vendor/brotli/c/enc/fast_log.h +2 -1
- data/vendor/brotli/c/enc/find_match_length.h +15 -22
- data/vendor/brotli/c/enc/hash.h +285 -45
- data/vendor/brotli/c/enc/hash_composite_inc.h +26 -11
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +20 -18
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +34 -39
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +6 -10
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +4 -4
- data/vendor/brotli/c/enc/hash_rolling_inc.h +4 -4
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +6 -5
- data/vendor/brotli/c/enc/histogram.c +4 -4
- data/vendor/brotli/c/enc/histogram.h +7 -6
- data/vendor/brotli/c/enc/literal_cost.c +20 -15
- data/vendor/brotli/c/enc/literal_cost.h +4 -2
- data/vendor/brotli/c/enc/memory.c +29 -5
- data/vendor/brotli/c/enc/memory.h +19 -2
- data/vendor/brotli/c/enc/metablock.c +72 -58
- data/vendor/brotli/c/enc/metablock.h +9 -8
- data/vendor/brotli/c/enc/metablock_inc.h +8 -6
- data/vendor/brotli/c/enc/params.h +4 -3
- data/vendor/brotli/c/enc/prefix.h +3 -2
- data/vendor/brotli/c/enc/quality.h +40 -3
- data/vendor/brotli/c/enc/ringbuffer.h +4 -3
- data/vendor/brotli/c/enc/state.h +104 -0
- data/vendor/brotli/c/enc/static_dict.c +60 -4
- data/vendor/brotli/c/enc/static_dict.h +3 -2
- data/vendor/brotli/c/enc/static_dict_lut.h +2 -0
- data/vendor/brotli/c/enc/utf8_util.c +1 -1
- data/vendor/brotli/c/enc/utf8_util.h +2 -1
- data/vendor/brotli/c/enc/write_bits.h +2 -1
- data/vendor/brotli/c/include/brotli/decode.h +67 -2
- data/vendor/brotli/c/include/brotli/encode.h +55 -2
- data/vendor/brotli/c/include/brotli/port.h +28 -11
- data/vendor/brotli/c/include/brotli/shared_dictionary.h +100 -0
- metadata +9 -3
@@ -7,27 +7,27 @@
|
|
7
7
|
/* Algorithms for distributing the literals and commands of a metablock between
|
8
8
|
block types and contexts. */
|
9
9
|
|
10
|
-
#include "
|
10
|
+
#include "metablock.h"
|
11
|
+
|
12
|
+
#include <brotli/types.h>
|
11
13
|
|
12
14
|
#include "../common/constants.h"
|
13
15
|
#include "../common/context.h"
|
14
16
|
#include "../common/platform.h"
|
15
|
-
#include
|
16
|
-
#include "
|
17
|
-
#include "
|
18
|
-
#include "
|
19
|
-
#include "
|
20
|
-
#include "
|
21
|
-
#include "
|
22
|
-
#include "./quality.h"
|
17
|
+
#include "bit_cost.h"
|
18
|
+
#include "block_splitter.h"
|
19
|
+
#include "cluster.h"
|
20
|
+
#include "entropy_encode.h"
|
21
|
+
#include "histogram.h"
|
22
|
+
#include "memory.h"
|
23
|
+
#include "quality.h"
|
23
24
|
|
24
25
|
#if defined(__cplusplus) || defined(c_plusplus)
|
25
26
|
extern "C" {
|
26
27
|
#endif
|
27
28
|
|
28
|
-
void BrotliInitDistanceParams(
|
29
|
-
uint32_t npostfix, uint32_t ndirect) {
|
30
|
-
BrotliDistanceParams* dist_params = ¶ms->dist;
|
29
|
+
void BrotliInitDistanceParams(BrotliDistanceParams* dist_params,
|
30
|
+
uint32_t npostfix, uint32_t ndirect, BROTLI_BOOL large_window) {
|
31
31
|
uint32_t alphabet_size_max;
|
32
32
|
uint32_t alphabet_size_limit;
|
33
33
|
uint32_t max_distance;
|
@@ -41,7 +41,7 @@ void BrotliInitDistanceParams(BrotliEncoderParams* params,
|
|
41
41
|
max_distance = ndirect + (1U << (BROTLI_MAX_DISTANCE_BITS + npostfix + 2)) -
|
42
42
|
(1U << (npostfix + 2));
|
43
43
|
|
44
|
-
if (
|
44
|
+
if (large_window) {
|
45
45
|
BrotliDistanceCodeLimit limit = BrotliCalculateDistanceCodeLimit(
|
46
46
|
BROTLI_MAX_ALLOWED_DISTANCE, npostfix, ndirect);
|
47
47
|
alphabet_size_max = BROTLI_DISTANCE_ALPHABET_SIZE(
|
@@ -83,14 +83,14 @@ static BROTLI_BOOL ComputeDistanceCost(const Command* cmds,
|
|
83
83
|
size_t num_commands,
|
84
84
|
const BrotliDistanceParams* orig_params,
|
85
85
|
const BrotliDistanceParams* new_params,
|
86
|
-
double* cost
|
86
|
+
double* cost,
|
87
|
+
HistogramDistance* tmp) {
|
87
88
|
size_t i;
|
88
89
|
BROTLI_BOOL equal_params = BROTLI_FALSE;
|
89
90
|
uint16_t dist_prefix;
|
90
91
|
uint32_t dist_extra;
|
91
92
|
double extra_bits = 0.0;
|
92
|
-
|
93
|
-
HistogramClearDistance(&histo);
|
93
|
+
HistogramClearDistance(tmp);
|
94
94
|
|
95
95
|
if (orig_params->distance_postfix_bits == new_params->distance_postfix_bits &&
|
96
96
|
orig_params->num_direct_distance_codes ==
|
@@ -114,12 +114,12 @@ static BROTLI_BOOL ComputeDistanceCost(const Command* cmds,
|
|
114
114
|
&dist_prefix,
|
115
115
|
&dist_extra);
|
116
116
|
}
|
117
|
-
HistogramAddDistance(
|
117
|
+
HistogramAddDistance(tmp, dist_prefix & 0x3FF);
|
118
118
|
extra_bits += dist_prefix >> 10;
|
119
119
|
}
|
120
120
|
}
|
121
121
|
|
122
|
-
*cost = BrotliPopulationCostDistance(
|
122
|
+
*cost = BrotliPopulationCostDistance(tmp) + extra_bits;
|
123
123
|
return BROTLI_TRUE;
|
124
124
|
}
|
125
125
|
|
@@ -147,43 +147,46 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
147
147
|
uint32_t ndirect_msb = 0;
|
148
148
|
BROTLI_BOOL check_orig = BROTLI_TRUE;
|
149
149
|
double best_dist_cost = 1e99;
|
150
|
-
|
151
|
-
|
150
|
+
BrotliDistanceParams orig_params = params->dist;
|
151
|
+
BrotliDistanceParams new_params = params->dist;
|
152
|
+
HistogramDistance* tmp = BROTLI_ALLOC(m, HistogramDistance, 1);
|
153
|
+
|
154
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tmp)) return;
|
152
155
|
|
153
156
|
for (npostfix = 0; npostfix <= BROTLI_MAX_NPOSTFIX; npostfix++) {
|
154
157
|
for (; ndirect_msb < 16; ndirect_msb++) {
|
155
158
|
uint32_t ndirect = ndirect_msb << npostfix;
|
156
159
|
BROTLI_BOOL skip;
|
157
160
|
double dist_cost;
|
158
|
-
BrotliInitDistanceParams(&new_params, npostfix, ndirect
|
159
|
-
|
160
|
-
|
161
|
+
BrotliInitDistanceParams(&new_params, npostfix, ndirect,
|
162
|
+
params->large_window);
|
163
|
+
if (npostfix == orig_params.distance_postfix_bits &&
|
164
|
+
ndirect == orig_params.num_direct_distance_codes) {
|
161
165
|
check_orig = BROTLI_FALSE;
|
162
166
|
}
|
163
167
|
skip = !ComputeDistanceCost(
|
164
|
-
cmds, num_commands,
|
165
|
-
&orig_params.dist, &new_params.dist, &dist_cost);
|
168
|
+
cmds, num_commands, &orig_params, &new_params, &dist_cost, tmp);
|
166
169
|
if (skip || (dist_cost > best_dist_cost)) {
|
167
170
|
break;
|
168
171
|
}
|
169
172
|
best_dist_cost = dist_cost;
|
170
|
-
params->dist = new_params
|
173
|
+
params->dist = new_params;
|
171
174
|
}
|
172
175
|
if (ndirect_msb > 0) ndirect_msb--;
|
173
176
|
ndirect_msb /= 2;
|
174
177
|
}
|
175
178
|
if (check_orig) {
|
176
179
|
double dist_cost;
|
177
|
-
ComputeDistanceCost(cmds, num_commands,
|
178
|
-
&
|
180
|
+
ComputeDistanceCost(cmds, num_commands, &orig_params, &orig_params,
|
181
|
+
&dist_cost, tmp);
|
179
182
|
if (dist_cost < best_dist_cost) {
|
180
183
|
/* NB: currently unused; uncomment when more param tuning is added. */
|
181
184
|
/* best_dist_cost = dist_cost; */
|
182
|
-
params->dist = orig_params
|
185
|
+
params->dist = orig_params;
|
183
186
|
}
|
184
187
|
}
|
185
|
-
|
186
|
-
|
188
|
+
BROTLI_FREE(m, tmp);
|
189
|
+
RecomputeDistancePrefixes(cmds, num_commands, &orig_params, ¶ms->dist);
|
187
190
|
|
188
191
|
BrotliSplitBlock(m, cmds, num_commands,
|
189
192
|
ringbuffer, pos, mask, params,
|
@@ -284,15 +287,15 @@ void BrotliBuildMetaBlock(MemoryManager* m,
|
|
284
287
|
}
|
285
288
|
|
286
289
|
#define FN(X) X ## Literal
|
287
|
-
#include "
|
290
|
+
#include "metablock_inc.h" /* NOLINT(build/include) */
|
288
291
|
#undef FN
|
289
292
|
|
290
293
|
#define FN(X) X ## Command
|
291
|
-
#include "
|
294
|
+
#include "metablock_inc.h" /* NOLINT(build/include) */
|
292
295
|
#undef FN
|
293
296
|
|
294
297
|
#define FN(X) X ## Distance
|
295
|
-
#include "
|
298
|
+
#include "metablock_inc.h" /* NOLINT(build/include) */
|
296
299
|
#undef FN
|
297
300
|
|
298
301
|
#define BROTLI_MAX_STATIC_CONTEXTS 13
|
@@ -535,17 +538,21 @@ static void MapStaticContexts(MemoryManager* m,
|
|
535
538
|
}
|
536
539
|
}
|
537
540
|
|
538
|
-
|
539
|
-
MemoryManager* m, const uint8_t* ringbuffer, size_t pos, size_t mask,
|
540
|
-
uint8_t prev_byte, uint8_t prev_byte2, ContextLut literal_context_lut,
|
541
|
-
const size_t num_contexts, const uint32_t* static_context_map,
|
542
|
-
const Command* commands, size_t n_commands, MetaBlockSplit* mb) {
|
541
|
+
typedef struct GreedyMetablockArena {
|
543
542
|
union {
|
544
543
|
BlockSplitterLiteral plain;
|
545
544
|
ContextBlockSplitter ctx;
|
546
545
|
} lit_blocks;
|
547
546
|
BlockSplitterCommand cmd_blocks;
|
548
547
|
BlockSplitterDistance dist_blocks;
|
548
|
+
} GreedyMetablockArena;
|
549
|
+
|
550
|
+
static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
|
551
|
+
MemoryManager* m, GreedyMetablockArena* arena, const uint8_t* ringbuffer,
|
552
|
+
size_t pos, size_t mask, uint8_t prev_byte, uint8_t prev_byte2,
|
553
|
+
ContextLut literal_context_lut, const size_t num_contexts,
|
554
|
+
const uint32_t* static_context_map, const Command* commands,
|
555
|
+
size_t n_commands, MetaBlockSplit* mb) {
|
549
556
|
size_t num_literals = 0;
|
550
557
|
size_t i;
|
551
558
|
for (i = 0; i < n_commands; ++i) {
|
@@ -553,20 +560,20 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
|
|
553
560
|
}
|
554
561
|
|
555
562
|
if (num_contexts == 1) {
|
556
|
-
InitBlockSplitterLiteral(m, &lit_blocks.plain, 256, 512, 400.0,
|
563
|
+
InitBlockSplitterLiteral(m, &arena->lit_blocks.plain, 256, 512, 400.0,
|
557
564
|
num_literals, &mb->literal_split, &mb->literal_histograms,
|
558
565
|
&mb->literal_histograms_size);
|
559
566
|
} else {
|
560
|
-
InitContextBlockSplitter(m, &lit_blocks.ctx, 256, num_contexts, 512,
|
561
|
-
num_literals, &mb->literal_split, &mb->literal_histograms,
|
567
|
+
InitContextBlockSplitter(m, &arena->lit_blocks.ctx, 256, num_contexts, 512,
|
568
|
+
400.0, num_literals, &mb->literal_split, &mb->literal_histograms,
|
562
569
|
&mb->literal_histograms_size);
|
563
570
|
}
|
564
571
|
if (BROTLI_IS_OOM(m)) return;
|
565
|
-
InitBlockSplitterCommand(m, &cmd_blocks, BROTLI_NUM_COMMAND_SYMBOLS,
|
566
|
-
500.0, n_commands, &mb->command_split, &mb->command_histograms,
|
572
|
+
InitBlockSplitterCommand(m, &arena->cmd_blocks, BROTLI_NUM_COMMAND_SYMBOLS,
|
573
|
+
1024, 500.0, n_commands, &mb->command_split, &mb->command_histograms,
|
567
574
|
&mb->command_histograms_size);
|
568
575
|
if (BROTLI_IS_OOM(m)) return;
|
569
|
-
InitBlockSplitterDistance(m, &dist_blocks, 64, 512, 100.0, n_commands,
|
576
|
+
InitBlockSplitterDistance(m, &arena->dist_blocks, 64, 512, 100.0, n_commands,
|
570
577
|
&mb->distance_split, &mb->distance_histograms,
|
571
578
|
&mb->distance_histograms_size);
|
572
579
|
if (BROTLI_IS_OOM(m)) return;
|
@@ -574,15 +581,15 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
|
|
574
581
|
for (i = 0; i < n_commands; ++i) {
|
575
582
|
const Command cmd = commands[i];
|
576
583
|
size_t j;
|
577
|
-
BlockSplitterAddSymbolCommand(&cmd_blocks, cmd.cmd_prefix_);
|
584
|
+
BlockSplitterAddSymbolCommand(&arena->cmd_blocks, cmd.cmd_prefix_);
|
578
585
|
for (j = cmd.insert_len_; j != 0; --j) {
|
579
586
|
uint8_t literal = ringbuffer[pos & mask];
|
580
587
|
if (num_contexts == 1) {
|
581
|
-
BlockSplitterAddSymbolLiteral(&lit_blocks.plain, literal);
|
588
|
+
BlockSplitterAddSymbolLiteral(&arena->lit_blocks.plain, literal);
|
582
589
|
} else {
|
583
590
|
size_t context =
|
584
591
|
BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut);
|
585
|
-
ContextBlockSplitterAddSymbol(&lit_blocks.ctx, m, literal,
|
592
|
+
ContextBlockSplitterAddSymbol(&arena->lit_blocks.ctx, m, literal,
|
586
593
|
static_context_map[context]);
|
587
594
|
if (BROTLI_IS_OOM(m)) return;
|
588
595
|
}
|
@@ -595,21 +602,24 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
|
|
595
602
|
prev_byte2 = ringbuffer[(pos - 2) & mask];
|
596
603
|
prev_byte = ringbuffer[(pos - 1) & mask];
|
597
604
|
if (cmd.cmd_prefix_ >= 128) {
|
598
|
-
BlockSplitterAddSymbolDistance(
|
605
|
+
BlockSplitterAddSymbolDistance(
|
606
|
+
&arena->dist_blocks, cmd.dist_prefix_ & 0x3FF);
|
599
607
|
}
|
600
608
|
}
|
601
609
|
}
|
602
610
|
|
603
611
|
if (num_contexts == 1) {
|
604
612
|
BlockSplitterFinishBlockLiteral(
|
605
|
-
&lit_blocks.plain, /* is_final = */ BROTLI_TRUE);
|
613
|
+
&arena->lit_blocks.plain, /* is_final = */ BROTLI_TRUE);
|
606
614
|
} else {
|
607
615
|
ContextBlockSplitterFinishBlock(
|
608
|
-
&lit_blocks.ctx, m, /* is_final = */ BROTLI_TRUE);
|
616
|
+
&arena->lit_blocks.ctx, m, /* is_final = */ BROTLI_TRUE);
|
609
617
|
if (BROTLI_IS_OOM(m)) return;
|
610
618
|
}
|
611
|
-
BlockSplitterFinishBlockCommand(
|
612
|
-
|
619
|
+
BlockSplitterFinishBlockCommand(
|
620
|
+
&arena->cmd_blocks, /* is_final = */ BROTLI_TRUE);
|
621
|
+
BlockSplitterFinishBlockDistance(
|
622
|
+
&arena->dist_blocks, /* is_final = */ BROTLI_TRUE);
|
613
623
|
|
614
624
|
if (num_contexts > 1) {
|
615
625
|
MapStaticContexts(m, num_contexts, static_context_map, mb);
|
@@ -628,14 +638,18 @@ void BrotliBuildMetaBlockGreedy(MemoryManager* m,
|
|
628
638
|
const Command* commands,
|
629
639
|
size_t n_commands,
|
630
640
|
MetaBlockSplit* mb) {
|
641
|
+
GreedyMetablockArena* arena = BROTLI_ALLOC(m, GreedyMetablockArena, 1);
|
642
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
|
631
643
|
if (num_contexts == 1) {
|
632
|
-
BrotliBuildMetaBlockGreedyInternal(m, ringbuffer, pos, mask,
|
633
|
-
prev_byte2, literal_context_lut, 1, NULL, commands,
|
644
|
+
BrotliBuildMetaBlockGreedyInternal(m, arena, ringbuffer, pos, mask,
|
645
|
+
prev_byte, prev_byte2, literal_context_lut, 1, NULL, commands,
|
646
|
+
n_commands, mb);
|
634
647
|
} else {
|
635
|
-
BrotliBuildMetaBlockGreedyInternal(m, ringbuffer, pos, mask,
|
636
|
-
prev_byte2, literal_context_lut, num_contexts,
|
637
|
-
commands, n_commands, mb);
|
648
|
+
BrotliBuildMetaBlockGreedyInternal(m, arena, ringbuffer, pos, mask,
|
649
|
+
prev_byte, prev_byte2, literal_context_lut, num_contexts,
|
650
|
+
static_context_map, commands, n_commands, mb);
|
638
651
|
}
|
652
|
+
BROTLI_FREE(m, arena);
|
639
653
|
}
|
640
654
|
|
641
655
|
void BrotliOptimizeHistograms(uint32_t num_distance_codes,
|
@@ -10,14 +10,15 @@
|
|
10
10
|
#ifndef BROTLI_ENC_METABLOCK_H_
|
11
11
|
#define BROTLI_ENC_METABLOCK_H_
|
12
12
|
|
13
|
+
#include <brotli/types.h>
|
14
|
+
|
13
15
|
#include "../common/context.h"
|
14
16
|
#include "../common/platform.h"
|
15
|
-
#include
|
16
|
-
#include "
|
17
|
-
#include "
|
18
|
-
#include "
|
19
|
-
#include "
|
20
|
-
#include "./quality.h"
|
17
|
+
#include "block_splitter.h"
|
18
|
+
#include "command.h"
|
19
|
+
#include "histogram.h"
|
20
|
+
#include "memory.h"
|
21
|
+
#include "quality.h"
|
21
22
|
|
22
23
|
#if defined(__cplusplus) || defined(c_plusplus)
|
23
24
|
extern "C" {
|
@@ -95,8 +96,8 @@ BROTLI_INTERNAL void BrotliBuildMetaBlockGreedy(
|
|
95
96
|
BROTLI_INTERNAL void BrotliOptimizeHistograms(uint32_t num_distance_codes,
|
96
97
|
MetaBlockSplit* mb);
|
97
98
|
|
98
|
-
BROTLI_INTERNAL void BrotliInitDistanceParams(
|
99
|
-
uint32_t npostfix, uint32_t ndirect);
|
99
|
+
BROTLI_INTERNAL void BrotliInitDistanceParams(BrotliDistanceParams* params,
|
100
|
+
uint32_t npostfix, uint32_t ndirect, BROTLI_BOOL large_window);
|
100
101
|
|
101
102
|
#if defined(__cplusplus) || defined(c_plusplus)
|
102
103
|
} /* extern "C" */
|
@@ -27,6 +27,9 @@ typedef struct FN(BlockSplitter) {
|
|
27
27
|
HistogramType* histograms_; /* not owned */
|
28
28
|
size_t* histograms_size_; /* not owned */
|
29
29
|
|
30
|
+
/* Temporary storage for BlockSplitterFinishBlock. */
|
31
|
+
HistogramType combined_histo[2];
|
32
|
+
|
30
33
|
/* The number of symbols that we want to collect before deciding on whether
|
31
34
|
or not to merge the block with a previous one or emit a new block. */
|
32
35
|
size_t target_block_size_;
|
@@ -104,17 +107,16 @@ static void FN(BlockSplitterFinishBlock)(
|
|
104
107
|
} else if (self->block_size_ > 0) {
|
105
108
|
double entropy = BitsEntropy(histograms[self->curr_histogram_ix_].data_,
|
106
109
|
self->alphabet_size_);
|
107
|
-
HistogramType combined_histo[2];
|
108
110
|
double combined_entropy[2];
|
109
111
|
double diff[2];
|
110
112
|
size_t j;
|
111
113
|
for (j = 0; j < 2; ++j) {
|
112
114
|
size_t last_histogram_ix = self->last_histogram_ix_[j];
|
113
|
-
combined_histo[j] = histograms[self->curr_histogram_ix_];
|
114
|
-
FN(HistogramAddHistogram)(&combined_histo[j],
|
115
|
+
self->combined_histo[j] = histograms[self->curr_histogram_ix_];
|
116
|
+
FN(HistogramAddHistogram)(&self->combined_histo[j],
|
115
117
|
&histograms[last_histogram_ix]);
|
116
118
|
combined_entropy[j] = BitsEntropy(
|
117
|
-
&combined_histo[j].data_[0], self->alphabet_size_);
|
119
|
+
&self->combined_histo[j].data_[0], self->alphabet_size_);
|
118
120
|
diff[j] = combined_entropy[j] - entropy - last_entropy[j];
|
119
121
|
}
|
120
122
|
|
@@ -141,7 +143,7 @@ static void FN(BlockSplitterFinishBlock)(
|
|
141
143
|
split->lengths[self->num_blocks_] = (uint32_t)self->block_size_;
|
142
144
|
split->types[self->num_blocks_] = split->types[self->num_blocks_ - 2];
|
143
145
|
BROTLI_SWAP(size_t, self->last_histogram_ix_, 0, 1);
|
144
|
-
histograms[self->last_histogram_ix_[0]] = combined_histo[1];
|
146
|
+
histograms[self->last_histogram_ix_[0]] = self->combined_histo[1];
|
145
147
|
last_entropy[1] = last_entropy[0];
|
146
148
|
last_entropy[0] = combined_entropy[1];
|
147
149
|
++self->num_blocks_;
|
@@ -152,7 +154,7 @@ static void FN(BlockSplitterFinishBlock)(
|
|
152
154
|
} else {
|
153
155
|
/* Combine this block with last block. */
|
154
156
|
split->lengths[self->num_blocks_ - 1] += (uint32_t)self->block_size_;
|
155
|
-
histograms[self->last_histogram_ix_[0]] = combined_histo[0];
|
157
|
+
histograms[self->last_histogram_ix_[0]] = self->combined_histo[0];
|
156
158
|
last_entropy[0] = combined_entropy[0];
|
157
159
|
if (split->num_types == 1) {
|
158
160
|
last_entropy[1] = last_entropy[0];
|
@@ -10,13 +10,13 @@
|
|
10
10
|
#define BROTLI_ENC_PARAMS_H_
|
11
11
|
|
12
12
|
#include <brotli/encode.h>
|
13
|
-
|
13
|
+
|
14
|
+
#include "encoder_dict.h"
|
14
15
|
|
15
16
|
typedef struct BrotliHasherParams {
|
16
17
|
int type;
|
17
18
|
int bucket_bits;
|
18
19
|
int block_bits;
|
19
|
-
int hash_len;
|
20
20
|
int num_last_distances_to_check;
|
21
21
|
} BrotliHasherParams;
|
22
22
|
|
@@ -40,7 +40,8 @@ typedef struct BrotliEncoderParams {
|
|
40
40
|
BROTLI_BOOL large_window;
|
41
41
|
BrotliHasherParams hasher;
|
42
42
|
BrotliDistanceParams dist;
|
43
|
-
|
43
|
+
/* TODO(eustas): rename to BrotliShared... */
|
44
|
+
SharedEncoderDictionary dictionary;
|
44
45
|
} BrotliEncoderParams;
|
45
46
|
|
46
47
|
#endif /* BROTLI_ENC_PARAMS_H_ */
|
@@ -10,10 +10,11 @@
|
|
10
10
|
#ifndef BROTLI_ENC_PREFIX_H_
|
11
11
|
#define BROTLI_ENC_PREFIX_H_
|
12
12
|
|
13
|
+
#include <brotli/types.h>
|
14
|
+
|
13
15
|
#include "../common/constants.h"
|
14
16
|
#include "../common/platform.h"
|
15
|
-
#include
|
16
|
-
#include "./fast_log.h"
|
17
|
+
#include "fast_log.h"
|
17
18
|
|
18
19
|
#if defined(__cplusplus) || defined(c_plusplus)
|
19
20
|
extern "C" {
|
@@ -10,9 +10,10 @@
|
|
10
10
|
#ifndef BROTLI_ENC_QUALITY_H_
|
11
11
|
#define BROTLI_ENC_QUALITY_H_
|
12
12
|
|
13
|
-
#include "../common/platform.h"
|
14
13
|
#include <brotli/encode.h>
|
15
|
-
|
14
|
+
|
15
|
+
#include "../common/platform.h"
|
16
|
+
#include "params.h"
|
16
17
|
|
17
18
|
#define FAST_ONE_PASS_COMPRESSION_QUALITY 0
|
18
19
|
#define FAST_TWO_PASS_COMPRESSION_QUALITY 1
|
@@ -118,6 +119,41 @@ static BROTLI_INLINE size_t LiteralSpreeLengthForSparseSearch(
|
|
118
119
|
return params->quality < 9 ? 64 : 512;
|
119
120
|
}
|
120
121
|
|
122
|
+
/* Quality to hasher mapping:
|
123
|
+
|
124
|
+
- q02: h02 (longest_match_quickly), b16, l5
|
125
|
+
|
126
|
+
- q03: h03 (longest_match_quickly), b17, l5
|
127
|
+
|
128
|
+
- q04: h04 (longest_match_quickly), b17, l5
|
129
|
+
- q04: h54 (longest_match_quickly), b20, l7 | for large files
|
130
|
+
|
131
|
+
- q05: h05 (longest_match ), b14, l4
|
132
|
+
- q05: h06 (longest_match64 ), b15, l5 | for large files
|
133
|
+
- q05: h40 (forgetful_chain ), b15, l4 | for small window
|
134
|
+
|
135
|
+
- q06: h05 (longest_match ), b14, l4
|
136
|
+
- q06: h06 (longest_match64 ), b15, l5 | for large files
|
137
|
+
- q06: h40 (forgetful_chain ), b15, l4 | for small window
|
138
|
+
|
139
|
+
- q07: h05 (longest_match ), b15, l4
|
140
|
+
- q07: h06 (longest_match64 ), b15, l5 | for large files
|
141
|
+
- q07: h41 (forgetful_chain ), b15, l4 | for small window
|
142
|
+
|
143
|
+
- q08: h05 (longest_match ), b15, l4
|
144
|
+
- q08: h06 (longest_match64 ), b15, l5 | for large files
|
145
|
+
- q08: h41 (forgetful_chain ), b15, l4 | for small window
|
146
|
+
|
147
|
+
- q09: h05 (longest_match ), b15, l4
|
148
|
+
- q09: h06 (longest_match64 ), b15, l5 | for large files
|
149
|
+
- q09: h42 (forgetful_chain ), b15, l4 | for small window
|
150
|
+
|
151
|
+
- q10: t10 (to_binary_tree ), b17, l128
|
152
|
+
|
153
|
+
- q11: t10 (to_binary_tree ), b17, l128
|
154
|
+
|
155
|
+
Where "q" is quality, "h" is hasher type, "b" is bucket bits,
|
156
|
+
"l" is source len. */
|
121
157
|
static BROTLI_INLINE void ChooseHasher(const BrotliEncoderParams* params,
|
122
158
|
BrotliHasherParams* hparams) {
|
123
159
|
if (params->quality > 9) {
|
@@ -132,10 +168,11 @@ static BROTLI_INLINE void ChooseHasher(const BrotliEncoderParams* params,
|
|
132
168
|
hparams->type = 6;
|
133
169
|
hparams->block_bits = params->quality - 1;
|
134
170
|
hparams->bucket_bits = 15;
|
135
|
-
hparams->hash_len = 5;
|
136
171
|
hparams->num_last_distances_to_check =
|
137
172
|
params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16;
|
138
173
|
} else {
|
174
|
+
/* TODO(eustas): often previous setting (H6) is faster and denser; consider
|
175
|
+
adding an option to use it. */
|
139
176
|
hparams->type = 5;
|
140
177
|
hparams->block_bits = params->quality - 1;
|
141
178
|
hparams->bucket_bits = params->quality < 7 ? 14 : 15;
|
@@ -11,10 +11,11 @@
|
|
11
11
|
|
12
12
|
#include <string.h> /* memcpy */
|
13
13
|
|
14
|
-
#include "../common/platform.h"
|
15
14
|
#include <brotli/types.h>
|
16
|
-
|
17
|
-
#include "
|
15
|
+
|
16
|
+
#include "../common/platform.h"
|
17
|
+
#include "memory.h"
|
18
|
+
#include "quality.h"
|
18
19
|
|
19
20
|
#if defined(__cplusplus) || defined(c_plusplus)
|
20
21
|
extern "C" {
|
@@ -0,0 +1,104 @@
|
|
1
|
+
/* Copyright 2022 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
/* Encoder state. */
|
8
|
+
|
9
|
+
#ifndef BROTLI_ENC_STATE_H_
|
10
|
+
#define BROTLI_ENC_STATE_H_
|
11
|
+
|
12
|
+
#include <brotli/types.h>
|
13
|
+
|
14
|
+
#include "command.h"
|
15
|
+
#include "compress_fragment.h"
|
16
|
+
#include "compress_fragment_two_pass.h"
|
17
|
+
#include "hash.h"
|
18
|
+
#include "memory.h"
|
19
|
+
#include "params.h"
|
20
|
+
#include "ringbuffer.h"
|
21
|
+
|
22
|
+
typedef enum BrotliEncoderStreamState {
|
23
|
+
/* Default state. */
|
24
|
+
BROTLI_STREAM_PROCESSING = 0,
|
25
|
+
/* Intermediate state; after next block is emitted, byte-padding should be
|
26
|
+
performed before getting back to default state. */
|
27
|
+
BROTLI_STREAM_FLUSH_REQUESTED = 1,
|
28
|
+
/* Last metablock was produced; no more input is acceptable. */
|
29
|
+
BROTLI_STREAM_FINISHED = 2,
|
30
|
+
/* Flushing compressed block and writing meta-data block header. */
|
31
|
+
BROTLI_STREAM_METADATA_HEAD = 3,
|
32
|
+
/* Writing metadata block body. */
|
33
|
+
BROTLI_STREAM_METADATA_BODY = 4
|
34
|
+
} BrotliEncoderStreamState;
|
35
|
+
|
36
|
+
typedef enum BrotliEncoderFlintState {
|
37
|
+
BROTLI_FLINT_NEEDS_2_BYTES = 2,
|
38
|
+
BROTLI_FLINT_NEEDS_1_BYTE = 1,
|
39
|
+
BROTLI_FLINT_WAITING_FOR_PROCESSING = 0,
|
40
|
+
BROTLI_FLINT_WAITING_FOR_FLUSHING = -1,
|
41
|
+
BROTLI_FLINT_DONE = -2
|
42
|
+
} BrotliEncoderFlintState;
|
43
|
+
|
44
|
+
typedef struct BrotliEncoderStateStruct {
|
45
|
+
BrotliEncoderParams params;
|
46
|
+
|
47
|
+
MemoryManager memory_manager_;
|
48
|
+
|
49
|
+
uint64_t input_pos_;
|
50
|
+
RingBuffer ringbuffer_;
|
51
|
+
size_t cmd_alloc_size_;
|
52
|
+
Command* commands_;
|
53
|
+
size_t num_commands_;
|
54
|
+
size_t num_literals_;
|
55
|
+
size_t last_insert_len_;
|
56
|
+
uint64_t last_flush_pos_;
|
57
|
+
uint64_t last_processed_pos_;
|
58
|
+
int dist_cache_[BROTLI_NUM_DISTANCE_SHORT_CODES];
|
59
|
+
int saved_dist_cache_[4];
|
60
|
+
uint16_t last_bytes_;
|
61
|
+
uint8_t last_bytes_bits_;
|
62
|
+
/* "Flint" is a tiny uncompressed block emitted before the continuation
|
63
|
+
block to unwire literal context from previous data. Despite being int8_t,
|
64
|
+
field is actually BrotliEncoderFlintState enum. */
|
65
|
+
int8_t flint_;
|
66
|
+
uint8_t prev_byte_;
|
67
|
+
uint8_t prev_byte2_;
|
68
|
+
size_t storage_size_;
|
69
|
+
uint8_t* storage_;
|
70
|
+
|
71
|
+
Hasher hasher_;
|
72
|
+
|
73
|
+
/* Hash table for FAST_ONE_PASS_COMPRESSION_QUALITY mode. */
|
74
|
+
int small_table_[1 << 10]; /* 4KiB */
|
75
|
+
int* large_table_; /* Allocated only when needed */
|
76
|
+
size_t large_table_size_;
|
77
|
+
|
78
|
+
BrotliOnePassArena* one_pass_arena_;
|
79
|
+
BrotliTwoPassArena* two_pass_arena_;
|
80
|
+
|
81
|
+
/* Command and literal buffers for FAST_TWO_PASS_COMPRESSION_QUALITY. */
|
82
|
+
uint32_t* command_buf_;
|
83
|
+
uint8_t* literal_buf_;
|
84
|
+
|
85
|
+
uint64_t total_in_;
|
86
|
+
uint8_t* next_out_;
|
87
|
+
size_t available_out_;
|
88
|
+
uint64_t total_out_;
|
89
|
+
/* Temporary buffer for padding flush bits or metadata block header / body. */
|
90
|
+
union {
|
91
|
+
uint64_t u64[2];
|
92
|
+
uint8_t u8[16];
|
93
|
+
} tiny_buf_;
|
94
|
+
uint32_t remaining_metadata_bytes_;
|
95
|
+
BrotliEncoderStreamState stream_state_;
|
96
|
+
|
97
|
+
BROTLI_BOOL is_last_block_emitted_;
|
98
|
+
BROTLI_BOOL is_initialized_;
|
99
|
+
} BrotliEncoderStateStruct;
|
100
|
+
|
101
|
+
typedef struct BrotliEncoderStateStruct BrotliEncoderStateInternal;
|
102
|
+
#define BrotliEncoderState BrotliEncoderStateInternal
|
103
|
+
|
104
|
+
#endif // BROTLI_ENC_STATE_H_
|