brotli 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +6 -3
- data/.github/workflows/publish.yml +7 -17
- data/.gitmodules +1 -1
- data/README.md +2 -2
- data/ext/brotli/brotli.c +1 -0
- data/ext/brotli/extconf.rb +6 -0
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +4 -1
- data/vendor/brotli/c/common/constants.c +1 -1
- data/vendor/brotli/c/common/constants.h +2 -1
- data/vendor/brotli/c/common/context.c +1 -1
- data/vendor/brotli/c/common/dictionary.c +5 -3
- data/vendor/brotli/c/common/platform.c +2 -1
- data/vendor/brotli/c/common/platform.h +60 -113
- data/vendor/brotli/c/common/shared_dictionary.c +521 -0
- data/vendor/brotli/c/common/shared_dictionary_internal.h +75 -0
- data/vendor/brotli/c/common/transform.c +1 -1
- data/vendor/brotli/c/common/version.h +31 -6
- data/vendor/brotli/c/dec/bit_reader.c +10 -8
- data/vendor/brotli/c/dec/bit_reader.h +172 -100
- data/vendor/brotli/c/dec/decode.c +467 -200
- data/vendor/brotli/c/dec/huffman.c +7 -4
- data/vendor/brotli/c/dec/huffman.h +2 -1
- data/vendor/brotli/c/dec/prefix.h +2 -1
- data/vendor/brotli/c/dec/state.c +33 -9
- data/vendor/brotli/c/dec/state.h +70 -35
- data/vendor/brotli/c/enc/backward_references.c +81 -19
- data/vendor/brotli/c/enc/backward_references.h +5 -4
- data/vendor/brotli/c/enc/backward_references_hq.c +148 -52
- data/vendor/brotli/c/enc/backward_references_hq.h +6 -5
- data/vendor/brotli/c/enc/backward_references_inc.h +31 -5
- data/vendor/brotli/c/enc/bit_cost.c +8 -7
- data/vendor/brotli/c/enc/bit_cost.h +5 -4
- data/vendor/brotli/c/enc/block_splitter.c +37 -14
- data/vendor/brotli/c/enc/block_splitter.h +5 -4
- data/vendor/brotli/c/enc/block_splitter_inc.h +86 -45
- data/vendor/brotli/c/enc/brotli_bit_stream.c +132 -110
- data/vendor/brotli/c/enc/brotli_bit_stream.h +11 -6
- data/vendor/brotli/c/enc/cluster.c +10 -9
- data/vendor/brotli/c/enc/cluster.h +7 -6
- data/vendor/brotli/c/enc/cluster_inc.h +25 -20
- data/vendor/brotli/c/enc/command.c +1 -1
- data/vendor/brotli/c/enc/command.h +5 -4
- data/vendor/brotli/c/enc/compound_dictionary.c +207 -0
- data/vendor/brotli/c/enc/compound_dictionary.h +74 -0
- data/vendor/brotli/c/enc/compress_fragment.c +93 -83
- data/vendor/brotli/c/enc/compress_fragment.h +32 -7
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +99 -87
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +21 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +3 -1
- data/vendor/brotli/c/enc/encode.c +473 -404
- data/vendor/brotli/c/enc/encoder_dict.c +611 -4
- data/vendor/brotli/c/enc/encoder_dict.h +117 -3
- data/vendor/brotli/c/enc/entropy_encode.c +3 -2
- data/vendor/brotli/c/enc/entropy_encode.h +2 -1
- data/vendor/brotli/c/enc/entropy_encode_static.h +5 -2
- data/vendor/brotli/c/enc/fast_log.c +1 -1
- data/vendor/brotli/c/enc/fast_log.h +2 -1
- data/vendor/brotli/c/enc/find_match_length.h +15 -22
- data/vendor/brotli/c/enc/hash.h +285 -45
- data/vendor/brotli/c/enc/hash_composite_inc.h +26 -11
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +20 -18
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +34 -39
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +6 -10
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +4 -4
- data/vendor/brotli/c/enc/hash_rolling_inc.h +4 -4
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +6 -5
- data/vendor/brotli/c/enc/histogram.c +4 -4
- data/vendor/brotli/c/enc/histogram.h +7 -6
- data/vendor/brotli/c/enc/literal_cost.c +20 -15
- data/vendor/brotli/c/enc/literal_cost.h +4 -2
- data/vendor/brotli/c/enc/memory.c +29 -5
- data/vendor/brotli/c/enc/memory.h +19 -2
- data/vendor/brotli/c/enc/metablock.c +72 -58
- data/vendor/brotli/c/enc/metablock.h +9 -8
- data/vendor/brotli/c/enc/metablock_inc.h +8 -6
- data/vendor/brotli/c/enc/params.h +4 -3
- data/vendor/brotli/c/enc/prefix.h +3 -2
- data/vendor/brotli/c/enc/quality.h +40 -3
- data/vendor/brotli/c/enc/ringbuffer.h +4 -3
- data/vendor/brotli/c/enc/state.h +104 -0
- data/vendor/brotli/c/enc/static_dict.c +60 -4
- data/vendor/brotli/c/enc/static_dict.h +3 -2
- data/vendor/brotli/c/enc/static_dict_lut.h +2 -0
- data/vendor/brotli/c/enc/utf8_util.c +1 -1
- data/vendor/brotli/c/enc/utf8_util.h +2 -1
- data/vendor/brotli/c/enc/write_bits.h +2 -1
- data/vendor/brotli/c/include/brotli/decode.h +67 -2
- data/vendor/brotli/c/include/brotli/encode.h +55 -2
- data/vendor/brotli/c/include/brotli/port.h +28 -11
- data/vendor/brotli/c/include/brotli/shared_dictionary.h +100 -0
- metadata +9 -3
@@ -8,20 +8,21 @@
|
|
8
8
|
compression algorithms here, just the right ordering of bits to match the
|
9
9
|
specs. */
|
10
10
|
|
11
|
-
#include "
|
11
|
+
#include "brotli_bit_stream.h"
|
12
12
|
|
13
13
|
#include <string.h> /* memcpy, memset */
|
14
14
|
|
15
|
+
#include <brotli/types.h>
|
16
|
+
|
15
17
|
#include "../common/constants.h"
|
16
18
|
#include "../common/context.h"
|
17
19
|
#include "../common/platform.h"
|
18
|
-
#include
|
19
|
-
#include "
|
20
|
-
#include "
|
21
|
-
#include "
|
22
|
-
#include "
|
23
|
-
#include "
|
24
|
-
#include "./write_bits.h"
|
20
|
+
#include "entropy_encode.h"
|
21
|
+
#include "entropy_encode_static.h"
|
22
|
+
#include "fast_log.h"
|
23
|
+
#include "histogram.h"
|
24
|
+
#include "memory.h"
|
25
|
+
#include "write_bits.h"
|
25
26
|
|
26
27
|
#if defined(__cplusplus) || defined(c_plusplus)
|
27
28
|
extern "C" {
|
@@ -286,6 +287,7 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
|
|
286
287
|
/* Write the Huffman tree into the brotli-representation.
|
287
288
|
The command alphabet is the largest, so this allocation will fit all
|
288
289
|
alphabets. */
|
290
|
+
/* TODO(eustas): fix me */
|
289
291
|
uint8_t huffman_tree[BROTLI_NUM_COMMAND_SYMBOLS];
|
290
292
|
uint8_t huffman_tree_extra_bits[BROTLI_NUM_COMMAND_SYMBOLS];
|
291
293
|
size_t huffman_tree_size = 0;
|
@@ -400,7 +402,7 @@ static BROTLI_INLINE BROTLI_BOOL SortHuffmanTree(
|
|
400
402
|
return TO_BROTLI_BOOL(v0->total_count_ < v1->total_count_);
|
401
403
|
}
|
402
404
|
|
403
|
-
void BrotliBuildAndStoreHuffmanTreeFast(
|
405
|
+
void BrotliBuildAndStoreHuffmanTreeFast(HuffmanTree* tree,
|
404
406
|
const uint32_t* histogram,
|
405
407
|
const size_t histogram_total,
|
406
408
|
const size_t max_bits,
|
@@ -432,10 +434,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
|
|
432
434
|
|
433
435
|
memset(depth, 0, length * sizeof(depth[0]));
|
434
436
|
{
|
435
|
-
const size_t max_tree_size = 2 * length + 1;
|
436
|
-
HuffmanTree* tree = BROTLI_ALLOC(m, HuffmanTree, max_tree_size);
|
437
437
|
uint32_t count_limit;
|
438
|
-
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
|
439
438
|
for (count_limit = 1; ; count_limit *= 2) {
|
440
439
|
HuffmanTree* node = tree;
|
441
440
|
size_t l;
|
@@ -500,7 +499,6 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
|
|
500
499
|
}
|
501
500
|
}
|
502
501
|
}
|
503
|
-
BROTLI_FREE(m, tree);
|
504
502
|
}
|
505
503
|
BrotliConvertBitDepthsToSymbols(depth, length, bits);
|
506
504
|
if (count <= 4) {
|
@@ -677,7 +675,14 @@ static void RunLengthCodeZeros(const size_t in_size,
|
|
677
675
|
|
678
676
|
#define SYMBOL_BITS 9
|
679
677
|
|
678
|
+
typedef struct EncodeContextMapArena {
|
679
|
+
uint32_t histogram[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
|
680
|
+
uint8_t depths[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
|
681
|
+
uint16_t bits[BROTLI_MAX_CONTEXT_MAP_SYMBOLS];
|
682
|
+
} EncodeContextMapArena;
|
683
|
+
|
680
684
|
static void EncodeContextMap(MemoryManager* m,
|
685
|
+
EncodeContextMapArena* arena,
|
681
686
|
const uint32_t* context_map,
|
682
687
|
size_t context_map_size,
|
683
688
|
size_t num_clusters,
|
@@ -687,10 +692,10 @@ static void EncodeContextMap(MemoryManager* m,
|
|
687
692
|
uint32_t* rle_symbols;
|
688
693
|
uint32_t max_run_length_prefix = 6;
|
689
694
|
size_t num_rle_symbols = 0;
|
690
|
-
uint32_t histogram
|
695
|
+
uint32_t* BROTLI_RESTRICT const histogram = arena->histogram;
|
691
696
|
static const uint32_t kSymbolMask = (1u << SYMBOL_BITS) - 1u;
|
692
|
-
uint8_t depths
|
693
|
-
uint16_t bits
|
697
|
+
uint8_t* BROTLI_RESTRICT const depths = arena->depths;
|
698
|
+
uint16_t* BROTLI_RESTRICT const bits = arena->bits;
|
694
699
|
|
695
700
|
StoreVarLenUint8(num_clusters - 1, storage_ix, storage);
|
696
701
|
|
@@ -703,7 +708,7 @@ static void EncodeContextMap(MemoryManager* m,
|
|
703
708
|
MoveToFrontTransform(context_map, context_map_size, rle_symbols);
|
704
709
|
RunLengthCodeZeros(context_map_size, rle_symbols,
|
705
710
|
&num_rle_symbols, &max_run_length_prefix);
|
706
|
-
memset(histogram, 0, sizeof(histogram));
|
711
|
+
memset(histogram, 0, sizeof(arena->histogram));
|
707
712
|
for (i = 0; i < num_rle_symbols; ++i) {
|
708
713
|
++histogram[rle_symbols[i] & kSymbolMask];
|
709
714
|
}
|
@@ -774,7 +779,7 @@ static void BuildAndStoreBlockSplitCode(const uint8_t* types,
|
|
774
779
|
++length_histo[BlockLengthPrefixCode(lengths[i])];
|
775
780
|
}
|
776
781
|
StoreVarLenUint8(num_types - 1, storage_ix, storage);
|
777
|
-
if (num_types > 1) { /* TODO: else? could StoreBlockSwitch occur? */
|
782
|
+
if (num_types > 1) { /* TODO(eustas): else? could StoreBlockSwitch occur? */
|
778
783
|
BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, num_types + 2, tree,
|
779
784
|
&code->type_depths[0], &code->type_bits[0],
|
780
785
|
storage_ix, storage);
|
@@ -787,7 +792,8 @@ static void BuildAndStoreBlockSplitCode(const uint8_t* types,
|
|
787
792
|
}
|
788
793
|
|
789
794
|
/* Stores a context map where the histogram type is always the block type. */
|
790
|
-
static void StoreTrivialContextMap(
|
795
|
+
static void StoreTrivialContextMap(EncodeContextMapArena* arena,
|
796
|
+
size_t num_types,
|
791
797
|
size_t context_bits,
|
792
798
|
HuffmanTree* tree,
|
793
799
|
size_t* storage_ix,
|
@@ -797,9 +803,9 @@ static void StoreTrivialContextMap(size_t num_types,
|
|
797
803
|
size_t repeat_code = context_bits - 1u;
|
798
804
|
size_t repeat_bits = (1u << repeat_code) - 1u;
|
799
805
|
size_t alphabet_size = num_types + repeat_code;
|
800
|
-
uint32_t histogram
|
801
|
-
uint8_t depths
|
802
|
-
uint16_t bits
|
806
|
+
uint32_t* BROTLI_RESTRICT const histogram = arena->histogram;
|
807
|
+
uint8_t* BROTLI_RESTRICT const depths = arena->depths;
|
808
|
+
uint16_t* BROTLI_RESTRICT const bits = arena->bits;
|
803
809
|
size_t i;
|
804
810
|
memset(histogram, 0, alphabet_size * sizeof(histogram[0]));
|
805
811
|
/* Write RLEMAX. */
|
@@ -914,17 +920,17 @@ static void StoreSymbolWithContext(BlockEncoder* self, size_t symbol,
|
|
914
920
|
|
915
921
|
#define FN(X) X ## Literal
|
916
922
|
/* NOLINTNEXTLINE(build/include) */
|
917
|
-
#include "
|
923
|
+
#include "block_encoder_inc.h"
|
918
924
|
#undef FN
|
919
925
|
|
920
926
|
#define FN(X) X ## Command
|
921
927
|
/* NOLINTNEXTLINE(build/include) */
|
922
|
-
#include "
|
928
|
+
#include "block_encoder_inc.h"
|
923
929
|
#undef FN
|
924
930
|
|
925
931
|
#define FN(X) X ## Distance
|
926
932
|
/* NOLINTNEXTLINE(build/include) */
|
927
|
-
#include "
|
933
|
+
#include "block_encoder_inc.h"
|
928
934
|
#undef FN
|
929
935
|
|
930
936
|
static void JumpToByteBoundary(size_t* storage_ix, uint8_t* storage) {
|
@@ -932,6 +938,13 @@ static void JumpToByteBoundary(size_t* storage_ix, uint8_t* storage) {
|
|
932
938
|
storage[*storage_ix >> 3] = 0;
|
933
939
|
}
|
934
940
|
|
941
|
+
typedef struct StoreMetablockArena {
|
942
|
+
BlockEncoder literal_enc;
|
943
|
+
BlockEncoder command_enc;
|
944
|
+
BlockEncoder distance_enc;
|
945
|
+
EncodeContextMapArena context_map_arena;
|
946
|
+
} StoreMetablockArena;
|
947
|
+
|
935
948
|
void BrotliStoreMetaBlock(MemoryManager* m,
|
936
949
|
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
|
937
950
|
uint8_t prev_byte, uint8_t prev_byte2, BROTLI_BOOL is_last,
|
@@ -945,9 +958,10 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
945
958
|
uint32_t num_effective_distance_symbols = params->dist.alphabet_size_limit;
|
946
959
|
HuffmanTree* tree;
|
947
960
|
ContextLut literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
|
948
|
-
|
949
|
-
BlockEncoder
|
950
|
-
BlockEncoder
|
961
|
+
StoreMetablockArena* arena = NULL;
|
962
|
+
BlockEncoder* literal_enc = NULL;
|
963
|
+
BlockEncoder* command_enc = NULL;
|
964
|
+
BlockEncoder* distance_enc = NULL;
|
951
965
|
const BrotliDistanceParams* dist = ¶ms->dist;
|
952
966
|
BROTLI_DCHECK(
|
953
967
|
num_effective_distance_symbols <= BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS);
|
@@ -955,21 +969,24 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
955
969
|
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
|
956
970
|
|
957
971
|
tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
|
958
|
-
|
959
|
-
|
972
|
+
arena = BROTLI_ALLOC(m, StoreMetablockArena, 1);
|
973
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree) || BROTLI_IS_NULL(arena)) return;
|
974
|
+
literal_enc = &arena->literal_enc;
|
975
|
+
command_enc = &arena->command_enc;
|
976
|
+
distance_enc = &arena->distance_enc;
|
977
|
+
InitBlockEncoder(literal_enc, BROTLI_NUM_LITERAL_SYMBOLS,
|
960
978
|
mb->literal_split.num_types, mb->literal_split.types,
|
961
979
|
mb->literal_split.lengths, mb->literal_split.num_blocks);
|
962
|
-
InitBlockEncoder(
|
980
|
+
InitBlockEncoder(command_enc, BROTLI_NUM_COMMAND_SYMBOLS,
|
963
981
|
mb->command_split.num_types, mb->command_split.types,
|
964
982
|
mb->command_split.lengths, mb->command_split.num_blocks);
|
965
|
-
InitBlockEncoder(
|
983
|
+
InitBlockEncoder(distance_enc, num_effective_distance_symbols,
|
966
984
|
mb->distance_split.num_types, mb->distance_split.types,
|
967
985
|
mb->distance_split.lengths, mb->distance_split.num_blocks);
|
968
986
|
|
969
|
-
BuildAndStoreBlockSwitchEntropyCodes(
|
970
|
-
BuildAndStoreBlockSwitchEntropyCodes(
|
971
|
-
BuildAndStoreBlockSwitchEntropyCodes(
|
972
|
-
&distance_enc, tree, storage_ix, storage);
|
987
|
+
BuildAndStoreBlockSwitchEntropyCodes(literal_enc, tree, storage_ix, storage);
|
988
|
+
BuildAndStoreBlockSwitchEntropyCodes(command_enc, tree, storage_ix, storage);
|
989
|
+
BuildAndStoreBlockSwitchEntropyCodes(distance_enc, tree, storage_ix, storage);
|
973
990
|
|
974
991
|
BrotliWriteBits(2, dist->distance_postfix_bits, storage_ix, storage);
|
975
992
|
BrotliWriteBits(
|
@@ -980,34 +997,36 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
980
997
|
}
|
981
998
|
|
982
999
|
if (mb->literal_context_map_size == 0) {
|
983
|
-
StoreTrivialContextMap(
|
1000
|
+
StoreTrivialContextMap(
|
1001
|
+
&arena->context_map_arena, mb->literal_histograms_size,
|
984
1002
|
BROTLI_LITERAL_CONTEXT_BITS, tree, storage_ix, storage);
|
985
1003
|
} else {
|
986
|
-
EncodeContextMap(m,
|
1004
|
+
EncodeContextMap(m, &arena->context_map_arena,
|
987
1005
|
mb->literal_context_map, mb->literal_context_map_size,
|
988
1006
|
mb->literal_histograms_size, tree, storage_ix, storage);
|
989
1007
|
if (BROTLI_IS_OOM(m)) return;
|
990
1008
|
}
|
991
1009
|
|
992
1010
|
if (mb->distance_context_map_size == 0) {
|
993
|
-
StoreTrivialContextMap(
|
1011
|
+
StoreTrivialContextMap(
|
1012
|
+
&arena->context_map_arena, mb->distance_histograms_size,
|
994
1013
|
BROTLI_DISTANCE_CONTEXT_BITS, tree, storage_ix, storage);
|
995
1014
|
} else {
|
996
|
-
EncodeContextMap(m,
|
1015
|
+
EncodeContextMap(m, &arena->context_map_arena,
|
997
1016
|
mb->distance_context_map, mb->distance_context_map_size,
|
998
1017
|
mb->distance_histograms_size, tree, storage_ix, storage);
|
999
1018
|
if (BROTLI_IS_OOM(m)) return;
|
1000
1019
|
}
|
1001
1020
|
|
1002
|
-
BuildAndStoreEntropyCodesLiteral(m,
|
1021
|
+
BuildAndStoreEntropyCodesLiteral(m, literal_enc, mb->literal_histograms,
|
1003
1022
|
mb->literal_histograms_size, BROTLI_NUM_LITERAL_SYMBOLS, tree,
|
1004
1023
|
storage_ix, storage);
|
1005
1024
|
if (BROTLI_IS_OOM(m)) return;
|
1006
|
-
BuildAndStoreEntropyCodesCommand(m,
|
1025
|
+
BuildAndStoreEntropyCodesCommand(m, command_enc, mb->command_histograms,
|
1007
1026
|
mb->command_histograms_size, BROTLI_NUM_COMMAND_SYMBOLS, tree,
|
1008
1027
|
storage_ix, storage);
|
1009
1028
|
if (BROTLI_IS_OOM(m)) return;
|
1010
|
-
BuildAndStoreEntropyCodesDistance(m,
|
1029
|
+
BuildAndStoreEntropyCodesDistance(m, distance_enc, mb->distance_histograms,
|
1011
1030
|
mb->distance_histograms_size, num_distance_symbols, tree,
|
1012
1031
|
storage_ix, storage);
|
1013
1032
|
if (BROTLI_IS_OOM(m)) return;
|
@@ -1016,12 +1035,12 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
1016
1035
|
for (i = 0; i < n_commands; ++i) {
|
1017
1036
|
const Command cmd = commands[i];
|
1018
1037
|
size_t cmd_code = cmd.cmd_prefix_;
|
1019
|
-
StoreSymbol(
|
1038
|
+
StoreSymbol(command_enc, cmd_code, storage_ix, storage);
|
1020
1039
|
StoreCommandExtra(&cmd, storage_ix, storage);
|
1021
1040
|
if (mb->literal_context_map_size == 0) {
|
1022
1041
|
size_t j;
|
1023
1042
|
for (j = cmd.insert_len_; j != 0; --j) {
|
1024
|
-
StoreSymbol(
|
1043
|
+
StoreSymbol(literal_enc, input[pos & mask], storage_ix, storage);
|
1025
1044
|
++pos;
|
1026
1045
|
}
|
1027
1046
|
} else {
|
@@ -1030,7 +1049,7 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
1030
1049
|
size_t context =
|
1031
1050
|
BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut);
|
1032
1051
|
uint8_t literal = input[pos & mask];
|
1033
|
-
StoreSymbolWithContext(
|
1052
|
+
StoreSymbolWithContext(literal_enc, literal, context,
|
1034
1053
|
mb->literal_context_map, storage_ix, storage,
|
1035
1054
|
BROTLI_LITERAL_CONTEXT_BITS);
|
1036
1055
|
prev_byte2 = prev_byte;
|
@@ -1047,10 +1066,10 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
1047
1066
|
uint32_t distnumextra = cmd.dist_prefix_ >> 10;
|
1048
1067
|
uint64_t distextra = cmd.dist_extra_;
|
1049
1068
|
if (mb->distance_context_map_size == 0) {
|
1050
|
-
StoreSymbol(
|
1069
|
+
StoreSymbol(distance_enc, dist_code, storage_ix, storage);
|
1051
1070
|
} else {
|
1052
1071
|
size_t context = CommandDistanceContext(&cmd);
|
1053
|
-
StoreSymbolWithContext(
|
1072
|
+
StoreSymbolWithContext(distance_enc, dist_code, context,
|
1054
1073
|
mb->distance_context_map, storage_ix, storage,
|
1055
1074
|
BROTLI_DISTANCE_CONTEXT_BITS);
|
1056
1075
|
}
|
@@ -1058,9 +1077,10 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
1058
1077
|
}
|
1059
1078
|
}
|
1060
1079
|
}
|
1061
|
-
CleanupBlockEncoder(m,
|
1062
|
-
CleanupBlockEncoder(m,
|
1063
|
-
CleanupBlockEncoder(m,
|
1080
|
+
CleanupBlockEncoder(m, distance_enc);
|
1081
|
+
CleanupBlockEncoder(m, command_enc);
|
1082
|
+
CleanupBlockEncoder(m, literal_enc);
|
1083
|
+
BROTLI_FREE(m, arena);
|
1064
1084
|
if (is_last) {
|
1065
1085
|
JumpToByteBoundary(storage_ix, storage);
|
1066
1086
|
}
|
@@ -1131,54 +1151,60 @@ static void StoreDataWithHuffmanCodes(const uint8_t* input,
|
|
1131
1151
|
}
|
1132
1152
|
}
|
1133
1153
|
|
1134
|
-
|
1135
|
-
|
1136
|
-
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
|
1137
|
-
const Command* commands, size_t n_commands,
|
1138
|
-
size_t* storage_ix, uint8_t* storage) {
|
1154
|
+
/* TODO(eustas): pull alloc/dealloc to caller? */
|
1155
|
+
typedef struct MetablockArena {
|
1139
1156
|
HistogramLiteral lit_histo;
|
1140
1157
|
HistogramCommand cmd_histo;
|
1141
1158
|
HistogramDistance dist_histo;
|
1159
|
+
/* TODO(eustas): merge bits and depth? */
|
1142
1160
|
uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS];
|
1143
1161
|
uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
|
1144
1162
|
uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
|
1145
1163
|
uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
|
1146
1164
|
uint8_t dist_depth[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1147
1165
|
uint16_t dist_bits[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1148
|
-
HuffmanTree
|
1166
|
+
HuffmanTree tree[MAX_HUFFMAN_TREE_SIZE];
|
1167
|
+
} MetablockArena;
|
1168
|
+
|
1169
|
+
void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
1170
|
+
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
|
1171
|
+
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
|
1172
|
+
const Command* commands, size_t n_commands,
|
1173
|
+
size_t* storage_ix, uint8_t* storage) {
|
1174
|
+
MetablockArena* arena = BROTLI_ALLOC(m, MetablockArena, 1);
|
1149
1175
|
uint32_t num_distance_symbols = params->dist.alphabet_size_max;
|
1176
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
|
1150
1177
|
|
1151
1178
|
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
|
1152
1179
|
|
1153
|
-
HistogramClearLiteral(&lit_histo);
|
1154
|
-
HistogramClearCommand(&cmd_histo);
|
1155
|
-
HistogramClearDistance(&dist_histo);
|
1180
|
+
HistogramClearLiteral(&arena->lit_histo);
|
1181
|
+
HistogramClearCommand(&arena->cmd_histo);
|
1182
|
+
HistogramClearDistance(&arena->dist_histo);
|
1156
1183
|
|
1157
1184
|
BuildHistograms(input, start_pos, mask, commands, n_commands,
|
1158
|
-
&lit_histo, &cmd_histo, &dist_histo);
|
1185
|
+
&arena->lit_histo, &arena->cmd_histo, &arena->dist_histo);
|
1159
1186
|
|
1160
1187
|
BrotliWriteBits(13, 0, storage_ix, storage);
|
1161
1188
|
|
1162
|
-
|
1163
|
-
|
1164
|
-
|
1165
|
-
BROTLI_NUM_LITERAL_SYMBOLS, tree,
|
1166
|
-
lit_depth, lit_bits,
|
1189
|
+
BuildAndStoreHuffmanTree(arena->lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS,
|
1190
|
+
BROTLI_NUM_LITERAL_SYMBOLS, arena->tree,
|
1191
|
+
arena->lit_depth, arena->lit_bits,
|
1167
1192
|
storage_ix, storage);
|
1168
|
-
BuildAndStoreHuffmanTree(cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS,
|
1169
|
-
BROTLI_NUM_COMMAND_SYMBOLS, tree,
|
1170
|
-
cmd_depth, cmd_bits,
|
1193
|
+
BuildAndStoreHuffmanTree(arena->cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS,
|
1194
|
+
BROTLI_NUM_COMMAND_SYMBOLS, arena->tree,
|
1195
|
+
arena->cmd_depth, arena->cmd_bits,
|
1171
1196
|
storage_ix, storage);
|
1172
|
-
BuildAndStoreHuffmanTree(dist_histo.data_,
|
1173
|
-
|
1174
|
-
|
1197
|
+
BuildAndStoreHuffmanTree(arena->dist_histo.data_,
|
1198
|
+
MAX_SIMPLE_DISTANCE_ALPHABET_SIZE,
|
1199
|
+
num_distance_symbols, arena->tree,
|
1200
|
+
arena->dist_depth, arena->dist_bits,
|
1175
1201
|
storage_ix, storage);
|
1176
|
-
BROTLI_FREE(m, tree);
|
1177
1202
|
StoreDataWithHuffmanCodes(input, start_pos, mask, commands,
|
1178
|
-
n_commands, lit_depth, lit_bits,
|
1179
|
-
cmd_depth, cmd_bits,
|
1180
|
-
dist_depth, dist_bits,
|
1203
|
+
n_commands, arena->lit_depth, arena->lit_bits,
|
1204
|
+
arena->cmd_depth, arena->cmd_bits,
|
1205
|
+
arena->dist_depth, arena->dist_bits,
|
1181
1206
|
storage_ix, storage);
|
1207
|
+
BROTLI_FREE(m, arena);
|
1182
1208
|
if (is_last) {
|
1183
1209
|
JumpToByteBoundary(storage_ix, storage);
|
1184
1210
|
}
|
@@ -1189,9 +1215,11 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
|
|
1189
1215
|
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
|
1190
1216
|
const Command* commands, size_t n_commands,
|
1191
1217
|
size_t* storage_ix, uint8_t* storage) {
|
1218
|
+
MetablockArena* arena = BROTLI_ALLOC(m, MetablockArena, 1);
|
1192
1219
|
uint32_t num_distance_symbols = params->dist.alphabet_size_max;
|
1193
1220
|
uint32_t distance_alphabet_bits =
|
1194
1221
|
Log2FloorNonZero(num_distance_symbols - 1) + 1;
|
1222
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
|
1195
1223
|
|
1196
1224
|
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
|
1197
1225
|
|
@@ -1202,8 +1230,6 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
|
|
1202
1230
|
size_t pos = start_pos;
|
1203
1231
|
size_t num_literals = 0;
|
1204
1232
|
size_t i;
|
1205
|
-
uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS];
|
1206
|
-
uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
|
1207
1233
|
for (i = 0; i < n_commands; ++i) {
|
1208
1234
|
const Command cmd = commands[i];
|
1209
1235
|
size_t j;
|
@@ -1214,61 +1240,50 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
|
|
1214
1240
|
num_literals += cmd.insert_len_;
|
1215
1241
|
pos += CommandCopyLen(&cmd);
|
1216
1242
|
}
|
1217
|
-
BrotliBuildAndStoreHuffmanTreeFast(
|
1243
|
+
BrotliBuildAndStoreHuffmanTreeFast(arena->tree, histogram, num_literals,
|
1218
1244
|
/* max_bits = */ 8,
|
1219
|
-
lit_depth, lit_bits,
|
1245
|
+
arena->lit_depth, arena->lit_bits,
|
1220
1246
|
storage_ix, storage);
|
1221
|
-
if (BROTLI_IS_OOM(m)) return;
|
1222
1247
|
StoreStaticCommandHuffmanTree(storage_ix, storage);
|
1223
1248
|
StoreStaticDistanceHuffmanTree(storage_ix, storage);
|
1224
1249
|
StoreDataWithHuffmanCodes(input, start_pos, mask, commands,
|
1225
|
-
n_commands, lit_depth, lit_bits,
|
1250
|
+
n_commands, arena->lit_depth, arena->lit_bits,
|
1226
1251
|
kStaticCommandCodeDepth,
|
1227
1252
|
kStaticCommandCodeBits,
|
1228
1253
|
kStaticDistanceCodeDepth,
|
1229
1254
|
kStaticDistanceCodeBits,
|
1230
1255
|
storage_ix, storage);
|
1231
1256
|
} else {
|
1232
|
-
|
1233
|
-
|
1234
|
-
|
1235
|
-
uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS];
|
1236
|
-
uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
|
1237
|
-
uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
|
1238
|
-
uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
|
1239
|
-
uint8_t dist_depth[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1240
|
-
uint16_t dist_bits[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1241
|
-
HistogramClearLiteral(&lit_histo);
|
1242
|
-
HistogramClearCommand(&cmd_histo);
|
1243
|
-
HistogramClearDistance(&dist_histo);
|
1257
|
+
HistogramClearLiteral(&arena->lit_histo);
|
1258
|
+
HistogramClearCommand(&arena->cmd_histo);
|
1259
|
+
HistogramClearDistance(&arena->dist_histo);
|
1244
1260
|
BuildHistograms(input, start_pos, mask, commands, n_commands,
|
1245
|
-
&lit_histo, &cmd_histo, &dist_histo);
|
1246
|
-
BrotliBuildAndStoreHuffmanTreeFast(
|
1247
|
-
lit_histo.total_count_,
|
1261
|
+
&arena->lit_histo, &arena->cmd_histo, &arena->dist_histo);
|
1262
|
+
BrotliBuildAndStoreHuffmanTreeFast(arena->tree, arena->lit_histo.data_,
|
1263
|
+
arena->lit_histo.total_count_,
|
1248
1264
|
/* max_bits = */ 8,
|
1249
|
-
lit_depth, lit_bits,
|
1265
|
+
arena->lit_depth, arena->lit_bits,
|
1250
1266
|
storage_ix, storage);
|
1251
|
-
|
1252
|
-
|
1253
|
-
cmd_histo.total_count_,
|
1267
|
+
BrotliBuildAndStoreHuffmanTreeFast(arena->tree, arena->cmd_histo.data_,
|
1268
|
+
arena->cmd_histo.total_count_,
|
1254
1269
|
/* max_bits = */ 10,
|
1255
|
-
cmd_depth, cmd_bits,
|
1270
|
+
arena->cmd_depth, arena->cmd_bits,
|
1256
1271
|
storage_ix, storage);
|
1257
|
-
|
1258
|
-
|
1259
|
-
dist_histo.total_count_,
|
1272
|
+
BrotliBuildAndStoreHuffmanTreeFast(arena->tree, arena->dist_histo.data_,
|
1273
|
+
arena->dist_histo.total_count_,
|
1260
1274
|
/* max_bits = */
|
1261
1275
|
distance_alphabet_bits,
|
1262
|
-
dist_depth, dist_bits,
|
1276
|
+
arena->dist_depth, arena->dist_bits,
|
1263
1277
|
storage_ix, storage);
|
1264
|
-
if (BROTLI_IS_OOM(m)) return;
|
1265
1278
|
StoreDataWithHuffmanCodes(input, start_pos, mask, commands,
|
1266
|
-
n_commands, lit_depth, lit_bits,
|
1267
|
-
cmd_depth, cmd_bits,
|
1268
|
-
dist_depth, dist_bits,
|
1279
|
+
n_commands, arena->lit_depth, arena->lit_bits,
|
1280
|
+
arena->cmd_depth, arena->cmd_bits,
|
1281
|
+
arena->dist_depth, arena->dist_bits,
|
1269
1282
|
storage_ix, storage);
|
1270
1283
|
}
|
1271
1284
|
|
1285
|
+
BROTLI_FREE(m, arena);
|
1286
|
+
|
1272
1287
|
if (is_last) {
|
1273
1288
|
JumpToByteBoundary(storage_ix, storage);
|
1274
1289
|
}
|
@@ -1309,6 +1324,13 @@ void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
|
|
1309
1324
|
}
|
1310
1325
|
}
|
1311
1326
|
|
1327
|
+
#if defined(BROTLI_TEST)
|
1328
|
+
void GetBlockLengthPrefixCodeForTest(uint32_t len, size_t* code,
|
1329
|
+
uint32_t* n_extra, uint32_t* extra) {
|
1330
|
+
GetBlockLengthPrefixCode(len, code, n_extra, extra);
|
1331
|
+
}
|
1332
|
+
#endif
|
1333
|
+
|
1312
1334
|
#if defined(__cplusplus) || defined(c_plusplus)
|
1313
1335
|
} /* extern "C" */
|
1314
1336
|
#endif
|
@@ -16,13 +16,14 @@
|
|
16
16
|
#ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
17
17
|
#define BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
18
18
|
|
19
|
+
#include <brotli/types.h>
|
20
|
+
|
19
21
|
#include "../common/context.h"
|
20
22
|
#include "../common/platform.h"
|
21
|
-
#include
|
22
|
-
#include "
|
23
|
-
#include "
|
24
|
-
#include "
|
25
|
-
#include "./metablock.h"
|
23
|
+
#include "command.h"
|
24
|
+
#include "entropy_encode.h"
|
25
|
+
#include "memory.h"
|
26
|
+
#include "metablock.h"
|
26
27
|
|
27
28
|
#if defined(__cplusplus) || defined(c_plusplus)
|
28
29
|
extern "C" {
|
@@ -35,7 +36,7 @@ BROTLI_INTERNAL void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
|
|
35
36
|
HuffmanTree* tree, size_t* storage_ix, uint8_t* storage);
|
36
37
|
|
37
38
|
BROTLI_INTERNAL void BrotliBuildAndStoreHuffmanTreeFast(
|
38
|
-
|
39
|
+
HuffmanTree* tree, const uint32_t* histogram, const size_t histogram_total,
|
39
40
|
const size_t max_bits, uint8_t* depth, uint16_t* bits, size_t* storage_ix,
|
40
41
|
uint8_t* storage);
|
41
42
|
|
@@ -77,6 +78,10 @@ BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
|
|
77
78
|
size_t position, size_t mask, size_t len,
|
78
79
|
size_t* BROTLI_RESTRICT storage_ix, uint8_t* BROTLI_RESTRICT storage);
|
79
80
|
|
81
|
+
#if defined(BROTLI_TEST)
|
82
|
+
void GetBlockLengthPrefixCodeForTest(uint32_t, size_t*, uint32_t*, uint32_t*);
|
83
|
+
#endif
|
84
|
+
|
80
85
|
#if defined(__cplusplus) || defined(c_plusplus)
|
81
86
|
} /* extern "C" */
|
82
87
|
#endif
|
@@ -6,14 +6,15 @@
|
|
6
6
|
|
7
7
|
/* Functions for clustering similar histograms together. */
|
8
8
|
|
9
|
-
#include "
|
9
|
+
#include "cluster.h"
|
10
10
|
|
11
|
-
#include "../common/platform.h"
|
12
11
|
#include <brotli/types.h>
|
13
|
-
|
14
|
-
#include "
|
15
|
-
#include "
|
16
|
-
#include "
|
12
|
+
|
13
|
+
#include "../common/platform.h"
|
14
|
+
#include "bit_cost.h" /* BrotliPopulationCost */
|
15
|
+
#include "fast_log.h"
|
16
|
+
#include "histogram.h"
|
17
|
+
#include "memory.h"
|
17
18
|
|
18
19
|
#if defined(__cplusplus) || defined(c_plusplus)
|
19
20
|
extern "C" {
|
@@ -38,15 +39,15 @@ static BROTLI_INLINE double ClusterCostDiff(size_t size_a, size_t size_b) {
|
|
38
39
|
#define CODE(X) X
|
39
40
|
|
40
41
|
#define FN(X) X ## Literal
|
41
|
-
#include "
|
42
|
+
#include "cluster_inc.h" /* NOLINT(build/include) */
|
42
43
|
#undef FN
|
43
44
|
|
44
45
|
#define FN(X) X ## Command
|
45
|
-
#include "
|
46
|
+
#include "cluster_inc.h" /* NOLINT(build/include) */
|
46
47
|
#undef FN
|
47
48
|
|
48
49
|
#define FN(X) X ## Distance
|
49
|
-
#include "
|
50
|
+
#include "cluster_inc.h" /* NOLINT(build/include) */
|
50
51
|
#undef FN
|
51
52
|
|
52
53
|
#undef CODE
|
@@ -9,10 +9,11 @@
|
|
9
9
|
#ifndef BROTLI_ENC_CLUSTER_H_
|
10
10
|
#define BROTLI_ENC_CLUSTER_H_
|
11
11
|
|
12
|
-
#include "../common/platform.h"
|
13
12
|
#include <brotli/types.h>
|
14
|
-
|
15
|
-
#include "
|
13
|
+
|
14
|
+
#include "../common/platform.h"
|
15
|
+
#include "histogram.h"
|
16
|
+
#include "memory.h"
|
16
17
|
|
17
18
|
#if defined(__cplusplus) || defined(c_plusplus)
|
18
19
|
extern "C" {
|
@@ -28,15 +29,15 @@ typedef struct HistogramPair {
|
|
28
29
|
#define CODE(X) /* Declaration */;
|
29
30
|
|
30
31
|
#define FN(X) X ## Literal
|
31
|
-
#include "
|
32
|
+
#include "cluster_inc.h" /* NOLINT(build/include) */
|
32
33
|
#undef FN
|
33
34
|
|
34
35
|
#define FN(X) X ## Command
|
35
|
-
#include "
|
36
|
+
#include "cluster_inc.h" /* NOLINT(build/include) */
|
36
37
|
#undef FN
|
37
38
|
|
38
39
|
#define FN(X) X ## Distance
|
39
|
-
#include "
|
40
|
+
#include "cluster_inc.h" /* NOLINT(build/include) */
|
40
41
|
#undef FN
|
41
42
|
|
42
43
|
#undef CODE
|