brotli 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/main.yml +34 -0
- data/.github/workflows/publish.yml +34 -0
- data/Gemfile +6 -2
- data/Rakefile +18 -6
- data/bin/before_install.sh +9 -0
- data/brotli.gemspec +7 -13
- data/ext/brotli/brotli.c +209 -11
- data/ext/brotli/buffer.c +1 -7
- data/ext/brotli/buffer.h +1 -1
- data/ext/brotli/extconf.rb +45 -26
- data/lib/brotli/version.rb +1 -1
- data/smoke.sh +1 -1
- data/test/brotli_test.rb +104 -0
- data/test/brotli_writer_test.rb +36 -0
- data/test/test_helper.rb +8 -0
- data/vendor/brotli/c/common/constants.c +15 -0
- data/vendor/brotli/c/common/constants.h +149 -6
- data/vendor/brotli/c/{dec/context.h → common/context.c} +91 -186
- data/vendor/brotli/c/common/context.h +113 -0
- data/vendor/brotli/c/common/dictionary.bin +0 -0
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +11 -2
- data/vendor/brotli/c/common/dictionary.h +4 -4
- data/vendor/brotli/c/common/platform.c +22 -0
- data/vendor/brotli/c/common/platform.h +594 -0
- data/vendor/brotli/c/common/transform.c +291 -0
- data/vendor/brotli/c/common/transform.h +85 -0
- data/vendor/brotli/c/common/version.h +8 -1
- data/vendor/brotli/c/dec/bit_reader.c +29 -1
- data/vendor/brotli/c/dec/bit_reader.h +91 -100
- data/vendor/brotli/c/dec/decode.c +665 -437
- data/vendor/brotli/c/dec/huffman.c +65 -84
- data/vendor/brotli/c/dec/huffman.h +67 -14
- data/vendor/brotli/c/dec/prefix.h +1 -20
- data/vendor/brotli/c/dec/state.c +32 -45
- data/vendor/brotli/c/dec/state.h +173 -55
- data/vendor/brotli/c/enc/backward_references.c +27 -16
- data/vendor/brotli/c/enc/backward_references.h +7 -7
- data/vendor/brotli/c/enc/backward_references_hq.c +155 -116
- data/vendor/brotli/c/enc/backward_references_hq.h +22 -23
- data/vendor/brotli/c/enc/backward_references_inc.h +32 -22
- data/vendor/brotli/c/enc/bit_cost.c +1 -1
- data/vendor/brotli/c/enc/bit_cost.h +5 -5
- data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
- data/vendor/brotli/c/enc/block_splitter.c +5 -6
- data/vendor/brotli/c/enc/block_splitter.h +1 -1
- data/vendor/brotli/c/enc/block_splitter_inc.h +26 -17
- data/vendor/brotli/c/enc/brotli_bit_stream.c +107 -123
- data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
- data/vendor/brotli/c/enc/cluster.c +1 -1
- data/vendor/brotli/c/enc/cluster.h +1 -1
- data/vendor/brotli/c/enc/cluster_inc.h +6 -3
- data/vendor/brotli/c/enc/command.c +28 -0
- data/vendor/brotli/c/enc/command.h +52 -42
- data/vendor/brotli/c/enc/compress_fragment.c +21 -22
- data/vendor/brotli/c/enc/compress_fragment.h +1 -1
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +102 -69
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1827 -1101
- data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
- data/vendor/brotli/c/enc/encode.c +358 -195
- data/vendor/brotli/c/enc/encoder_dict.c +33 -0
- data/vendor/brotli/c/enc/encoder_dict.h +43 -0
- data/vendor/brotli/c/enc/entropy_encode.c +16 -14
- data/vendor/brotli/c/enc/entropy_encode.h +7 -7
- data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
- data/vendor/brotli/c/enc/fast_log.c +105 -0
- data/vendor/brotli/c/enc/fast_log.h +20 -99
- data/vendor/brotli/c/enc/find_match_length.h +5 -6
- data/vendor/brotli/c/enc/hash.h +145 -103
- data/vendor/brotli/c/enc/hash_composite_inc.h +125 -0
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +93 -53
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +54 -53
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +58 -54
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +95 -63
- data/vendor/brotli/c/enc/hash_rolling_inc.h +212 -0
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +46 -43
- data/vendor/brotli/c/enc/histogram.c +9 -6
- data/vendor/brotli/c/enc/histogram.h +6 -3
- data/vendor/brotli/c/enc/histogram_inc.h +1 -1
- data/vendor/brotli/c/enc/literal_cost.c +5 -5
- data/vendor/brotli/c/enc/literal_cost.h +2 -2
- data/vendor/brotli/c/enc/memory.c +5 -16
- data/vendor/brotli/c/enc/memory.h +52 -1
- data/vendor/brotli/c/enc/metablock.c +171 -36
- data/vendor/brotli/c/enc/metablock.h +13 -8
- data/vendor/brotli/c/enc/metablock_inc.h +2 -2
- data/vendor/brotli/c/enc/params.h +46 -0
- data/vendor/brotli/c/enc/prefix.h +3 -4
- data/vendor/brotli/c/enc/quality.h +29 -24
- data/vendor/brotli/c/enc/ringbuffer.h +19 -12
- data/vendor/brotli/c/enc/static_dict.c +49 -45
- data/vendor/brotli/c/enc/static_dict.h +4 -3
- data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
- data/vendor/brotli/c/enc/utf8_util.c +21 -21
- data/vendor/brotli/c/enc/utf8_util.h +1 -1
- data/vendor/brotli/c/enc/write_bits.h +35 -38
- data/vendor/brotli/c/include/brotli/decode.h +13 -8
- data/vendor/brotli/c/include/brotli/encode.h +54 -8
- data/vendor/brotli/c/include/brotli/port.h +225 -83
- data/vendor/brotli/c/include/brotli/types.h +0 -7
- metadata +28 -87
- data/.travis.yml +0 -30
- data/spec/brotli_spec.rb +0 -88
- data/spec/inflate_spec.rb +0 -75
- data/spec/spec_helper.rb +0 -4
- data/vendor/brotli/c/dec/port.h +0 -168
- data/vendor/brotli/c/dec/transform.h +0 -300
- data/vendor/brotli/c/enc/context.h +0 -184
- data/vendor/brotli/c/enc/port.h +0 -184
@@ -13,13 +13,14 @@
|
|
13
13
|
#include <string.h> /* memcpy, memset */
|
14
14
|
|
15
15
|
#include "../common/constants.h"
|
16
|
+
#include "../common/context.h"
|
17
|
+
#include "../common/platform.h"
|
16
18
|
#include <brotli/types.h>
|
17
|
-
#include "./context.h"
|
18
19
|
#include "./entropy_encode.h"
|
19
20
|
#include "./entropy_encode_static.h"
|
20
21
|
#include "./fast_log.h"
|
22
|
+
#include "./histogram.h"
|
21
23
|
#include "./memory.h"
|
22
|
-
#include "./port.h"
|
23
24
|
#include "./write_bits.h"
|
24
25
|
|
25
26
|
#if defined(__cplusplus) || defined(c_plusplus)
|
@@ -27,40 +28,24 @@ extern "C" {
|
|
27
28
|
#endif
|
28
29
|
|
29
30
|
#define MAX_HUFFMAN_TREE_SIZE (2 * BROTLI_NUM_COMMAND_SYMBOLS + 1)
|
30
|
-
/* The size of Huffman dictionary for distances assuming that
|
31
|
-
NDIRECT = 0. */
|
32
|
-
#define
|
33
|
-
|
34
|
-
/*
|
35
|
-
#define SIMPLE_DISTANCE_ALPHABET_BITS 6
|
36
|
-
|
37
|
-
/* Represents the range of values belonging to a prefix code:
|
38
|
-
[offset, offset + 2^nbits) */
|
39
|
-
typedef struct PrefixCodeRange {
|
40
|
-
uint32_t offset;
|
41
|
-
uint32_t nbits;
|
42
|
-
} PrefixCodeRange;
|
43
|
-
|
44
|
-
static const PrefixCodeRange
|
45
|
-
kBlockLengthPrefixCode[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = {
|
46
|
-
{ 1, 2}, { 5, 2}, { 9, 2}, {13, 2}, {17, 3}, { 25, 3}, { 33, 3},
|
47
|
-
{41, 3}, {49, 4}, {65, 4}, {81, 4}, {97, 4}, {113, 5}, {145, 5},
|
48
|
-
{177, 5}, { 209, 5}, { 241, 6}, { 305, 6}, { 369, 7}, { 497, 8},
|
49
|
-
{753, 9}, {1265, 10}, {2289, 11}, {4337, 12}, {8433, 13}, {16625, 24}
|
50
|
-
};
|
31
|
+
/* The maximum size of Huffman dictionary for distances assuming that
|
32
|
+
NPOSTFIX = 0 and NDIRECT = 0. */
|
33
|
+
#define MAX_SIMPLE_DISTANCE_ALPHABET_SIZE \
|
34
|
+
BROTLI_DISTANCE_ALPHABET_SIZE(0, 0, BROTLI_LARGE_MAX_DISTANCE_BITS)
|
35
|
+
/* MAX_SIMPLE_DISTANCE_ALPHABET_SIZE == 140 */
|
51
36
|
|
52
37
|
static BROTLI_INLINE uint32_t BlockLengthPrefixCode(uint32_t len) {
|
53
38
|
uint32_t code = (len >= 177) ? (len >= 753 ? 20 : 14) : (len >= 41 ? 7 : 0);
|
54
39
|
while (code < (BROTLI_NUM_BLOCK_LEN_SYMBOLS - 1) &&
|
55
|
-
len >=
|
40
|
+
len >= _kBrotliPrefixCodeRanges[code + 1].offset) ++code;
|
56
41
|
return code;
|
57
42
|
}
|
58
43
|
|
59
44
|
static BROTLI_INLINE void GetBlockLengthPrefixCode(uint32_t len, size_t* code,
|
60
45
|
uint32_t* n_extra, uint32_t* extra) {
|
61
46
|
*code = BlockLengthPrefixCode(len);
|
62
|
-
*n_extra =
|
63
|
-
*extra = len -
|
47
|
+
*n_extra = _kBrotliPrefixCodeRanges[*code].nbits;
|
48
|
+
*extra = len - _kBrotliPrefixCodeRanges[*code].offset;
|
64
49
|
}
|
65
50
|
|
66
51
|
typedef struct BlockTypeCodeCalculator {
|
@@ -89,9 +74,9 @@ static void BrotliEncodeMlen(size_t length, uint64_t* bits,
|
|
89
74
|
size_t* numbits, uint64_t* nibblesbits) {
|
90
75
|
size_t lg = (length == 1) ? 1 : Log2FloorNonZero((uint32_t)(length - 1)) + 1;
|
91
76
|
size_t mnibbles = (lg < 16 ? 16 : (lg + 3)) / 4;
|
92
|
-
|
93
|
-
|
94
|
-
|
77
|
+
BROTLI_DCHECK(length > 0);
|
78
|
+
BROTLI_DCHECK(length <= (1 << 24));
|
79
|
+
BROTLI_DCHECK(lg <= 24);
|
95
80
|
*nibblesbits = mnibbles - 4;
|
96
81
|
*numbits = mnibbles * 4;
|
97
82
|
*bits = length - 1;
|
@@ -258,7 +243,7 @@ static void StoreSimpleHuffmanTree(const uint8_t* depths,
|
|
258
243
|
size_t symbols[4],
|
259
244
|
size_t num_symbols,
|
260
245
|
size_t max_bits,
|
261
|
-
size_t
|
246
|
+
size_t* storage_ix, uint8_t* storage) {
|
262
247
|
/* value of 1 indicates a simple Huffman code */
|
263
248
|
BrotliWriteBits(2, 1, storage_ix, storage);
|
264
249
|
BrotliWriteBits(2, num_symbols - 1, storage_ix, storage); /* NSYM - 1 */
|
@@ -297,7 +282,7 @@ static void StoreSimpleHuffmanTree(const uint8_t* depths,
|
|
297
282
|
depths = symbol depths */
|
298
283
|
void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
|
299
284
|
HuffmanTree* tree,
|
300
|
-
size_t
|
285
|
+
size_t* storage_ix, uint8_t* storage) {
|
301
286
|
/* Write the Huffman tree into the brotli-representation.
|
302
287
|
The command alphabet is the largest, so this allocation will fit all
|
303
288
|
alphabets. */
|
@@ -311,7 +296,7 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
|
|
311
296
|
int num_codes = 0;
|
312
297
|
size_t code = 0;
|
313
298
|
|
314
|
-
|
299
|
+
BROTLI_DCHECK(num <= BROTLI_NUM_COMMAND_SYMBOLS);
|
315
300
|
|
316
301
|
BrotliWriteHuffmanTree(depths, num, &huffman_tree_size, huffman_tree,
|
317
302
|
huffman_tree_extra_bits);
|
@@ -360,8 +345,9 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
|
|
360
345
|
|
361
346
|
/* Builds a Huffman tree from histogram[0:length] into depth[0:length] and
|
362
347
|
bits[0:length] and stores the encoded tree to the bit stream. */
|
363
|
-
static void BuildAndStoreHuffmanTree(const uint32_t
|
364
|
-
const size_t
|
348
|
+
static void BuildAndStoreHuffmanTree(const uint32_t* histogram,
|
349
|
+
const size_t histogram_length,
|
350
|
+
const size_t alphabet_size,
|
365
351
|
HuffmanTree* tree,
|
366
352
|
uint8_t* depth,
|
367
353
|
uint16_t* bits,
|
@@ -371,7 +357,7 @@ static void BuildAndStoreHuffmanTree(const uint32_t *histogram,
|
|
371
357
|
size_t s4[4] = { 0 };
|
372
358
|
size_t i;
|
373
359
|
size_t max_bits = 0;
|
374
|
-
for (i = 0; i <
|
360
|
+
for (i = 0; i < histogram_length; i++) {
|
375
361
|
if (histogram[i]) {
|
376
362
|
if (count < 4) {
|
377
363
|
s4[count] = i;
|
@@ -383,7 +369,7 @@ static void BuildAndStoreHuffmanTree(const uint32_t *histogram,
|
|
383
369
|
}
|
384
370
|
|
385
371
|
{
|
386
|
-
size_t max_bits_counter =
|
372
|
+
size_t max_bits_counter = alphabet_size - 1;
|
387
373
|
while (max_bits_counter) {
|
388
374
|
max_bits_counter >>= 1;
|
389
375
|
++max_bits;
|
@@ -398,14 +384,14 @@ static void BuildAndStoreHuffmanTree(const uint32_t *histogram,
|
|
398
384
|
return;
|
399
385
|
}
|
400
386
|
|
401
|
-
memset(depth, 0,
|
402
|
-
BrotliCreateHuffmanTree(histogram,
|
403
|
-
BrotliConvertBitDepthsToSymbols(depth,
|
387
|
+
memset(depth, 0, histogram_length * sizeof(depth[0]));
|
388
|
+
BrotliCreateHuffmanTree(histogram, histogram_length, 15, tree, depth);
|
389
|
+
BrotliConvertBitDepthsToSymbols(depth, histogram_length, bits);
|
404
390
|
|
405
391
|
if (count <= 4) {
|
406
392
|
StoreSimpleHuffmanTree(depth, s4, count, max_bits, storage_ix, storage);
|
407
393
|
} else {
|
408
|
-
BrotliStoreHuffmanTree(depth,
|
394
|
+
BrotliStoreHuffmanTree(depth, histogram_length, tree, storage_ix, storage);
|
409
395
|
}
|
410
396
|
}
|
411
397
|
|
@@ -449,7 +435,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
|
|
449
435
|
const size_t max_tree_size = 2 * length + 1;
|
450
436
|
HuffmanTree* tree = BROTLI_ALLOC(m, HuffmanTree, max_tree_size);
|
451
437
|
uint32_t count_limit;
|
452
|
-
if (BROTLI_IS_OOM(m)) return;
|
438
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
|
453
439
|
for (count_limit = 1; ; count_limit *= 2) {
|
454
440
|
HuffmanTree* node = tree;
|
455
441
|
size_t l;
|
@@ -619,7 +605,7 @@ static void MoveToFrontTransform(const uint32_t* BROTLI_RESTRICT v_in,
|
|
619
605
|
for (i = 1; i < v_size; ++i) {
|
620
606
|
if (v_in[i] > max_value) max_value = v_in[i];
|
621
607
|
}
|
622
|
-
|
608
|
+
BROTLI_DCHECK(max_value < 256u);
|
623
609
|
for (i = 0; i <= max_value; ++i) {
|
624
610
|
mtf[i] = (uint8_t)i;
|
625
611
|
}
|
@@ -627,7 +613,7 @@ static void MoveToFrontTransform(const uint32_t* BROTLI_RESTRICT v_in,
|
|
627
613
|
size_t mtf_size = max_value + 1;
|
628
614
|
for (i = 0; i < v_size; ++i) {
|
629
615
|
size_t index = IndexOf(mtf, mtf_size, (uint8_t)v_in[i]);
|
630
|
-
|
616
|
+
BROTLI_DCHECK(index < mtf_size);
|
631
617
|
v_out[i] = (uint32_t)index;
|
632
618
|
MoveToFront(mtf, index);
|
633
619
|
}
|
@@ -659,7 +645,7 @@ static void RunLengthCodeZeros(const size_t in_size,
|
|
659
645
|
*max_run_length_prefix = max_prefix;
|
660
646
|
*out_size = 0;
|
661
647
|
for (i = 0; i < in_size;) {
|
662
|
-
|
648
|
+
BROTLI_DCHECK(*out_size <= i);
|
663
649
|
if (v[i] != 0) {
|
664
650
|
v[*out_size] = v[i] + *max_run_length_prefix;
|
665
651
|
++i;
|
@@ -713,7 +699,7 @@ static void EncodeContextMap(MemoryManager* m,
|
|
713
699
|
}
|
714
700
|
|
715
701
|
rle_symbols = BROTLI_ALLOC(m, uint32_t, context_map_size);
|
716
|
-
if (BROTLI_IS_OOM(m)) return;
|
702
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(rle_symbols)) return;
|
717
703
|
MoveToFrontTransform(context_map, context_map_size, rle_symbols);
|
718
704
|
RunLengthCodeZeros(context_map_size, rle_symbols,
|
719
705
|
&num_rle_symbols, &max_run_length_prefix);
|
@@ -729,6 +715,7 @@ static void EncodeContextMap(MemoryManager* m,
|
|
729
715
|
}
|
730
716
|
}
|
731
717
|
BuildAndStoreHuffmanTree(histogram, num_clusters + max_run_length_prefix,
|
718
|
+
num_clusters + max_run_length_prefix,
|
732
719
|
tree, depths, bits, storage_ix, storage);
|
733
720
|
for (i = 0; i < num_rle_symbols; ++i) {
|
734
721
|
const uint32_t rle_symbol = rle_symbols[i] & kSymbolMask;
|
@@ -788,10 +775,11 @@ static void BuildAndStoreBlockSplitCode(const uint8_t* types,
|
|
788
775
|
}
|
789
776
|
StoreVarLenUint8(num_types - 1, storage_ix, storage);
|
790
777
|
if (num_types > 1) { /* TODO: else? could StoreBlockSwitch occur? */
|
791
|
-
BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, tree,
|
778
|
+
BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, num_types + 2, tree,
|
792
779
|
&code->type_depths[0], &code->type_bits[0],
|
793
780
|
storage_ix, storage);
|
794
781
|
BuildAndStoreHuffmanTree(&length_histo[0], BROTLI_NUM_BLOCK_LEN_SYMBOLS,
|
782
|
+
BROTLI_NUM_BLOCK_LEN_SYMBOLS,
|
795
783
|
tree, &code->length_depths[0],
|
796
784
|
&code->length_bits[0], storage_ix, storage);
|
797
785
|
StoreBlockSwitch(code, lengths[0], types[0], 1, storage_ix, storage);
|
@@ -822,8 +810,8 @@ static void StoreTrivialContextMap(size_t num_types,
|
|
822
810
|
for (i = context_bits; i < alphabet_size; ++i) {
|
823
811
|
histogram[i] = 1;
|
824
812
|
}
|
825
|
-
BuildAndStoreHuffmanTree(histogram, alphabet_size,
|
826
|
-
depths, bits, storage_ix, storage);
|
813
|
+
BuildAndStoreHuffmanTree(histogram, alphabet_size, alphabet_size,
|
814
|
+
tree, depths, bits, storage_ix, storage);
|
827
815
|
for (i = 0; i < num_types; ++i) {
|
828
816
|
size_t code = (i == 0 ? 0 : i + context_bits - 1);
|
829
817
|
BrotliWriteBits(depths[code], bits[code], storage_ix, storage);
|
@@ -838,7 +826,7 @@ static void StoreTrivialContextMap(size_t num_types,
|
|
838
826
|
|
839
827
|
/* Manages the encoding of one block category (literal, command or distance). */
|
840
828
|
typedef struct BlockEncoder {
|
841
|
-
size_t
|
829
|
+
size_t histogram_length_;
|
842
830
|
size_t num_block_types_;
|
843
831
|
const uint8_t* block_types_; /* Not owned. */
|
844
832
|
const uint32_t* block_lengths_; /* Not owned. */
|
@@ -851,10 +839,10 @@ typedef struct BlockEncoder {
|
|
851
839
|
uint16_t* bits_;
|
852
840
|
} BlockEncoder;
|
853
841
|
|
854
|
-
static void InitBlockEncoder(BlockEncoder* self, size_t
|
842
|
+
static void InitBlockEncoder(BlockEncoder* self, size_t histogram_length,
|
855
843
|
size_t num_block_types, const uint8_t* block_types,
|
856
844
|
const uint32_t* block_lengths, const size_t num_blocks) {
|
857
|
-
self->
|
845
|
+
self->histogram_length_ = histogram_length;
|
858
846
|
self->num_block_types_ = num_block_types;
|
859
847
|
self->block_types_ = block_types;
|
860
848
|
self->block_lengths_ = block_lengths;
|
@@ -890,7 +878,7 @@ static void StoreSymbol(BlockEncoder* self, size_t symbol, size_t* storage_ix,
|
|
890
878
|
uint32_t block_len = self->block_lengths_[block_ix];
|
891
879
|
uint8_t block_type = self->block_types_[block_ix];
|
892
880
|
self->block_len_ = block_len;
|
893
|
-
self->entropy_ix_ = block_type * self->
|
881
|
+
self->entropy_ix_ = block_type * self->histogram_length_;
|
894
882
|
StoreBlockSwitch(&self->block_split_code_, block_len, block_type, 0,
|
895
883
|
storage_ix, storage);
|
896
884
|
}
|
@@ -919,7 +907,7 @@ static void StoreSymbolWithContext(BlockEncoder* self, size_t symbol,
|
|
919
907
|
--self->block_len_;
|
920
908
|
{
|
921
909
|
size_t histo_ix = context_map[self->entropy_ix_ + context];
|
922
|
-
size_t ix = histo_ix * self->
|
910
|
+
size_t ix = histo_ix * self->histogram_length_ + symbol;
|
923
911
|
BrotliWriteBits(self->depths_[ix], self->bits_[ix], storage_ix, storage);
|
924
912
|
}
|
925
913
|
}
|
@@ -945,42 +933,36 @@ static void JumpToByteBoundary(size_t* storage_ix, uint8_t* storage) {
|
|
945
933
|
}
|
946
934
|
|
947
935
|
void BrotliStoreMetaBlock(MemoryManager* m,
|
948
|
-
|
949
|
-
|
950
|
-
|
951
|
-
|
952
|
-
|
953
|
-
|
954
|
-
BROTLI_BOOL is_last,
|
955
|
-
uint32_t num_direct_distance_codes,
|
956
|
-
uint32_t distance_postfix_bits,
|
957
|
-
ContextType literal_context_mode,
|
958
|
-
const Command *commands,
|
959
|
-
size_t n_commands,
|
960
|
-
const MetaBlockSplit* mb,
|
961
|
-
size_t *storage_ix,
|
962
|
-
uint8_t *storage) {
|
936
|
+
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
|
937
|
+
uint8_t prev_byte, uint8_t prev_byte2, BROTLI_BOOL is_last,
|
938
|
+
const BrotliEncoderParams* params, ContextType literal_context_mode,
|
939
|
+
const Command* commands, size_t n_commands, const MetaBlockSplit* mb,
|
940
|
+
size_t* storage_ix, uint8_t* storage) {
|
941
|
+
|
963
942
|
size_t pos = start_pos;
|
964
943
|
size_t i;
|
965
|
-
|
966
|
-
|
967
|
-
(48u << distance_postfix_bits);
|
944
|
+
uint32_t num_distance_symbols = params->dist.alphabet_size_max;
|
945
|
+
uint32_t num_effective_distance_symbols = params->dist.alphabet_size_limit;
|
968
946
|
HuffmanTree* tree;
|
947
|
+
ContextLut literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
|
969
948
|
BlockEncoder literal_enc;
|
970
949
|
BlockEncoder command_enc;
|
971
950
|
BlockEncoder distance_enc;
|
951
|
+
const BrotliDistanceParams* dist = ¶ms->dist;
|
952
|
+
BROTLI_DCHECK(
|
953
|
+
num_effective_distance_symbols <= BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS);
|
972
954
|
|
973
955
|
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
|
974
956
|
|
975
957
|
tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
|
976
|
-
if (BROTLI_IS_OOM(m)) return;
|
977
|
-
InitBlockEncoder(&literal_enc,
|
978
|
-
mb->literal_split.
|
979
|
-
mb->literal_split.num_blocks);
|
958
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
|
959
|
+
InitBlockEncoder(&literal_enc, BROTLI_NUM_LITERAL_SYMBOLS,
|
960
|
+
mb->literal_split.num_types, mb->literal_split.types,
|
961
|
+
mb->literal_split.lengths, mb->literal_split.num_blocks);
|
980
962
|
InitBlockEncoder(&command_enc, BROTLI_NUM_COMMAND_SYMBOLS,
|
981
963
|
mb->command_split.num_types, mb->command_split.types,
|
982
964
|
mb->command_split.lengths, mb->command_split.num_blocks);
|
983
|
-
InitBlockEncoder(&distance_enc,
|
965
|
+
InitBlockEncoder(&distance_enc, num_effective_distance_symbols,
|
984
966
|
mb->distance_split.num_types, mb->distance_split.types,
|
985
967
|
mb->distance_split.lengths, mb->distance_split.num_blocks);
|
986
968
|
|
@@ -989,9 +971,10 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
989
971
|
BuildAndStoreBlockSwitchEntropyCodes(
|
990
972
|
&distance_enc, tree, storage_ix, storage);
|
991
973
|
|
992
|
-
BrotliWriteBits(2, distance_postfix_bits, storage_ix, storage);
|
993
|
-
BrotliWriteBits(
|
994
|
-
|
974
|
+
BrotliWriteBits(2, dist->distance_postfix_bits, storage_ix, storage);
|
975
|
+
BrotliWriteBits(
|
976
|
+
4, dist->num_direct_distance_codes >> dist->distance_postfix_bits,
|
977
|
+
storage_ix, storage);
|
995
978
|
for (i = 0; i < mb->literal_split.num_types; ++i) {
|
996
979
|
BrotliWriteBits(2, literal_context_mode, storage_ix, storage);
|
997
980
|
}
|
@@ -1017,13 +1000,16 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
1017
1000
|
}
|
1018
1001
|
|
1019
1002
|
BuildAndStoreEntropyCodesLiteral(m, &literal_enc, mb->literal_histograms,
|
1020
|
-
mb->literal_histograms_size,
|
1003
|
+
mb->literal_histograms_size, BROTLI_NUM_LITERAL_SYMBOLS, tree,
|
1004
|
+
storage_ix, storage);
|
1021
1005
|
if (BROTLI_IS_OOM(m)) return;
|
1022
1006
|
BuildAndStoreEntropyCodesCommand(m, &command_enc, mb->command_histograms,
|
1023
|
-
mb->command_histograms_size,
|
1007
|
+
mb->command_histograms_size, BROTLI_NUM_COMMAND_SYMBOLS, tree,
|
1008
|
+
storage_ix, storage);
|
1024
1009
|
if (BROTLI_IS_OOM(m)) return;
|
1025
1010
|
BuildAndStoreEntropyCodesDistance(m, &distance_enc, mb->distance_histograms,
|
1026
|
-
mb->distance_histograms_size,
|
1011
|
+
mb->distance_histograms_size, num_distance_symbols, tree,
|
1012
|
+
storage_ix, storage);
|
1027
1013
|
if (BROTLI_IS_OOM(m)) return;
|
1028
1014
|
BROTLI_FREE(m, tree);
|
1029
1015
|
|
@@ -1041,7 +1027,8 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
1041
1027
|
} else {
|
1042
1028
|
size_t j;
|
1043
1029
|
for (j = cmd.insert_len_; j != 0; --j) {
|
1044
|
-
size_t context =
|
1030
|
+
size_t context =
|
1031
|
+
BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut);
|
1045
1032
|
uint8_t literal = input[pos & mask];
|
1046
1033
|
StoreSymbolWithContext(&literal_enc, literal, context,
|
1047
1034
|
mb->literal_context_map, storage_ix, storage,
|
@@ -1056,9 +1043,9 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
1056
1043
|
prev_byte2 = input[(pos - 2) & mask];
|
1057
1044
|
prev_byte = input[(pos - 1) & mask];
|
1058
1045
|
if (cmd.cmd_prefix_ >= 128) {
|
1059
|
-
size_t dist_code = cmd.dist_prefix_;
|
1060
|
-
uint32_t distnumextra = cmd.
|
1061
|
-
uint64_t distextra = cmd.dist_extra_
|
1046
|
+
size_t dist_code = cmd.dist_prefix_ & 0x3FF;
|
1047
|
+
uint32_t distnumextra = cmd.dist_prefix_ >> 10;
|
1048
|
+
uint64_t distextra = cmd.dist_extra_;
|
1062
1049
|
if (mb->distance_context_map_size == 0) {
|
1063
1050
|
StoreSymbol(&distance_enc, dist_code, storage_ix, storage);
|
1064
1051
|
} else {
|
@@ -1082,7 +1069,7 @@ void BrotliStoreMetaBlock(MemoryManager* m,
|
|
1082
1069
|
static void BuildHistograms(const uint8_t* input,
|
1083
1070
|
size_t start_pos,
|
1084
1071
|
size_t mask,
|
1085
|
-
const Command
|
1072
|
+
const Command* commands,
|
1086
1073
|
size_t n_commands,
|
1087
1074
|
HistogramLiteral* lit_histo,
|
1088
1075
|
HistogramCommand* cmd_histo,
|
@@ -1099,7 +1086,7 @@ static void BuildHistograms(const uint8_t* input,
|
|
1099
1086
|
}
|
1100
1087
|
pos += CommandCopyLen(&cmd);
|
1101
1088
|
if (CommandCopyLen(&cmd) && cmd.cmd_prefix_ >= 128) {
|
1102
|
-
HistogramAddDistance(dist_histo, cmd.dist_prefix_);
|
1089
|
+
HistogramAddDistance(dist_histo, cmd.dist_prefix_ & 0x3FF);
|
1103
1090
|
}
|
1104
1091
|
}
|
1105
1092
|
}
|
@@ -1107,7 +1094,7 @@ static void BuildHistograms(const uint8_t* input,
|
|
1107
1094
|
static void StoreDataWithHuffmanCodes(const uint8_t* input,
|
1108
1095
|
size_t start_pos,
|
1109
1096
|
size_t mask,
|
1110
|
-
const Command
|
1097
|
+
const Command* commands,
|
1111
1098
|
size_t n_commands,
|
1112
1099
|
const uint8_t* lit_depth,
|
1113
1100
|
const uint16_t* lit_bits,
|
@@ -1134,9 +1121,9 @@ static void StoreDataWithHuffmanCodes(const uint8_t* input,
|
|
1134
1121
|
}
|
1135
1122
|
pos += CommandCopyLen(&cmd);
|
1136
1123
|
if (CommandCopyLen(&cmd) && cmd.cmd_prefix_ >= 128) {
|
1137
|
-
const size_t dist_code = cmd.dist_prefix_;
|
1138
|
-
const uint32_t distnumextra = cmd.
|
1139
|
-
const uint32_t distextra = cmd.dist_extra_
|
1124
|
+
const size_t dist_code = cmd.dist_prefix_ & 0x3FF;
|
1125
|
+
const uint32_t distnumextra = cmd.dist_prefix_ >> 10;
|
1126
|
+
const uint32_t distextra = cmd.dist_extra_;
|
1140
1127
|
BrotliWriteBits(dist_depth[dist_code], dist_bits[dist_code],
|
1141
1128
|
storage_ix, storage);
|
1142
1129
|
BrotliWriteBits(distnumextra, distextra, storage_ix, storage);
|
@@ -1145,15 +1132,10 @@ static void StoreDataWithHuffmanCodes(const uint8_t* input,
|
|
1145
1132
|
}
|
1146
1133
|
|
1147
1134
|
void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1151
|
-
|
1152
|
-
BROTLI_BOOL is_last,
|
1153
|
-
const Command *commands,
|
1154
|
-
size_t n_commands,
|
1155
|
-
size_t *storage_ix,
|
1156
|
-
uint8_t *storage) {
|
1135
|
+
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
|
1136
|
+
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
|
1137
|
+
const Command* commands, size_t n_commands,
|
1138
|
+
size_t* storage_ix, uint8_t* storage) {
|
1157
1139
|
HistogramLiteral lit_histo;
|
1158
1140
|
HistogramCommand cmd_histo;
|
1159
1141
|
HistogramDistance dist_histo;
|
@@ -1161,9 +1143,10 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
|
1161
1143
|
uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
|
1162
1144
|
uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
|
1163
1145
|
uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
|
1164
|
-
uint8_t dist_depth[
|
1165
|
-
uint16_t dist_bits[
|
1146
|
+
uint8_t dist_depth[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1147
|
+
uint16_t dist_bits[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1166
1148
|
HuffmanTree* tree;
|
1149
|
+
uint32_t num_distance_symbols = params->dist.alphabet_size_max;
|
1167
1150
|
|
1168
1151
|
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
|
1169
1152
|
|
@@ -1177,15 +1160,17 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
|
1177
1160
|
BrotliWriteBits(13, 0, storage_ix, storage);
|
1178
1161
|
|
1179
1162
|
tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
|
1180
|
-
if (BROTLI_IS_OOM(m)) return;
|
1181
|
-
BuildAndStoreHuffmanTree(lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS,
|
1163
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tree)) return;
|
1164
|
+
BuildAndStoreHuffmanTree(lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS,
|
1165
|
+
BROTLI_NUM_LITERAL_SYMBOLS, tree,
|
1182
1166
|
lit_depth, lit_bits,
|
1183
1167
|
storage_ix, storage);
|
1184
|
-
BuildAndStoreHuffmanTree(cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS,
|
1168
|
+
BuildAndStoreHuffmanTree(cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS,
|
1169
|
+
BROTLI_NUM_COMMAND_SYMBOLS, tree,
|
1185
1170
|
cmd_depth, cmd_bits,
|
1186
1171
|
storage_ix, storage);
|
1187
|
-
BuildAndStoreHuffmanTree(dist_histo.data_,
|
1188
|
-
tree,
|
1172
|
+
BuildAndStoreHuffmanTree(dist_histo.data_, MAX_SIMPLE_DISTANCE_ALPHABET_SIZE,
|
1173
|
+
num_distance_symbols, tree,
|
1189
1174
|
dist_depth, dist_bits,
|
1190
1175
|
storage_ix, storage);
|
1191
1176
|
BROTLI_FREE(m, tree);
|
@@ -1200,15 +1185,14 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
|
1200
1185
|
}
|
1201
1186
|
|
1202
1187
|
void BrotliStoreMetaBlockFast(MemoryManager* m,
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1207
|
-
|
1208
|
-
|
1209
|
-
|
1210
|
-
|
1211
|
-
uint8_t *storage) {
|
1188
|
+
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
|
1189
|
+
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
|
1190
|
+
const Command* commands, size_t n_commands,
|
1191
|
+
size_t* storage_ix, uint8_t* storage) {
|
1192
|
+
uint32_t num_distance_symbols = params->dist.alphabet_size_max;
|
1193
|
+
uint32_t distance_alphabet_bits =
|
1194
|
+
Log2FloorNonZero(num_distance_symbols - 1) + 1;
|
1195
|
+
|
1212
1196
|
StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
|
1213
1197
|
|
1214
1198
|
BrotliWriteBits(13, 0, storage_ix, storage);
|
@@ -1252,8 +1236,8 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
|
|
1252
1236
|
uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
|
1253
1237
|
uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
|
1254
1238
|
uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
|
1255
|
-
uint8_t dist_depth[
|
1256
|
-
uint16_t dist_bits[
|
1239
|
+
uint8_t dist_depth[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1240
|
+
uint16_t dist_bits[MAX_SIMPLE_DISTANCE_ALPHABET_SIZE];
|
1257
1241
|
HistogramClearLiteral(&lit_histo);
|
1258
1242
|
HistogramClearCommand(&cmd_histo);
|
1259
1243
|
HistogramClearDistance(&dist_histo);
|
@@ -1274,7 +1258,7 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
|
|
1274
1258
|
BrotliBuildAndStoreHuffmanTreeFast(m, dist_histo.data_,
|
1275
1259
|
dist_histo.total_count_,
|
1276
1260
|
/* max_bits = */
|
1277
|
-
|
1261
|
+
distance_alphabet_bits,
|
1278
1262
|
dist_depth, dist_bits,
|
1279
1263
|
storage_ix, storage);
|
1280
1264
|
if (BROTLI_IS_OOM(m)) return;
|
@@ -1293,11 +1277,11 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
|
|
1293
1277
|
/* This is for storing uncompressed blocks (simple raw storage of
|
1294
1278
|
bytes-as-bytes). */
|
1295
1279
|
void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
|
1296
|
-
const uint8_t
|
1280
|
+
const uint8_t* BROTLI_RESTRICT input,
|
1297
1281
|
size_t position, size_t mask,
|
1298
1282
|
size_t len,
|
1299
|
-
size_t
|
1300
|
-
uint8_t
|
1283
|
+
size_t* BROTLI_RESTRICT storage_ix,
|
1284
|
+
uint8_t* BROTLI_RESTRICT storage) {
|
1301
1285
|
size_t masked_pos = position & mask;
|
1302
1286
|
BrotliStoreUncompressedMetaBlockHeader(len, storage_ix, storage);
|
1303
1287
|
JumpToByteBoundary(storage_ix, storage);
|