brotli 0.2.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +37 -0
- data/.github/workflows/publish.yml +24 -0
- data/.gitmodules +1 -1
- data/Gemfile +6 -3
- data/README.md +2 -2
- data/Rakefile +16 -9
- data/brotli.gemspec +7 -13
- data/ext/brotli/brotli.c +210 -31
- data/ext/brotli/buffer.c +1 -7
- data/ext/brotli/buffer.h +1 -1
- data/ext/brotli/extconf.rb +25 -17
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +107 -0
- data/test/brotli_writer_test.rb +36 -0
- data/test/test_helper.rb +8 -0
- data/vendor/brotli/c/common/constants.c +15 -0
- data/vendor/brotli/c/common/constants.h +137 -0
- data/vendor/brotli/c/common/context.c +156 -0
- data/vendor/brotli/c/common/context.h +4 -152
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +14 -3
- data/vendor/brotli/c/common/platform.c +23 -0
- data/vendor/brotli/c/common/platform.h +95 -122
- data/vendor/brotli/c/common/shared_dictionary.c +521 -0
- data/vendor/brotli/c/common/shared_dictionary_internal.h +75 -0
- data/vendor/brotli/c/common/transform.c +60 -4
- data/vendor/brotli/c/common/transform.h +5 -0
- data/vendor/brotli/c/common/version.h +31 -6
- data/vendor/brotli/c/dec/bit_reader.c +34 -4
- data/vendor/brotli/c/dec/bit_reader.h +221 -107
- data/vendor/brotli/c/dec/decode.c +772 -403
- data/vendor/brotli/c/dec/huffman.c +7 -4
- data/vendor/brotli/c/dec/huffman.h +8 -13
- data/vendor/brotli/c/dec/prefix.h +1 -18
- data/vendor/brotli/c/dec/state.c +40 -21
- data/vendor/brotli/c/dec/state.h +201 -59
- data/vendor/brotli/c/enc/backward_references.c +88 -25
- data/vendor/brotli/c/enc/backward_references.h +10 -8
- data/vendor/brotli/c/enc/backward_references_hq.c +194 -80
- data/vendor/brotli/c/enc/backward_references_hq.h +17 -13
- data/vendor/brotli/c/enc/backward_references_inc.h +52 -16
- data/vendor/brotli/c/enc/bit_cost.c +8 -7
- data/vendor/brotli/c/enc/bit_cost.h +5 -4
- data/vendor/brotli/c/enc/block_splitter.c +40 -17
- data/vendor/brotli/c/enc/block_splitter.h +5 -4
- data/vendor/brotli/c/enc/block_splitter_inc.h +99 -49
- data/vendor/brotli/c/enc/brotli_bit_stream.c +142 -137
- data/vendor/brotli/c/enc/brotli_bit_stream.h +11 -6
- data/vendor/brotli/c/enc/cluster.c +10 -9
- data/vendor/brotli/c/enc/cluster.h +7 -6
- data/vendor/brotli/c/enc/cluster_inc.h +30 -22
- data/vendor/brotli/c/enc/command.c +28 -0
- data/vendor/brotli/c/enc/command.h +17 -16
- data/vendor/brotli/c/enc/compound_dictionary.c +207 -0
- data/vendor/brotli/c/enc/compound_dictionary.h +74 -0
- data/vendor/brotli/c/enc/compress_fragment.c +93 -83
- data/vendor/brotli/c/enc/compress_fragment.h +32 -7
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +100 -88
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +21 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +1829 -1101
- data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
- data/vendor/brotli/c/enc/encode.c +550 -416
- data/vendor/brotli/c/enc/encoder_dict.c +613 -5
- data/vendor/brotli/c/enc/encoder_dict.h +120 -4
- data/vendor/brotli/c/enc/entropy_encode.c +5 -2
- data/vendor/brotli/c/enc/entropy_encode.h +4 -3
- data/vendor/brotli/c/enc/entropy_encode_static.h +5 -2
- data/vendor/brotli/c/enc/fast_log.c +105 -0
- data/vendor/brotli/c/enc/fast_log.h +21 -101
- data/vendor/brotli/c/enc/find_match_length.h +17 -25
- data/vendor/brotli/c/enc/hash.h +350 -120
- data/vendor/brotli/c/enc/hash_composite_inc.h +71 -67
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +92 -51
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +79 -84
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +53 -54
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +93 -62
- data/vendor/brotli/c/enc/hash_rolling_inc.h +25 -29
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +42 -40
- data/vendor/brotli/c/enc/histogram.c +4 -4
- data/vendor/brotli/c/enc/histogram.h +7 -6
- data/vendor/brotli/c/enc/literal_cost.c +20 -15
- data/vendor/brotli/c/enc/literal_cost.h +4 -2
- data/vendor/brotli/c/enc/memory.c +29 -5
- data/vendor/brotli/c/enc/memory.h +43 -14
- data/vendor/brotli/c/enc/metablock.c +95 -85
- data/vendor/brotli/c/enc/metablock.h +9 -8
- data/vendor/brotli/c/enc/metablock_inc.h +9 -7
- data/vendor/brotli/c/enc/params.h +7 -4
- data/vendor/brotli/c/enc/prefix.h +3 -2
- data/vendor/brotli/c/enc/quality.h +40 -3
- data/vendor/brotli/c/enc/ringbuffer.h +8 -4
- data/vendor/brotli/c/enc/state.h +104 -0
- data/vendor/brotli/c/enc/static_dict.c +60 -4
- data/vendor/brotli/c/enc/static_dict.h +3 -2
- data/vendor/brotli/c/enc/static_dict_lut.h +2 -0
- data/vendor/brotli/c/enc/utf8_util.c +2 -2
- data/vendor/brotli/c/enc/utf8_util.h +2 -1
- data/vendor/brotli/c/enc/write_bits.h +29 -26
- data/vendor/brotli/c/include/brotli/decode.h +67 -2
- data/vendor/brotli/c/include/brotli/encode.h +77 -3
- data/vendor/brotli/c/include/brotli/port.h +34 -3
- data/vendor/brotli/c/include/brotli/shared_dictionary.h +100 -0
- metadata +23 -97
- data/.travis.yml +0 -31
- data/docs/Brotli/Error.html +0 -124
- data/docs/Brotli.html +0 -485
- data/docs/_index.html +0 -122
- data/docs/class_list.html +0 -51
- data/docs/css/common.css +0 -1
- data/docs/css/full_list.css +0 -58
- data/docs/css/style.css +0 -496
- data/docs/file.README.html +0 -127
- data/docs/file_list.html +0 -56
- data/docs/frames.html +0 -17
- data/docs/index.html +0 -127
- data/docs/js/app.js +0 -292
- data/docs/js/full_list.js +0 -216
- data/docs/js/jquery.js +0 -4
- data/docs/method_list.html +0 -67
- data/docs/top-level-namespace.html +0 -110
- data/spec/brotli_spec.rb +0 -88
- data/spec/inflate_spec.rb +0 -75
- data/spec/spec_helper.rb +0 -4
@@ -0,0 +1,104 @@
|
|
1
|
+
/* Copyright 2022 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
/* Encoder state. */
|
8
|
+
|
9
|
+
#ifndef BROTLI_ENC_STATE_H_
|
10
|
+
#define BROTLI_ENC_STATE_H_
|
11
|
+
|
12
|
+
#include <brotli/types.h>
|
13
|
+
|
14
|
+
#include "command.h"
|
15
|
+
#include "compress_fragment.h"
|
16
|
+
#include "compress_fragment_two_pass.h"
|
17
|
+
#include "hash.h"
|
18
|
+
#include "memory.h"
|
19
|
+
#include "params.h"
|
20
|
+
#include "ringbuffer.h"
|
21
|
+
|
22
|
+
typedef enum BrotliEncoderStreamState {
|
23
|
+
/* Default state. */
|
24
|
+
BROTLI_STREAM_PROCESSING = 0,
|
25
|
+
/* Intermediate state; after next block is emitted, byte-padding should be
|
26
|
+
performed before getting back to default state. */
|
27
|
+
BROTLI_STREAM_FLUSH_REQUESTED = 1,
|
28
|
+
/* Last metablock was produced; no more input is acceptable. */
|
29
|
+
BROTLI_STREAM_FINISHED = 2,
|
30
|
+
/* Flushing compressed block and writing meta-data block header. */
|
31
|
+
BROTLI_STREAM_METADATA_HEAD = 3,
|
32
|
+
/* Writing metadata block body. */
|
33
|
+
BROTLI_STREAM_METADATA_BODY = 4
|
34
|
+
} BrotliEncoderStreamState;
|
35
|
+
|
36
|
+
typedef enum BrotliEncoderFlintState {
|
37
|
+
BROTLI_FLINT_NEEDS_2_BYTES = 2,
|
38
|
+
BROTLI_FLINT_NEEDS_1_BYTE = 1,
|
39
|
+
BROTLI_FLINT_WAITING_FOR_PROCESSING = 0,
|
40
|
+
BROTLI_FLINT_WAITING_FOR_FLUSHING = -1,
|
41
|
+
BROTLI_FLINT_DONE = -2
|
42
|
+
} BrotliEncoderFlintState;
|
43
|
+
|
44
|
+
typedef struct BrotliEncoderStateStruct {
|
45
|
+
BrotliEncoderParams params;
|
46
|
+
|
47
|
+
MemoryManager memory_manager_;
|
48
|
+
|
49
|
+
uint64_t input_pos_;
|
50
|
+
RingBuffer ringbuffer_;
|
51
|
+
size_t cmd_alloc_size_;
|
52
|
+
Command* commands_;
|
53
|
+
size_t num_commands_;
|
54
|
+
size_t num_literals_;
|
55
|
+
size_t last_insert_len_;
|
56
|
+
uint64_t last_flush_pos_;
|
57
|
+
uint64_t last_processed_pos_;
|
58
|
+
int dist_cache_[BROTLI_NUM_DISTANCE_SHORT_CODES];
|
59
|
+
int saved_dist_cache_[4];
|
60
|
+
uint16_t last_bytes_;
|
61
|
+
uint8_t last_bytes_bits_;
|
62
|
+
/* "Flint" is a tiny uncompressed block emitted before the continuation
|
63
|
+
block to unwire literal context from previous data. Despite being int8_t,
|
64
|
+
field is actually BrotliEncoderFlintState enum. */
|
65
|
+
int8_t flint_;
|
66
|
+
uint8_t prev_byte_;
|
67
|
+
uint8_t prev_byte2_;
|
68
|
+
size_t storage_size_;
|
69
|
+
uint8_t* storage_;
|
70
|
+
|
71
|
+
Hasher hasher_;
|
72
|
+
|
73
|
+
/* Hash table for FAST_ONE_PASS_COMPRESSION_QUALITY mode. */
|
74
|
+
int small_table_[1 << 10]; /* 4KiB */
|
75
|
+
int* large_table_; /* Allocated only when needed */
|
76
|
+
size_t large_table_size_;
|
77
|
+
|
78
|
+
BrotliOnePassArena* one_pass_arena_;
|
79
|
+
BrotliTwoPassArena* two_pass_arena_;
|
80
|
+
|
81
|
+
/* Command and literal buffers for FAST_TWO_PASS_COMPRESSION_QUALITY. */
|
82
|
+
uint32_t* command_buf_;
|
83
|
+
uint8_t* literal_buf_;
|
84
|
+
|
85
|
+
uint64_t total_in_;
|
86
|
+
uint8_t* next_out_;
|
87
|
+
size_t available_out_;
|
88
|
+
uint64_t total_out_;
|
89
|
+
/* Temporary buffer for padding flush bits or metadata block header / body. */
|
90
|
+
union {
|
91
|
+
uint64_t u64[2];
|
92
|
+
uint8_t u8[16];
|
93
|
+
} tiny_buf_;
|
94
|
+
uint32_t remaining_metadata_bytes_;
|
95
|
+
BrotliEncoderStreamState stream_state_;
|
96
|
+
|
97
|
+
BROTLI_BOOL is_last_block_emitted_;
|
98
|
+
BROTLI_BOOL is_initialized_;
|
99
|
+
} BrotliEncoderStateStruct;
|
100
|
+
|
101
|
+
typedef struct BrotliEncoderStateStruct BrotliEncoderStateInternal;
|
102
|
+
#define BrotliEncoderState BrotliEncoderStateInternal
|
103
|
+
|
104
|
+
#endif // BROTLI_ENC_STATE_H_
|
@@ -4,13 +4,13 @@
|
|
4
4
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
5
|
*/
|
6
6
|
|
7
|
-
#include "
|
7
|
+
#include "static_dict.h"
|
8
8
|
|
9
9
|
#include "../common/dictionary.h"
|
10
10
|
#include "../common/platform.h"
|
11
11
|
#include "../common/transform.h"
|
12
|
-
#include "
|
13
|
-
#include "
|
12
|
+
#include "encoder_dict.h"
|
13
|
+
#include "find_match_length.h"
|
14
14
|
|
15
15
|
#if defined(__cplusplus) || defined(c_plusplus)
|
16
16
|
extern "C" {
|
@@ -74,10 +74,27 @@ static BROTLI_INLINE BROTLI_BOOL IsMatch(const BrotliDictionary* dictionary,
|
|
74
74
|
}
|
75
75
|
}
|
76
76
|
|
77
|
-
|
77
|
+
/* Finds matches for a single static dictionary */
|
78
|
+
static BROTLI_BOOL BrotliFindAllStaticDictionaryMatchesFor(
|
78
79
|
const BrotliEncoderDictionary* dictionary, const uint8_t* data,
|
79
80
|
size_t min_length, size_t max_length, uint32_t* matches) {
|
80
81
|
BROTLI_BOOL has_found_match = BROTLI_FALSE;
|
82
|
+
#if defined(BROTLI_EXPERIMENTAL)
|
83
|
+
if (dictionary->has_words_heavy) {
|
84
|
+
const BrotliTrieNode* node = &dictionary->trie.root;
|
85
|
+
size_t l = 0;
|
86
|
+
while (node && l < max_length) {
|
87
|
+
uint8_t c;
|
88
|
+
if (l >= min_length && node->len_) {
|
89
|
+
AddMatch(node->idx_, l, node->len_, matches);
|
90
|
+
has_found_match = BROTLI_TRUE;
|
91
|
+
}
|
92
|
+
c = data[l++];
|
93
|
+
node = BrotliTrieSub(&dictionary->trie, node, c);
|
94
|
+
}
|
95
|
+
return has_found_match;
|
96
|
+
}
|
97
|
+
#endif /* BROTLI_EXPERIMENTAL */
|
81
98
|
{
|
82
99
|
size_t offset = dictionary->buckets[Hash(data)];
|
83
100
|
BROTLI_BOOL end = !offset;
|
@@ -481,6 +498,45 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
481
498
|
return has_found_match;
|
482
499
|
}
|
483
500
|
|
501
|
+
/* Finds matches for one or more dictionaries, if multiple are present
|
502
|
+
in the contextual dictionary */
|
503
|
+
BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
504
|
+
const BrotliEncoderDictionary* dictionary, const uint8_t* data,
|
505
|
+
size_t min_length, size_t max_length, uint32_t* matches) {
|
506
|
+
BROTLI_BOOL has_found_match =
|
507
|
+
BrotliFindAllStaticDictionaryMatchesFor(
|
508
|
+
dictionary, data, min_length, max_length, matches);
|
509
|
+
|
510
|
+
if (!!dictionary->parent && dictionary->parent->num_dictionaries > 1) {
|
511
|
+
uint32_t matches2[BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1];
|
512
|
+
int l;
|
513
|
+
const BrotliEncoderDictionary* dictionary2 = dictionary->parent->dict[0];
|
514
|
+
if (dictionary2 == dictionary) {
|
515
|
+
dictionary2 = dictionary->parent->dict[1];
|
516
|
+
}
|
517
|
+
|
518
|
+
for (l = 0; l < BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1; l++) {
|
519
|
+
matches2[l] = kInvalidMatch;
|
520
|
+
}
|
521
|
+
|
522
|
+
has_found_match |= BrotliFindAllStaticDictionaryMatchesFor(
|
523
|
+
dictionary2, data, min_length, max_length, matches2);
|
524
|
+
|
525
|
+
for (l = 0; l < BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1; l++) {
|
526
|
+
if (matches2[l] != kInvalidMatch) {
|
527
|
+
uint32_t dist = (uint32_t)(matches2[l] >> 5);
|
528
|
+
uint32_t len_code = matches2[l] & 31;
|
529
|
+
uint32_t skipdist = (uint32_t)((uint32_t)(1 << dictionary->words->
|
530
|
+
size_bits_by_length[len_code]) & ~1u) *
|
531
|
+
(uint32_t)dictionary->num_transforms;
|
532
|
+
/* TODO(lode): check for dist overflow */
|
533
|
+
dist += skipdist;
|
534
|
+
AddMatch(dist, (size_t)l, len_code, matches);
|
535
|
+
}
|
536
|
+
}
|
537
|
+
}
|
538
|
+
return has_found_match;
|
539
|
+
}
|
484
540
|
#if defined(__cplusplus) || defined(c_plusplus)
|
485
541
|
} /* extern "C" */
|
486
542
|
#endif
|
@@ -9,10 +9,11 @@
|
|
9
9
|
#ifndef BROTLI_ENC_STATIC_DICT_H_
|
10
10
|
#define BROTLI_ENC_STATIC_DICT_H_
|
11
11
|
|
12
|
+
#include <brotli/types.h>
|
13
|
+
|
12
14
|
#include "../common/dictionary.h"
|
13
15
|
#include "../common/platform.h"
|
14
|
-
#include
|
15
|
-
#include "./encoder_dict.h"
|
16
|
+
#include "encoder_dict.h"
|
16
17
|
|
17
18
|
#if defined(__cplusplus) || defined(c_plusplus)
|
18
19
|
extern "C" {
|
@@ -22,6 +22,7 @@ typedef struct DictWord {
|
|
22
22
|
uint16_t idx;
|
23
23
|
} DictWord;
|
24
24
|
|
25
|
+
/* GENERATED CODE START */
|
25
26
|
static const int kDictNumBits = 15;
|
26
27
|
static const uint32_t kDictHashMul32 = 0x1E35A7BD;
|
27
28
|
|
@@ -5856,6 +5857,7 @@ static const DictWord kStaticDictionaryWords[31705] = {
|
|
5856
5857
|
,0,1735},{5,0,598},{7,0,791},{8,0,108},{9,0,123},{7,10,1570},{140,10,542},{142,
|
5857
5858
|
11,410},{9,11,660},{138,11,347}
|
5858
5859
|
};
|
5860
|
+
/* GENERATED CODE END */
|
5859
5861
|
|
5860
5862
|
#if defined(__cplusplus) || defined(c_plusplus)
|
5861
5863
|
} /* extern "C" */
|
@@ -6,7 +6,7 @@
|
|
6
6
|
|
7
7
|
/* Heuristics for deciding about the UTF8-ness of strings. */
|
8
8
|
|
9
|
-
#include "
|
9
|
+
#include "utf8_util.h"
|
10
10
|
|
11
11
|
#include <brotli/types.h>
|
12
12
|
|
@@ -77,7 +77,7 @@ BROTLI_BOOL BrotliIsMostlyUTF8(
|
|
77
77
|
i += bytes_read;
|
78
78
|
if (symbol < 0x110000) size_utf8 += bytes_read;
|
79
79
|
}
|
80
|
-
return TO_BROTLI_BOOL(size_utf8 > min_fraction * (double)length);
|
80
|
+
return TO_BROTLI_BOOL((double)size_utf8 > min_fraction * (double)length);
|
81
81
|
}
|
82
82
|
|
83
83
|
#if defined(__cplusplus) || defined(c_plusplus)
|
@@ -9,15 +9,14 @@
|
|
9
9
|
#ifndef BROTLI_ENC_WRITE_BITS_H_
|
10
10
|
#define BROTLI_ENC_WRITE_BITS_H_
|
11
11
|
|
12
|
-
#include "../common/platform.h"
|
13
12
|
#include <brotli/types.h>
|
14
13
|
|
14
|
+
#include "../common/platform.h"
|
15
|
+
|
15
16
|
#if defined(__cplusplus) || defined(c_plusplus)
|
16
17
|
extern "C" {
|
17
18
|
#endif
|
18
19
|
|
19
|
-
/*#define BIT_WRITER_DEBUG */
|
20
|
-
|
21
20
|
/* This function writes bits into bytes in increasing addresses, and within
|
22
21
|
a byte least-significant-bit first.
|
23
22
|
|
@@ -28,7 +27,7 @@ extern "C" {
|
|
28
27
|
|
29
28
|
0000 0RRR 0000 0000 0000 0000
|
30
29
|
|
31
|
-
Now, we could write 5 or less bits in MSB by just
|
30
|
+
Now, we could write 5 or less bits in MSB by just shifting by 3
|
32
31
|
and OR'ing to BYTE-0.
|
33
32
|
|
34
33
|
For n bits, we take the last 5 bits, OR that with high bits in BYTE-0,
|
@@ -37,37 +36,41 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits,
|
|
37
36
|
uint64_t bits,
|
38
37
|
size_t* BROTLI_RESTRICT pos,
|
39
38
|
uint8_t* BROTLI_RESTRICT array) {
|
39
|
+
BROTLI_LOG(("WriteBits %2d 0x%08x%08x %10d\n", (int)n_bits,
|
40
|
+
(uint32_t)(bits >> 32), (uint32_t)(bits & 0xFFFFFFFF),
|
41
|
+
(int)*pos));
|
42
|
+
BROTLI_DCHECK((bits >> n_bits) == 0);
|
43
|
+
BROTLI_DCHECK(n_bits <= 56);
|
40
44
|
#if defined(BROTLI_LITTLE_ENDIAN)
|
41
45
|
/* This branch of the code can write up to 56 bits at a time,
|
42
46
|
7 bits are lost by being perhaps already in *p and at least
|
43
47
|
1 bit is needed to initialize the bit-stream ahead (i.e. if 7
|
44
48
|
bits are in *p and we write 57 bits, then the next write will
|
45
49
|
access a byte that was never initialized). */
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
v |= bits << (*pos & 7);
|
54
|
-
BROTLI_UNALIGNED_STORE64LE(p, v); /* Set some bits. */
|
55
|
-
*pos += n_bits;
|
50
|
+
{
|
51
|
+
uint8_t* p = &array[*pos >> 3];
|
52
|
+
uint64_t v = (uint64_t)(*p); /* Zero-extend 8 to 64 bits. */
|
53
|
+
v |= bits << (*pos & 7);
|
54
|
+
BROTLI_UNALIGNED_STORE64LE(p, v); /* Set some bits. */
|
55
|
+
*pos += n_bits;
|
56
|
+
}
|
56
57
|
#else
|
57
58
|
/* implicit & 0xFF is assumed for uint8_t arithmetics */
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
59
|
+
{
|
60
|
+
uint8_t* array_pos = &array[*pos >> 3];
|
61
|
+
const size_t bits_reserved_in_first_byte = (*pos & 7);
|
62
|
+
size_t bits_left_to_write;
|
63
|
+
bits <<= bits_reserved_in_first_byte;
|
64
|
+
*array_pos++ |= (uint8_t)bits;
|
65
|
+
for (bits_left_to_write = n_bits + bits_reserved_in_first_byte;
|
66
|
+
bits_left_to_write >= 9;
|
67
|
+
bits_left_to_write -= 8) {
|
68
|
+
bits >>= 8;
|
69
|
+
*array_pos++ = (uint8_t)bits;
|
70
|
+
}
|
71
|
+
*array_pos = 0;
|
72
|
+
*pos += n_bits;
|
68
73
|
}
|
69
|
-
*array_pos = 0;
|
70
|
-
*pos += n_bits;
|
71
74
|
#endif
|
72
75
|
}
|
73
76
|
|
@@ -13,6 +13,7 @@
|
|
13
13
|
#define BROTLI_DEC_DECODE_H_
|
14
14
|
|
15
15
|
#include <brotli/port.h>
|
16
|
+
#include <brotli/shared_dictionary.h>
|
16
17
|
#include <brotli/types.h>
|
17
18
|
|
18
19
|
#if defined(__cplusplus) || defined(c_plusplus)
|
@@ -85,8 +86,9 @@ typedef enum {
|
|
85
86
|
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR \
|
86
87
|
BROTLI_ERROR_CODE(_ERROR_FORMAT_, DISTANCE, -16) SEPARATOR \
|
87
88
|
\
|
88
|
-
/* -17
|
89
|
+
/* -17 code is reserved */ \
|
89
90
|
\
|
91
|
+
BROTLI_ERROR_CODE(_ERROR_, COMPOUND_DICTIONARY, -18) SEPARATOR \
|
90
92
|
BROTLI_ERROR_CODE(_ERROR_, DICTIONARY_NOT_SET, -19) SEPARATOR \
|
91
93
|
BROTLI_ERROR_CODE(_ERROR_, INVALID_ARGUMENTS, -20) SEPARATOR \
|
92
94
|
\
|
@@ -154,6 +156,28 @@ typedef enum BrotliDecoderParameter {
|
|
154
156
|
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderSetParameter(
|
155
157
|
BrotliDecoderState* state, BrotliDecoderParameter param, uint32_t value);
|
156
158
|
|
159
|
+
/**
|
160
|
+
* Adds LZ77 prefix dictionary, adds or replaces built-in static dictionary and
|
161
|
+
* transforms.
|
162
|
+
*
|
163
|
+
* Attached dictionary ownership is not transferred.
|
164
|
+
* Data provided to this method should be kept accessible until
|
165
|
+
* decoding is finished and decoder instance is destroyed.
|
166
|
+
*
|
167
|
+
* @note Dictionaries can NOT be attached after actual decoding is started.
|
168
|
+
*
|
169
|
+
* @param state decoder instance
|
170
|
+
* @param type dictionary data format
|
171
|
+
* @param data_size length of memory region pointed by @p data
|
172
|
+
* @param data dictionary data in format corresponding to @p type
|
173
|
+
* @returns ::BROTLI_FALSE if dictionary is corrupted,
|
174
|
+
* or dictionary count limit is reached
|
175
|
+
* @returns ::BROTLI_TRUE if dictionary is accepted / attached
|
176
|
+
*/
|
177
|
+
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderAttachDictionary(
|
178
|
+
BrotliDecoderState* state, BrotliSharedDictionaryType type,
|
179
|
+
size_t data_size, const uint8_t data[BROTLI_ARRAY_PARAM(data_size)]);
|
180
|
+
|
157
181
|
/**
|
158
182
|
* Creates an instance of ::BrotliDecoderState and initializes it.
|
159
183
|
*
|
@@ -333,10 +357,51 @@ BROTLI_DEC_API const char* BrotliDecoderErrorString(BrotliDecoderErrorCode c);
|
|
333
357
|
/**
|
334
358
|
* Gets a decoder library version.
|
335
359
|
*
|
336
|
-
* Look at
|
360
|
+
* Look at BROTLI_MAKE_HEX_VERSION for more information.
|
337
361
|
*/
|
338
362
|
BROTLI_DEC_API uint32_t BrotliDecoderVersion(void);
|
339
363
|
|
364
|
+
/**
|
365
|
+
* Callback to fire on metadata block start.
|
366
|
+
*
|
367
|
+
* After this callback is fired, if @p size is not @c 0, it is followed by
|
368
|
+
* ::brotli_decoder_metadata_chunk_func as more metadata block contents become
|
369
|
+
* accessible.
|
370
|
+
*
|
371
|
+
* @param opaque callback handle
|
372
|
+
* @param size size of metadata block
|
373
|
+
*/
|
374
|
+
typedef void (*brotli_decoder_metadata_start_func)(void* opaque, size_t size);
|
375
|
+
|
376
|
+
/**
|
377
|
+
* Callback to fire on metadata block chunk becomes available.
|
378
|
+
*
|
379
|
+
* This function can be invoked multiple times per metadata block; block should
|
380
|
+
* be considered finished when sum of @p size matches the announced metadata
|
381
|
+
* block size. Chunks contents pointed by @p data are transient and shouln not
|
382
|
+
* be accessed after leaving the callback.
|
383
|
+
*
|
384
|
+
* @param opaque callback handle
|
385
|
+
* @param data pointer to metadata contents
|
386
|
+
* @param size size of metadata block chunk, at least @c 1
|
387
|
+
*/
|
388
|
+
typedef void (*brotli_decoder_metadata_chunk_func)(void* opaque,
|
389
|
+
const uint8_t* data,
|
390
|
+
size_t size);
|
391
|
+
|
392
|
+
/**
|
393
|
+
* Sets callback for receiving metadata blocks.
|
394
|
+
*
|
395
|
+
* @param state decoder instance
|
396
|
+
* @param start_func callback on metadata block start
|
397
|
+
* @param chunk_func callback on metadata block chunk
|
398
|
+
* @param opaque callback handle
|
399
|
+
*/
|
400
|
+
BROTLI_DEC_API void BrotliDecoderSetMetadataCallbacks(
|
401
|
+
BrotliDecoderState* state,
|
402
|
+
brotli_decoder_metadata_start_func start_func,
|
403
|
+
brotli_decoder_metadata_chunk_func chunk_func, void* opaque);
|
404
|
+
|
340
405
|
#if defined(__cplusplus) || defined(c_plusplus)
|
341
406
|
} /* extern "C" */
|
342
407
|
#endif
|
@@ -13,6 +13,7 @@
|
|
13
13
|
#define BROTLI_ENC_ENCODE_H_
|
14
14
|
|
15
15
|
#include <brotli/port.h>
|
16
|
+
#include <brotli/shared_dictionary.h>
|
16
17
|
#include <brotli/types.h>
|
17
18
|
|
18
19
|
#if defined(__cplusplus) || defined(c_plusplus)
|
@@ -201,7 +202,23 @@ typedef enum BrotliEncoderParameter {
|
|
201
202
|
*
|
202
203
|
* Range is from 0 to (15 << NPOSTFIX) in steps of (1 << NPOSTFIX).
|
203
204
|
*/
|
204
|
-
BROTLI_PARAM_NDIRECT = 8
|
205
|
+
BROTLI_PARAM_NDIRECT = 8,
|
206
|
+
/**
|
207
|
+
* Number of bytes of input stream already processed by a different instance.
|
208
|
+
*
|
209
|
+
* @note It is important to configure all the encoder instances with same
|
210
|
+
* parameters (except this one) in order to allow all the encoded parts
|
211
|
+
* obey the same restrictions implied by header.
|
212
|
+
*
|
213
|
+
* If offset is not 0, then stream header is omitted.
|
214
|
+
* In any case output start is byte aligned, so for proper streams stitching
|
215
|
+
* "predecessor" stream must be flushed.
|
216
|
+
*
|
217
|
+
* Range is not artificially limited, but all the values greater or equal to
|
218
|
+
* maximal window size have the same effect. Values greater than 2**30 are not
|
219
|
+
* allowed.
|
220
|
+
*/
|
221
|
+
BROTLI_PARAM_STREAM_OFFSET = 9
|
205
222
|
} BrotliEncoderParameter;
|
206
223
|
|
207
224
|
/**
|
@@ -253,6 +270,51 @@ BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance(
|
|
253
270
|
*/
|
254
271
|
BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state);
|
255
272
|
|
273
|
+
/* Opaque type for pointer to different possible internal structures containing
|
274
|
+
dictionary prepared for the encoder */
|
275
|
+
typedef struct BrotliEncoderPreparedDictionaryStruct
|
276
|
+
BrotliEncoderPreparedDictionary;
|
277
|
+
|
278
|
+
/**
|
279
|
+
* Prepares a shared dictionary from the given file format for the encoder.
|
280
|
+
*
|
281
|
+
* @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
|
282
|
+
* case they are both zero, default memory allocators are used. @p opaque is
|
283
|
+
* passed to @p alloc_func and @p free_func when they are called. @p free_func
|
284
|
+
* has to return without doing anything when asked to free a NULL pointer.
|
285
|
+
*
|
286
|
+
* @param type type of dictionary stored in data
|
287
|
+
* @param data_size size of @p data buffer
|
288
|
+
* @param data pointer to the dictionary data
|
289
|
+
* @param quality the maximum Brotli quality to prepare the dictionary for,
|
290
|
+
* use BROTLI_MAX_QUALITY by default
|
291
|
+
* @param alloc_func custom memory allocation function
|
292
|
+
* @param free_func custom memory free function
|
293
|
+
* @param opaque custom memory manager handle
|
294
|
+
*/
|
295
|
+
BROTLI_ENC_API BrotliEncoderPreparedDictionary*
|
296
|
+
BrotliEncoderPrepareDictionary(BrotliSharedDictionaryType type,
|
297
|
+
size_t data_size, const uint8_t data[BROTLI_ARRAY_PARAM(data_size)],
|
298
|
+
int quality,
|
299
|
+
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
|
300
|
+
|
301
|
+
BROTLI_ENC_API void BrotliEncoderDestroyPreparedDictionary(
|
302
|
+
BrotliEncoderPreparedDictionary* dictionary);
|
303
|
+
|
304
|
+
/**
|
305
|
+
* Attaches a prepared dictionary of any type to the encoder. Can be used
|
306
|
+
* multiple times to attach multiple dictionaries. The dictionary type was
|
307
|
+
* determined by BrotliEncoderPrepareDictionary. Multiple raw prefix
|
308
|
+
* dictionaries and/or max 1 serialized dictionary with custom words can be
|
309
|
+
* attached.
|
310
|
+
*
|
311
|
+
* @returns ::BROTLI_FALSE in case of error
|
312
|
+
* @returns ::BROTLI_TRUE otherwise
|
313
|
+
*/
|
314
|
+
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderAttachPreparedDictionary(
|
315
|
+
BrotliEncoderState* state,
|
316
|
+
const BrotliEncoderPreparedDictionary* dictionary);
|
317
|
+
|
256
318
|
/**
|
257
319
|
* Calculates the output size bound for the given @p input_size.
|
258
320
|
*
|
@@ -274,6 +336,11 @@ BROTLI_ENC_API size_t BrotliEncoderMaxCompressedSize(size_t input_size);
|
|
274
336
|
* @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero
|
275
337
|
* value, then output is guaranteed to be no longer than that.
|
276
338
|
*
|
339
|
+
* @note If @p lgwin is greater than ::BROTLI_MAX_WINDOW_BITS then resulting
|
340
|
+
* stream might be incompatible with RFC 7932; to decode such streams,
|
341
|
+
* decoder should be configured with
|
342
|
+
* ::BROTLI_DECODER_PARAM_LARGE_WINDOW = @c 1
|
343
|
+
*
|
277
344
|
* @param quality quality parameter value, e.g. ::BROTLI_DEFAULT_QUALITY
|
278
345
|
* @param lgwin lgwin parameter value, e.g. ::BROTLI_DEFAULT_WINDOW
|
279
346
|
* @param mode mode parameter value, e.g. ::BROTLI_DEFAULT_MODE
|
@@ -386,7 +453,7 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
|
|
386
453
|
*
|
387
454
|
* This method is used to make language bindings easier and more efficient:
|
388
455
|
* -# push data to ::BrotliEncoderCompressStream,
|
389
|
-
* until ::BrotliEncoderHasMoreOutput returns
|
456
|
+
* until ::BrotliEncoderHasMoreOutput returns BROTLI_TRUE
|
390
457
|
* -# use ::BrotliEncoderTakeOutput to peek bytes and copy to language-specific
|
391
458
|
* entity
|
392
459
|
*
|
@@ -412,11 +479,18 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
|
|
412
479
|
BROTLI_ENC_API const uint8_t* BrotliEncoderTakeOutput(
|
413
480
|
BrotliEncoderState* state, size_t* size);
|
414
481
|
|
482
|
+
/* Returns the estimated peak memory usage (in bytes) of the BrotliCompress()
|
483
|
+
function, not counting the memory needed for the input and output. */
|
484
|
+
BROTLI_ENC_EXTRA_API size_t BrotliEncoderEstimatePeakMemoryUsage(
|
485
|
+
int quality, int lgwin, size_t input_size);
|
486
|
+
/* Returns 0 if dictionary is not valid; otherwise returns allocation size. */
|
487
|
+
BROTLI_ENC_EXTRA_API size_t BrotliEncoderGetPreparedDictionarySize(
|
488
|
+
const BrotliEncoderPreparedDictionary* dictionary);
|
415
489
|
|
416
490
|
/**
|
417
491
|
* Gets an encoder library version.
|
418
492
|
*
|
419
|
-
* Look at
|
493
|
+
* Look at BROTLI_MAKE_HEX_VERSION for more information.
|
420
494
|
*/
|
421
495
|
BROTLI_ENC_API uint32_t BrotliEncoderVersion(void);
|
422
496
|
|
@@ -218,6 +218,12 @@
|
|
218
218
|
BROTLI_GNUC_VERSION_CHECK(major, minor, patch)
|
219
219
|
#endif
|
220
220
|
|
221
|
+
#if defined(__has_feature)
|
222
|
+
#define BROTLI_HAS_FEATURE(feature) __has_feature(feature)
|
223
|
+
#else
|
224
|
+
#define BROTLI_HAS_FEATURE(feature) (0)
|
225
|
+
#endif
|
226
|
+
|
221
227
|
#if defined(_WIN32) || defined(__CYGWIN__)
|
222
228
|
#define BROTLI_PUBLIC
|
223
229
|
#elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) || \
|
@@ -233,9 +239,28 @@
|
|
233
239
|
#define BROTLI_PUBLIC
|
234
240
|
#endif
|
235
241
|
|
236
|
-
|
237
|
-
|
238
|
-
|
242
|
+
/* BROTLI_INTERNAL could be defined to override visibility, e.g. for tests. */
|
243
|
+
#if !defined(BROTLI_INTERNAL)
|
244
|
+
#if defined(_WIN32) || defined(__CYGWIN__)
|
245
|
+
#define BROTLI_INTERNAL
|
246
|
+
#elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) || \
|
247
|
+
BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
|
248
|
+
BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
|
249
|
+
BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
|
250
|
+
BROTLI_IBM_VERSION_CHECK(13, 1, 0) || \
|
251
|
+
BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \
|
252
|
+
(BROTLI_TI_VERSION_CHECK(7, 3, 0) && \
|
253
|
+
defined(__TI_GNU_ATTRIBUTE_SUPPORT__) && defined(__TI_EABI__))
|
254
|
+
#define BROTLI_INTERNAL __attribute__ ((visibility ("hidden")))
|
255
|
+
#else
|
256
|
+
#define BROTLI_INTERNAL
|
257
|
+
#endif
|
258
|
+
#endif
|
259
|
+
|
260
|
+
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
|
261
|
+
!defined(__STDC_NO_VLA__) && !defined(__cplusplus) && \
|
262
|
+
!defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__) && \
|
263
|
+
!defined(__clang__)
|
239
264
|
#define BROTLI_ARRAY_PARAM(name) (name)
|
240
265
|
#else
|
241
266
|
#define BROTLI_ARRAY_PARAM(name)
|
@@ -271,4 +296,10 @@
|
|
271
296
|
#define BROTLI_ENC_API
|
272
297
|
#endif
|
273
298
|
|
299
|
+
#if defined(BROTLI_BUILD_ENC_EXTRA_API)
|
300
|
+
#define BROTLI_ENC_EXTRA_API BROTLI_ENC_API
|
301
|
+
#else
|
302
|
+
#define BROTLI_ENC_EXTRA_API BROTLI_INTERNAL
|
303
|
+
#endif
|
304
|
+
|
274
305
|
#endif /* BROTLI_COMMON_PORT_H_ */
|