brotli 0.1.8 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +7 -3
- data/brotli.gemspec +1 -1
- data/ext/brotli/brotli.c +4 -4
- data/ext/brotli/brotli.h +2 -2
- data/ext/brotli/extconf.rb +9 -16
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/{common → c/common}/constants.h +11 -1
- data/vendor/brotli/c/common/dictionary.bin +432 -0
- data/vendor/brotli/c/common/dictionary.c +5905 -0
- data/vendor/brotli/c/common/dictionary.h +64 -0
- data/vendor/brotli/c/common/version.h +19 -0
- data/vendor/brotli/{dec → c/dec}/bit_reader.c +2 -2
- data/vendor/brotli/{dec → c/dec}/bit_reader.h +11 -34
- data/vendor/brotli/{dec → c/dec}/context.h +1 -1
- data/vendor/brotli/{dec → c/dec}/decode.c +389 -356
- data/vendor/brotli/{dec → c/dec}/huffman.c +24 -23
- data/vendor/brotli/{dec → c/dec}/huffman.h +1 -1
- data/vendor/brotli/{dec → c/dec}/port.h +19 -10
- data/vendor/brotli/{dec → c/dec}/prefix.h +1 -1
- data/vendor/brotli/{dec → c/dec}/state.c +23 -19
- data/vendor/brotli/{dec → c/dec}/state.h +18 -17
- data/vendor/brotli/{dec → c/dec}/transform.h +2 -2
- data/vendor/brotli/c/enc/backward_references.c +134 -0
- data/vendor/brotli/c/enc/backward_references.h +39 -0
- data/vendor/brotli/{enc/backward_references.c → c/enc/backward_references_hq.c} +144 -232
- data/vendor/brotli/{enc/backward_references.h → c/enc/backward_references_hq.h} +28 -31
- data/vendor/brotli/{enc → c/enc}/backward_references_inc.h +37 -31
- data/vendor/brotli/{enc → c/enc}/bit_cost.c +1 -1
- data/vendor/brotli/{enc → c/enc}/bit_cost.h +1 -1
- data/vendor/brotli/{enc → c/enc}/bit_cost_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/block_encoder_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/block_splitter.c +2 -4
- data/vendor/brotli/{enc → c/enc}/block_splitter.h +1 -1
- data/vendor/brotli/{enc → c/enc}/block_splitter_inc.h +6 -7
- data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.c +22 -26
- data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.h +1 -5
- data/vendor/brotli/{enc → c/enc}/cluster.c +1 -1
- data/vendor/brotli/{enc → c/enc}/cluster.h +1 -1
- data/vendor/brotli/{enc → c/enc}/cluster_inc.h +2 -0
- data/vendor/brotli/{enc → c/enc}/command.h +34 -17
- data/vendor/brotli/{enc → c/enc}/compress_fragment.c +97 -53
- data/vendor/brotli/{enc → c/enc}/compress_fragment.h +5 -2
- data/vendor/brotli/{enc → c/enc}/compress_fragment_two_pass.c +106 -51
- data/vendor/brotli/{enc → c/enc}/compress_fragment_two_pass.h +5 -2
- data/vendor/brotli/{enc → c/enc}/context.h +3 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +1120 -0
- data/vendor/brotli/c/enc/dictionary_hash.h +24 -0
- data/vendor/brotli/{enc → c/enc}/encode.c +442 -240
- data/vendor/brotli/{enc → c/enc}/entropy_encode.c +9 -9
- data/vendor/brotli/{enc → c/enc}/entropy_encode.h +4 -4
- data/vendor/brotli/{enc → c/enc}/entropy_encode_static.h +4 -4
- data/vendor/brotli/{enc → c/enc}/fast_log.h +3 -3
- data/vendor/brotli/{enc → c/enc}/find_match_length.h +8 -8
- data/vendor/brotli/c/enc/hash.h +446 -0
- data/vendor/brotli/{enc → c/enc}/hash_forgetful_chain_inc.h +72 -68
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +266 -0
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +258 -0
- data/vendor/brotli/{enc → c/enc}/hash_longest_match_quickly_inc.h +81 -77
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +326 -0
- data/vendor/brotli/{enc → c/enc}/histogram.c +4 -2
- data/vendor/brotli/{enc → c/enc}/histogram.h +1 -1
- data/vendor/brotli/{enc → c/enc}/histogram_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/literal_cost.c +4 -7
- data/vendor/brotli/{enc → c/enc}/literal_cost.h +2 -2
- data/vendor/brotli/{enc → c/enc}/memory.c +1 -1
- data/vendor/brotli/{enc → c/enc}/memory.h +3 -2
- data/vendor/brotli/{enc → c/enc}/metablock.c +136 -123
- data/vendor/brotli/{enc → c/enc}/metablock.h +2 -12
- data/vendor/brotli/{enc → c/enc}/metablock_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/port.h +49 -33
- data/vendor/brotli/{enc → c/enc}/prefix.h +4 -2
- data/vendor/brotli/{enc → c/enc}/quality.h +47 -17
- data/vendor/brotli/{enc → c/enc}/ringbuffer.h +6 -6
- data/vendor/brotli/{enc → c/enc}/static_dict.c +26 -22
- data/vendor/brotli/{enc → c/enc}/static_dict.h +3 -1
- data/vendor/brotli/c/enc/static_dict_lut.h +5864 -0
- data/vendor/brotli/{enc → c/enc}/utf8_util.c +1 -1
- data/vendor/brotli/{enc → c/enc}/utf8_util.h +2 -2
- data/vendor/brotli/{enc → c/enc}/write_bits.h +3 -3
- data/vendor/brotli/c/include/brotli/decode.h +339 -0
- data/vendor/brotli/c/include/brotli/encode.h +402 -0
- data/vendor/brotli/c/include/brotli/port.h +146 -0
- data/vendor/brotli/c/include/brotli/types.h +90 -0
- metadata +80 -79
- data/vendor/brotli/common/dictionary.c +0 -9474
- data/vendor/brotli/common/dictionary.h +0 -29
- data/vendor/brotli/common/port.h +0 -107
- data/vendor/brotli/common/types.h +0 -58
- data/vendor/brotli/dec/decode.h +0 -188
- data/vendor/brotli/enc/compressor.cc +0 -139
- data/vendor/brotli/enc/compressor.h +0 -161
- data/vendor/brotli/enc/dictionary_hash.h +0 -4121
- data/vendor/brotli/enc/encode.h +0 -221
- data/vendor/brotli/enc/encode_parallel.cc +0 -289
- data/vendor/brotli/enc/encode_parallel.h +0 -27
- data/vendor/brotli/enc/hash.h +0 -717
- data/vendor/brotli/enc/hash_longest_match_inc.h +0 -241
- data/vendor/brotli/enc/static_dict_lut.h +0 -11241
- data/vendor/brotli/enc/streams.cc +0 -114
- data/vendor/brotli/enc/streams.h +0 -121
@@ -10,7 +10,7 @@
|
|
10
10
|
#ifndef BROTLI_ENC_METABLOCK_H_
|
11
11
|
#define BROTLI_ENC_METABLOCK_H_
|
12
12
|
|
13
|
-
#include
|
13
|
+
#include <brotli/types.h>
|
14
14
|
#include "./block_splitter.h"
|
15
15
|
#include "./command.h"
|
16
16
|
#include "./context.h"
|
@@ -80,20 +80,10 @@ BROTLI_INTERNAL void BrotliBuildMetaBlock(MemoryManager* m,
|
|
80
80
|
ContextType literal_context_mode,
|
81
81
|
MetaBlockSplit* mb);
|
82
82
|
|
83
|
-
/* Uses a fast greedy block splitter that tries to merge current block with the
|
84
|
-
last or the second last block and does not do any context modeling. */
|
85
|
-
BROTLI_INTERNAL void BrotliBuildMetaBlockGreedy(MemoryManager* m,
|
86
|
-
const uint8_t* ringbuffer,
|
87
|
-
size_t pos,
|
88
|
-
size_t mask,
|
89
|
-
const Command* commands,
|
90
|
-
size_t n_commands,
|
91
|
-
MetaBlockSplit* mb);
|
92
|
-
|
93
83
|
/* Uses a fast greedy block splitter that tries to merge current block with the
|
94
84
|
last or the second last block and uses a static context clustering which
|
95
85
|
is the same for all block types. */
|
96
|
-
BROTLI_INTERNAL void
|
86
|
+
BROTLI_INTERNAL void BrotliBuildMetaBlockGreedy(
|
97
87
|
MemoryManager* m, const uint8_t* ringbuffer, size_t pos, size_t mask,
|
98
88
|
uint8_t prev_byte, uint8_t prev_byte2, ContextType literal_context_mode,
|
99
89
|
size_t num_contexts, const uint32_t* static_context_map,
|
File without changes
|
@@ -12,8 +12,8 @@
|
|
12
12
|
#include <assert.h>
|
13
13
|
#include <string.h> /* memcpy */
|
14
14
|
|
15
|
-
#include
|
16
|
-
#include
|
15
|
+
#include <brotli/port.h>
|
16
|
+
#include <brotli/types.h>
|
17
17
|
|
18
18
|
#if defined OS_LINUX || defined OS_CYGWIN
|
19
19
|
#include <endian.h>
|
@@ -26,36 +26,37 @@
|
|
26
26
|
#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
27
27
|
#endif
|
28
28
|
|
29
|
-
/* define the macro
|
29
|
+
/* define the macro BROTLI_LITTLE_ENDIAN
|
30
30
|
using the above endian definitions from endian.h if
|
31
31
|
endian.h was included */
|
32
32
|
#ifdef __BYTE_ORDER
|
33
33
|
#if __BYTE_ORDER == __LITTLE_ENDIAN
|
34
|
-
#define
|
34
|
+
#define BROTLI_LITTLE_ENDIAN
|
35
35
|
#endif
|
36
36
|
|
37
37
|
#else
|
38
38
|
|
39
39
|
#if defined(__LITTLE_ENDIAN__)
|
40
|
-
#define
|
40
|
+
#define BROTLI_LITTLE_ENDIAN
|
41
41
|
#endif
|
42
42
|
#endif /* __BYTE_ORDER */
|
43
43
|
|
44
44
|
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
45
|
-
#define
|
45
|
+
#define BROTLI_LITTLE_ENDIAN
|
46
46
|
#endif
|
47
47
|
|
48
48
|
/* Enable little-endian optimization for x64 architecture on Windows. */
|
49
49
|
#if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
|
50
|
-
#define
|
50
|
+
#define BROTLI_LITTLE_ENDIAN
|
51
51
|
#endif
|
52
52
|
|
53
53
|
/* Portable handling of unaligned loads, stores, and copies.
|
54
54
|
On some platforms, like ARM, the copy functions can be more efficient
|
55
55
|
then a load and a store. */
|
56
56
|
|
57
|
-
#if defined(
|
58
|
-
|
57
|
+
#if defined(BROTLI_LITTLE_ENDIAN) && (\
|
58
|
+
defined(ARCH_PIII) || defined(ARCH_ATHLON) || \
|
59
|
+
defined(ARCH_K8) || defined(_ARCH_PPC))
|
59
60
|
|
60
61
|
/* x86 and x86-64 can perform unaligned loads/stores directly;
|
61
62
|
modern PowerPC hardware can also do unaligned integer loads and stores;
|
@@ -63,14 +64,12 @@
|
|
63
64
|
*/
|
64
65
|
|
65
66
|
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
|
66
|
-
#define
|
67
|
+
#define BROTLI_UNALIGNED_LOAD64LE(_p) (*(const uint64_t *)(_p))
|
67
68
|
|
68
|
-
#define
|
69
|
-
(*(uint32_t *)(_p) = (_val))
|
70
|
-
#define BROTLI_UNALIGNED_STORE64(_p, _val) \
|
69
|
+
#define BROTLI_UNALIGNED_STORE64LE(_p, _val) \
|
71
70
|
(*(uint64_t *)(_p) = (_val))
|
72
71
|
|
73
|
-
#elif defined(__arm__) && \
|
72
|
+
#elif defined(BROTLI_LITTLE_ENDIAN) && defined(__arm__) && \
|
74
73
|
!defined(__ARM_ARCH_5__) && \
|
75
74
|
!defined(__ARM_ARCH_5T__) && \
|
76
75
|
!defined(__ARM_ARCH_5TE__) && \
|
@@ -88,16 +87,14 @@
|
|
88
87
|
slowly (trip through kernel mode). */
|
89
88
|
|
90
89
|
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
|
91
|
-
#define BROTLI_UNALIGNED_STORE32(_p, _val) \
|
92
|
-
(*(uint32_t *)(_p) = (_val))
|
93
90
|
|
94
|
-
static BROTLI_INLINE uint64_t
|
91
|
+
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
|
95
92
|
uint64_t t;
|
96
93
|
memcpy(&t, p, sizeof t);
|
97
94
|
return t;
|
98
95
|
}
|
99
96
|
|
100
|
-
static BROTLI_INLINE void
|
97
|
+
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
|
101
98
|
memcpy(p, &v, sizeof v);
|
102
99
|
}
|
103
100
|
|
@@ -112,36 +109,55 @@ static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) {
|
|
112
109
|
return t;
|
113
110
|
}
|
114
111
|
|
115
|
-
|
112
|
+
#if defined(BROTLI_LITTLE_ENDIAN)
|
113
|
+
|
114
|
+
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
|
116
115
|
uint64_t t;
|
117
116
|
memcpy(&t, p, sizeof t);
|
118
117
|
return t;
|
119
118
|
}
|
120
119
|
|
121
|
-
static BROTLI_INLINE void
|
120
|
+
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
|
122
121
|
memcpy(p, &v, sizeof v);
|
123
122
|
}
|
124
123
|
|
125
|
-
|
126
|
-
|
124
|
+
#else /* BROTLI_LITTLE_ENDIAN */
|
125
|
+
|
126
|
+
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
|
127
|
+
const uint8_t* in = (const uint8_t*)p;
|
128
|
+
uint64_t value = (uint64_t)(in[0]);
|
129
|
+
value |= (uint64_t)(in[1]) << 8;
|
130
|
+
value |= (uint64_t)(in[2]) << 16;
|
131
|
+
value |= (uint64_t)(in[3]) << 24;
|
132
|
+
value |= (uint64_t)(in[4]) << 32;
|
133
|
+
value |= (uint64_t)(in[5]) << 40;
|
134
|
+
value |= (uint64_t)(in[6]) << 48;
|
135
|
+
value |= (uint64_t)(in[7]) << 56;
|
136
|
+
return value;
|
127
137
|
}
|
128
138
|
|
129
|
-
|
139
|
+
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
|
140
|
+
uint8_t* out = (uint8_t*)p;
|
141
|
+
out[0] = (uint8_t)v;
|
142
|
+
out[1] = (uint8_t)(v >> 8);
|
143
|
+
out[2] = (uint8_t)(v >> 16);
|
144
|
+
out[3] = (uint8_t)(v >> 24);
|
145
|
+
out[4] = (uint8_t)(v >> 32);
|
146
|
+
out[5] = (uint8_t)(v >> 40);
|
147
|
+
out[6] = (uint8_t)(v >> 48);
|
148
|
+
out[7] = (uint8_t)(v >> 56);
|
149
|
+
}
|
150
|
+
|
151
|
+
#endif /* BROTLI_LITTLE_ENDIAN */
|
130
152
|
|
131
|
-
#if !defined(__cplusplus) && !defined(c_plusplus) && __STDC_VERSION__ >= 199901L
|
132
|
-
#define BROTLI_RESTRICT restrict
|
133
|
-
#elif BROTLI_GCC_VERSION > 295 || defined(__llvm__)
|
134
|
-
#define BROTLI_RESTRICT __restrict
|
135
|
-
#else
|
136
|
-
#define BROTLI_RESTRICT
|
137
153
|
#endif
|
138
154
|
|
139
|
-
#define
|
155
|
+
#define TEMPLATE_(T) \
|
140
156
|
static BROTLI_INLINE T brotli_min_ ## T (T a, T b) { return a < b ? a : b; } \
|
141
157
|
static BROTLI_INLINE T brotli_max_ ## T (T a, T b) { return a > b ? a : b; }
|
142
|
-
|
143
|
-
|
144
|
-
#undef
|
158
|
+
TEMPLATE_(double) TEMPLATE_(float) TEMPLATE_(int)
|
159
|
+
TEMPLATE_(size_t) TEMPLATE_(uint32_t) TEMPLATE_(uint8_t)
|
160
|
+
#undef TEMPLATE_
|
145
161
|
#define BROTLI_MIN(T, A, B) (brotli_min_ ## T((A), (B)))
|
146
162
|
#define BROTLI_MAX(T, A, B) (brotli_max_ ## T((A), (B)))
|
147
163
|
|
@@ -11,14 +11,16 @@
|
|
11
11
|
#define BROTLI_ENC_PREFIX_H_
|
12
12
|
|
13
13
|
#include "../common/constants.h"
|
14
|
-
#include
|
15
|
-
#include
|
14
|
+
#include <brotli/port.h>
|
15
|
+
#include <brotli/types.h>
|
16
16
|
#include "./fast_log.h"
|
17
17
|
|
18
18
|
#if defined(__cplusplus) || defined(c_plusplus)
|
19
19
|
extern "C" {
|
20
20
|
#endif
|
21
21
|
|
22
|
+
/* Here distance_code is an intermediate code, i.e. one of the special codes or
|
23
|
+
the actual distance increased by BROTLI_NUM_DISTANCE_SHORT_CODES - 1. */
|
22
24
|
static BROTLI_INLINE void PrefixEncodeCopyDistance(size_t distance_code,
|
23
25
|
size_t num_direct_codes,
|
24
26
|
size_t postfix_bits,
|
@@ -10,14 +10,14 @@
|
|
10
10
|
#ifndef BROTLI_ENC_QUALITY_H_
|
11
11
|
#define BROTLI_ENC_QUALITY_H_
|
12
12
|
|
13
|
-
#include
|
13
|
+
#include <brotli/encode.h>
|
14
14
|
|
15
15
|
#define FAST_ONE_PASS_COMPRESSION_QUALITY 0
|
16
16
|
#define FAST_TWO_PASS_COMPRESSION_QUALITY 1
|
17
17
|
#define ZOPFLIFICATION_QUALITY 10
|
18
18
|
#define HQ_ZOPFLIFICATION_QUALITY 11
|
19
19
|
|
20
|
-
#define
|
20
|
+
#define MAX_QUALITY_FOR_STATIC_ENTROPY_CODES 2
|
21
21
|
#define MIN_QUALITY_FOR_BLOCK_SPLIT 4
|
22
22
|
#define MIN_QUALITY_FOR_OPTIMIZE_HISTOGRAMS 4
|
23
23
|
#define MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH 5
|
@@ -31,15 +31,26 @@
|
|
31
31
|
so we buffer at most this much literals and commands. */
|
32
32
|
#define MAX_NUM_DELAYED_SYMBOLS 0x2fff
|
33
33
|
|
34
|
+
typedef struct BrotliHasherParams {
|
35
|
+
int type;
|
36
|
+
int bucket_bits;
|
37
|
+
int block_bits;
|
38
|
+
int hash_len;
|
39
|
+
int num_last_distances_to_check;
|
40
|
+
} BrotliHasherParams;
|
41
|
+
|
34
42
|
/* Encoding parameters */
|
35
43
|
typedef struct BrotliEncoderParams {
|
36
44
|
BrotliEncoderMode mode;
|
37
45
|
int quality;
|
38
46
|
int lgwin;
|
39
47
|
int lgblock;
|
48
|
+
size_t size_hint;
|
49
|
+
BROTLI_BOOL disable_literal_context_modeling;
|
50
|
+
BrotliHasherParams hasher;
|
40
51
|
} BrotliEncoderParams;
|
41
52
|
|
42
|
-
/* Returns
|
53
|
+
/* Returns hash-table size for quality levels 0 and 1. */
|
43
54
|
static BROTLI_INLINE size_t MaxHashTableSize(int quality) {
|
44
55
|
return quality == FAST_ONE_PASS_COMPRESSION_QUALITY ? 1 << 15 : 1 << 17;
|
45
56
|
}
|
@@ -48,13 +59,16 @@ static BROTLI_INLINE size_t MaxHashTableSize(int quality) {
|
|
48
59
|
#define MAX_ZOPFLI_LEN_QUALITY_10 150
|
49
60
|
#define MAX_ZOPFLI_LEN_QUALITY_11 325
|
50
61
|
|
62
|
+
/* Do not thoroughly search when a long copy is found. */
|
63
|
+
#define BROTLI_LONG_COPY_QUICK_STEP 16384
|
64
|
+
|
51
65
|
static BROTLI_INLINE size_t MaxZopfliLen(const BrotliEncoderParams* params) {
|
52
66
|
return params->quality <= 10 ?
|
53
67
|
MAX_ZOPFLI_LEN_QUALITY_10 :
|
54
68
|
MAX_ZOPFLI_LEN_QUALITY_11;
|
55
69
|
}
|
56
70
|
|
57
|
-
/* Number of best candidates to evaluate to expand
|
71
|
+
/* Number of best candidates to evaluate to expand Zopfli chain. */
|
58
72
|
static BROTLI_INLINE size_t MaxZopfliCandidates(
|
59
73
|
const BrotliEncoderParams* params) {
|
60
74
|
return params->quality <= 10 ? 1 : 5;
|
@@ -63,10 +77,10 @@ static BROTLI_INLINE size_t MaxZopfliCandidates(
|
|
63
77
|
static BROTLI_INLINE void SanitizeParams(BrotliEncoderParams* params) {
|
64
78
|
params->quality = BROTLI_MIN(int, BROTLI_MAX_QUALITY,
|
65
79
|
BROTLI_MAX(int, BROTLI_MIN_QUALITY, params->quality));
|
66
|
-
if (params->lgwin <
|
67
|
-
params->lgwin =
|
68
|
-
} else if (params->lgwin >
|
69
|
-
params->lgwin =
|
80
|
+
if (params->lgwin < BROTLI_MIN_WINDOW_BITS) {
|
81
|
+
params->lgwin = BROTLI_MIN_WINDOW_BITS;
|
82
|
+
} else if (params->lgwin > BROTLI_MAX_WINDOW_BITS) {
|
83
|
+
params->lgwin = BROTLI_MAX_WINDOW_BITS;
|
70
84
|
}
|
71
85
|
}
|
72
86
|
|
@@ -84,8 +98,8 @@ static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
|
|
84
98
|
lgblock = BROTLI_MIN(int, 18, params->lgwin);
|
85
99
|
}
|
86
100
|
} else {
|
87
|
-
lgblock = BROTLI_MIN(int,
|
88
|
-
BROTLI_MAX(int,
|
101
|
+
lgblock = BROTLI_MIN(int, BROTLI_MAX_INPUT_BLOCK_BITS,
|
102
|
+
BROTLI_MAX(int, BROTLI_MIN_INPUT_BLOCK_BITS, lgblock));
|
89
103
|
}
|
90
104
|
return lgblock;
|
91
105
|
}
|
@@ -94,14 +108,15 @@ static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
|
|
94
108
|
Allocate at least lgwin + 1 bits for the ring buffer so that the newly
|
95
109
|
added block fits there completely and we still get lgwin bits and at least
|
96
110
|
read_block_size_bits + 1 bits because the copy tail length needs to be
|
97
|
-
smaller than
|
111
|
+
smaller than ring-buffer size. */
|
98
112
|
static BROTLI_INLINE int ComputeRbBits(const BrotliEncoderParams* params) {
|
99
113
|
return 1 + BROTLI_MAX(int, params->lgwin, params->lgblock);
|
100
114
|
}
|
101
115
|
|
102
116
|
static BROTLI_INLINE size_t MaxMetablockSize(
|
103
117
|
const BrotliEncoderParams* params) {
|
104
|
-
int bits =
|
118
|
+
int bits =
|
119
|
+
BROTLI_MIN(int, ComputeRbBits(params), BROTLI_MAX_INPUT_BLOCK_BITS);
|
105
120
|
return (size_t)1 << bits;
|
106
121
|
}
|
107
122
|
|
@@ -116,15 +131,30 @@ static BROTLI_INLINE size_t LiteralSpreeLengthForSparseSearch(
|
|
116
131
|
return params->quality < 9 ? 64 : 512;
|
117
132
|
}
|
118
133
|
|
119
|
-
static BROTLI_INLINE
|
134
|
+
static BROTLI_INLINE void ChooseHasher(const BrotliEncoderParams* params,
|
135
|
+
BrotliHasherParams* hparams) {
|
120
136
|
if (params->quality > 9) {
|
121
|
-
|
137
|
+
hparams->type = 10;
|
138
|
+
} else if (params->quality == 4 && params->size_hint >= (1 << 20)) {
|
139
|
+
hparams->type = 54;
|
122
140
|
} else if (params->quality < 5) {
|
123
|
-
|
141
|
+
hparams->type = params->quality;
|
124
142
|
} else if (params->lgwin <= 16) {
|
125
|
-
|
143
|
+
hparams->type = params->quality < 7 ? 40 : params->quality < 9 ? 41 : 42;
|
144
|
+
} else if (params->size_hint >= (1 << 20) && params->lgwin >= 19) {
|
145
|
+
hparams->type = 6;
|
146
|
+
hparams->block_bits = params->quality - 1;
|
147
|
+
hparams->bucket_bits = 15;
|
148
|
+
hparams->hash_len = 5;
|
149
|
+
hparams->num_last_distances_to_check =
|
150
|
+
params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16;
|
151
|
+
} else {
|
152
|
+
hparams->type = 5;
|
153
|
+
hparams->block_bits = params->quality - 1;
|
154
|
+
hparams->bucket_bits = params->quality < 7 ? 14 : 15;
|
155
|
+
hparams->num_last_distances_to_check =
|
156
|
+
params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16;
|
126
157
|
}
|
127
|
-
return params->quality;
|
128
158
|
}
|
129
159
|
|
130
160
|
#endif /* BROTLI_ENC_QUALITY_H_ */
|
@@ -11,7 +11,7 @@
|
|
11
11
|
|
12
12
|
#include <string.h> /* memcpy */
|
13
13
|
|
14
|
-
#include
|
14
|
+
#include <brotli/types.h>
|
15
15
|
#include "./memory.h"
|
16
16
|
#include "./port.h"
|
17
17
|
#include "./quality.h"
|
@@ -30,7 +30,7 @@ extern "C" {
|
|
30
30
|
buffer_[-1] == buffer_[(1 << window_bits) - 1] and
|
31
31
|
buffer_[-2] == buffer_[(1 << window_bits) - 2]. */
|
32
32
|
typedef struct RingBuffer {
|
33
|
-
/* Size of the
|
33
|
+
/* Size of the ring-buffer is (1 << window_bits) + tail_size_. */
|
34
34
|
const uint32_t size_;
|
35
35
|
const uint32_t mask_;
|
36
36
|
const uint32_t tail_size_;
|
@@ -42,7 +42,7 @@ typedef struct RingBuffer {
|
|
42
42
|
/* The actual ring buffer containing the copy of the last two bytes, the data,
|
43
43
|
and the copy of the beginning as a tail. */
|
44
44
|
uint8_t *data_;
|
45
|
-
/* The start of the
|
45
|
+
/* The start of the ring-buffer. */
|
46
46
|
uint8_t *buffer_;
|
47
47
|
} RingBuffer;
|
48
48
|
|
@@ -93,7 +93,7 @@ static BROTLI_INLINE void RingBufferInitBuffer(
|
|
93
93
|
static BROTLI_INLINE void RingBufferWriteTail(
|
94
94
|
const uint8_t *bytes, size_t n, RingBuffer* rb) {
|
95
95
|
const size_t masked_pos = rb->pos_ & rb->mask_;
|
96
|
-
if (
|
96
|
+
if (BROTLI_PREDICT_FALSE(masked_pos < rb->tail_size_)) {
|
97
97
|
/* Just fill the tail buffer with the beginning data. */
|
98
98
|
const size_t p = rb->size_ + masked_pos;
|
99
99
|
memcpy(&rb->buffer_[p], bytes,
|
@@ -106,7 +106,7 @@ static BROTLI_INLINE void RingBufferWrite(
|
|
106
106
|
MemoryManager* m, const uint8_t *bytes, size_t n, RingBuffer* rb) {
|
107
107
|
if (rb->pos_ == 0 && n < rb->tail_size_) {
|
108
108
|
/* Special case for the first write: to process the first block, we don't
|
109
|
-
need to allocate the whole
|
109
|
+
need to allocate the whole ring-buffer and we don't need the tail
|
110
110
|
either. However, we do this memory usage optimization only if the
|
111
111
|
first write is less than the tail size, which is also the input block
|
112
112
|
size, otherwise it is likely that other blocks will follow and we
|
@@ -131,7 +131,7 @@ static BROTLI_INLINE void RingBufferWrite(
|
|
131
131
|
/* The length of the writes is limited so that we do not need to worry
|
132
132
|
about a write */
|
133
133
|
RingBufferWriteTail(bytes, n, rb);
|
134
|
-
if (
|
134
|
+
if (BROTLI_PREDICT_TRUE(masked_pos + n <= rb->size_)) {
|
135
135
|
/* A single write fits. */
|
136
136
|
memcpy(&rb->buffer_[masked_pos], bytes, n);
|
137
137
|
} else {
|
@@ -33,23 +33,24 @@ static BROTLI_INLINE void AddMatch(size_t distance, size_t len, size_t len_code,
|
|
33
33
|
matches[len] = BROTLI_MIN(uint32_t, matches[len], match);
|
34
34
|
}
|
35
35
|
|
36
|
-
static BROTLI_INLINE size_t DictMatchLength(const
|
36
|
+
static BROTLI_INLINE size_t DictMatchLength(const BrotliDictionary* dictionary,
|
37
|
+
const uint8_t* data,
|
37
38
|
size_t id,
|
38
39
|
size_t len,
|
39
40
|
size_t maxlen) {
|
40
|
-
const size_t offset =
|
41
|
-
return FindMatchLengthWithLimit(&
|
41
|
+
const size_t offset = dictionary->offsets_by_length[len] + len * id;
|
42
|
+
return FindMatchLengthWithLimit(&dictionary->data[offset], data,
|
42
43
|
BROTLI_MIN(size_t, len, maxlen));
|
43
44
|
}
|
44
45
|
|
45
|
-
static BROTLI_INLINE BROTLI_BOOL IsMatch(
|
46
|
+
static BROTLI_INLINE BROTLI_BOOL IsMatch(const BrotliDictionary* dictionary,
|
46
47
|
DictWord w, const uint8_t* data, size_t max_length) {
|
47
48
|
if (w.len > max_length) {
|
48
49
|
return BROTLI_FALSE;
|
49
50
|
} else {
|
50
|
-
const size_t offset =
|
51
|
+
const size_t offset = dictionary->offsets_by_length[w.len] +
|
51
52
|
(size_t)w.len * (size_t)w.idx;
|
52
|
-
const uint8_t* dict = &
|
53
|
+
const uint8_t* dict = &dictionary->data[offset];
|
53
54
|
if (w.transform == 0) {
|
54
55
|
/* Match against base dictionary word. */
|
55
56
|
return
|
@@ -78,21 +79,22 @@ static BROTLI_INLINE BROTLI_BOOL IsMatch(
|
|
78
79
|
}
|
79
80
|
|
80
81
|
BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
81
|
-
const uint8_t* data, size_t min_length,
|
82
|
-
uint32_t* matches) {
|
82
|
+
const BrotliDictionary* dictionary, const uint8_t* data, size_t min_length,
|
83
|
+
size_t max_length, uint32_t* matches) {
|
83
84
|
BROTLI_BOOL has_found_match = BROTLI_FALSE;
|
84
85
|
{
|
85
86
|
size_t offset = kStaticDictionaryBuckets[Hash(data)];
|
86
87
|
BROTLI_BOOL end = !offset;
|
87
88
|
while (!end) {
|
88
89
|
DictWord w = kStaticDictionaryWords[offset++];
|
89
|
-
const size_t l = w.len &
|
90
|
-
const size_t n = (size_t)1 <<
|
90
|
+
const size_t l = w.len & 0x1F;
|
91
|
+
const size_t n = (size_t)1 << dictionary->size_bits_by_length[l];
|
91
92
|
const size_t id = w.idx;
|
92
93
|
end = !!(w.len & 0x80);
|
93
94
|
w.len = (uint8_t)l;
|
94
95
|
if (w.transform == 0) {
|
95
|
-
const size_t matchlen =
|
96
|
+
const size_t matchlen =
|
97
|
+
DictMatchLength(dictionary, data, id, l, max_length);
|
96
98
|
const uint8_t* s;
|
97
99
|
size_t minlen;
|
98
100
|
size_t maxlen;
|
@@ -276,7 +278,7 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
276
278
|
const BROTLI_BOOL is_all_caps =
|
277
279
|
TO_BROTLI_BOOL(w.transform != kUppercaseFirst);
|
278
280
|
const uint8_t* s;
|
279
|
-
if (!IsMatch(w, data, max_length)) {
|
281
|
+
if (!IsMatch(dictionary, w, data, max_length)) {
|
280
282
|
continue;
|
281
283
|
}
|
282
284
|
/* Transform "" + kUppercase{First,All} + "" */
|
@@ -325,14 +327,14 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
325
327
|
BROTLI_BOOL end = !offset;
|
326
328
|
while (!end) {
|
327
329
|
DictWord w = kStaticDictionaryWords[offset++];
|
328
|
-
const size_t l = w.len &
|
329
|
-
const size_t n = (size_t)1 <<
|
330
|
+
const size_t l = w.len & 0x1F;
|
331
|
+
const size_t n = (size_t)1 << dictionary->size_bits_by_length[l];
|
330
332
|
const size_t id = w.idx;
|
331
333
|
end = !!(w.len & 0x80);
|
332
334
|
w.len = (uint8_t)l;
|
333
335
|
if (w.transform == 0) {
|
334
336
|
const uint8_t* s;
|
335
|
-
if (!IsMatch(w, &data[1], max_length - 1)) {
|
337
|
+
if (!IsMatch(dictionary, w, &data[1], max_length - 1)) {
|
336
338
|
continue;
|
337
339
|
}
|
338
340
|
/* Transforms " " + kIdentity + "" and "." + kIdentity + "" */
|
@@ -373,7 +375,7 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
373
375
|
const BROTLI_BOOL is_all_caps =
|
374
376
|
TO_BROTLI_BOOL(w.transform != kUppercaseFirst);
|
375
377
|
const uint8_t* s;
|
376
|
-
if (!IsMatch(w, &data[1], max_length - 1)) {
|
378
|
+
if (!IsMatch(dictionary, w, &data[1], max_length - 1)) {
|
377
379
|
continue;
|
378
380
|
}
|
379
381
|
/* Transforms " " + kUppercase{First,All} + "" */
|
@@ -417,12 +419,13 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
417
419
|
BROTLI_BOOL end = !offset;
|
418
420
|
while (!end) {
|
419
421
|
DictWord w = kStaticDictionaryWords[offset++];
|
420
|
-
const size_t l = w.len &
|
421
|
-
const size_t n = (size_t)1 <<
|
422
|
+
const size_t l = w.len & 0x1F;
|
423
|
+
const size_t n = (size_t)1 << dictionary->size_bits_by_length[l];
|
422
424
|
const size_t id = w.idx;
|
423
425
|
end = !!(w.len & 0x80);
|
424
426
|
w.len = (uint8_t)l;
|
425
|
-
if (w.transform == 0 &&
|
427
|
+
if (w.transform == 0 &&
|
428
|
+
IsMatch(dictionary, w, &data[2], max_length - 2)) {
|
426
429
|
if (data[0] == 0xc2) {
|
427
430
|
AddMatch(id + 102 * n, l + 2, l, matches);
|
428
431
|
has_found_match = BROTLI_TRUE;
|
@@ -445,12 +448,13 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
445
448
|
BROTLI_BOOL end = !offset;
|
446
449
|
while (!end) {
|
447
450
|
DictWord w = kStaticDictionaryWords[offset++];
|
448
|
-
const size_t l = w.len &
|
449
|
-
const size_t n = (size_t)1 <<
|
451
|
+
const size_t l = w.len & 0x1F;
|
452
|
+
const size_t n = (size_t)1 << dictionary->size_bits_by_length[l];
|
450
453
|
const size_t id = w.idx;
|
451
454
|
end = !!(w.len & 0x80);
|
452
455
|
w.len = (uint8_t)l;
|
453
|
-
if (w.transform == 0 &&
|
456
|
+
if (w.transform == 0 &&
|
457
|
+
IsMatch(dictionary, w, &data[5], max_length - 5)) {
|
454
458
|
AddMatch(id + (data[0] == ' ' ? 41 : 72) * n, l + 5, l, matches);
|
455
459
|
has_found_match = BROTLI_TRUE;
|
456
460
|
if (l + 5 < max_length) {
|