brotli 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +7 -3
- data/brotli.gemspec +1 -1
- data/ext/brotli/brotli.c +4 -4
- data/ext/brotli/brotli.h +2 -2
- data/ext/brotli/extconf.rb +9 -16
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/{common → c/common}/constants.h +11 -1
- data/vendor/brotli/c/common/dictionary.bin +432 -0
- data/vendor/brotli/c/common/dictionary.c +5905 -0
- data/vendor/brotli/c/common/dictionary.h +64 -0
- data/vendor/brotli/c/common/version.h +19 -0
- data/vendor/brotli/{dec → c/dec}/bit_reader.c +2 -2
- data/vendor/brotli/{dec → c/dec}/bit_reader.h +11 -34
- data/vendor/brotli/{dec → c/dec}/context.h +1 -1
- data/vendor/brotli/{dec → c/dec}/decode.c +389 -356
- data/vendor/brotli/{dec → c/dec}/huffman.c +24 -23
- data/vendor/brotli/{dec → c/dec}/huffman.h +1 -1
- data/vendor/brotli/{dec → c/dec}/port.h +19 -10
- data/vendor/brotli/{dec → c/dec}/prefix.h +1 -1
- data/vendor/brotli/{dec → c/dec}/state.c +23 -19
- data/vendor/brotli/{dec → c/dec}/state.h +18 -17
- data/vendor/brotli/{dec → c/dec}/transform.h +2 -2
- data/vendor/brotli/c/enc/backward_references.c +134 -0
- data/vendor/brotli/c/enc/backward_references.h +39 -0
- data/vendor/brotli/{enc/backward_references.c → c/enc/backward_references_hq.c} +144 -232
- data/vendor/brotli/{enc/backward_references.h → c/enc/backward_references_hq.h} +28 -31
- data/vendor/brotli/{enc → c/enc}/backward_references_inc.h +37 -31
- data/vendor/brotli/{enc → c/enc}/bit_cost.c +1 -1
- data/vendor/brotli/{enc → c/enc}/bit_cost.h +1 -1
- data/vendor/brotli/{enc → c/enc}/bit_cost_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/block_encoder_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/block_splitter.c +2 -4
- data/vendor/brotli/{enc → c/enc}/block_splitter.h +1 -1
- data/vendor/brotli/{enc → c/enc}/block_splitter_inc.h +6 -7
- data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.c +22 -26
- data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.h +1 -5
- data/vendor/brotli/{enc → c/enc}/cluster.c +1 -1
- data/vendor/brotli/{enc → c/enc}/cluster.h +1 -1
- data/vendor/brotli/{enc → c/enc}/cluster_inc.h +2 -0
- data/vendor/brotli/{enc → c/enc}/command.h +34 -17
- data/vendor/brotli/{enc → c/enc}/compress_fragment.c +97 -53
- data/vendor/brotli/{enc → c/enc}/compress_fragment.h +5 -2
- data/vendor/brotli/{enc → c/enc}/compress_fragment_two_pass.c +106 -51
- data/vendor/brotli/{enc → c/enc}/compress_fragment_two_pass.h +5 -2
- data/vendor/brotli/{enc → c/enc}/context.h +3 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +1120 -0
- data/vendor/brotli/c/enc/dictionary_hash.h +24 -0
- data/vendor/brotli/{enc → c/enc}/encode.c +442 -240
- data/vendor/brotli/{enc → c/enc}/entropy_encode.c +9 -9
- data/vendor/brotli/{enc → c/enc}/entropy_encode.h +4 -4
- data/vendor/brotli/{enc → c/enc}/entropy_encode_static.h +4 -4
- data/vendor/brotli/{enc → c/enc}/fast_log.h +3 -3
- data/vendor/brotli/{enc → c/enc}/find_match_length.h +8 -8
- data/vendor/brotli/c/enc/hash.h +446 -0
- data/vendor/brotli/{enc → c/enc}/hash_forgetful_chain_inc.h +72 -68
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +266 -0
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +258 -0
- data/vendor/brotli/{enc → c/enc}/hash_longest_match_quickly_inc.h +81 -77
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +326 -0
- data/vendor/brotli/{enc → c/enc}/histogram.c +4 -2
- data/vendor/brotli/{enc → c/enc}/histogram.h +1 -1
- data/vendor/brotli/{enc → c/enc}/histogram_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/literal_cost.c +4 -7
- data/vendor/brotli/{enc → c/enc}/literal_cost.h +2 -2
- data/vendor/brotli/{enc → c/enc}/memory.c +1 -1
- data/vendor/brotli/{enc → c/enc}/memory.h +3 -2
- data/vendor/brotli/{enc → c/enc}/metablock.c +136 -123
- data/vendor/brotli/{enc → c/enc}/metablock.h +2 -12
- data/vendor/brotli/{enc → c/enc}/metablock_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/port.h +49 -33
- data/vendor/brotli/{enc → c/enc}/prefix.h +4 -2
- data/vendor/brotli/{enc → c/enc}/quality.h +47 -17
- data/vendor/brotli/{enc → c/enc}/ringbuffer.h +6 -6
- data/vendor/brotli/{enc → c/enc}/static_dict.c +26 -22
- data/vendor/brotli/{enc → c/enc}/static_dict.h +3 -1
- data/vendor/brotli/c/enc/static_dict_lut.h +5864 -0
- data/vendor/brotli/{enc → c/enc}/utf8_util.c +1 -1
- data/vendor/brotli/{enc → c/enc}/utf8_util.h +2 -2
- data/vendor/brotli/{enc → c/enc}/write_bits.h +3 -3
- data/vendor/brotli/c/include/brotli/decode.h +339 -0
- data/vendor/brotli/c/include/brotli/encode.h +402 -0
- data/vendor/brotli/c/include/brotli/port.h +146 -0
- data/vendor/brotli/c/include/brotli/types.h +90 -0
- metadata +80 -79
- data/vendor/brotli/common/dictionary.c +0 -9474
- data/vendor/brotli/common/dictionary.h +0 -29
- data/vendor/brotli/common/port.h +0 -107
- data/vendor/brotli/common/types.h +0 -58
- data/vendor/brotli/dec/decode.h +0 -188
- data/vendor/brotli/enc/compressor.cc +0 -139
- data/vendor/brotli/enc/compressor.h +0 -161
- data/vendor/brotli/enc/dictionary_hash.h +0 -4121
- data/vendor/brotli/enc/encode.h +0 -221
- data/vendor/brotli/enc/encode_parallel.cc +0 -289
- data/vendor/brotli/enc/encode_parallel.h +0 -27
- data/vendor/brotli/enc/hash.h +0 -717
- data/vendor/brotli/enc/hash_longest_match_inc.h +0 -241
- data/vendor/brotli/enc/static_dict_lut.h +0 -11241
- data/vendor/brotli/enc/streams.cc +0 -114
- data/vendor/brotli/enc/streams.h +0 -121
@@ -10,7 +10,7 @@
|
|
10
10
|
#ifndef BROTLI_ENC_METABLOCK_H_
|
11
11
|
#define BROTLI_ENC_METABLOCK_H_
|
12
12
|
|
13
|
-
#include
|
13
|
+
#include <brotli/types.h>
|
14
14
|
#include "./block_splitter.h"
|
15
15
|
#include "./command.h"
|
16
16
|
#include "./context.h"
|
@@ -80,20 +80,10 @@ BROTLI_INTERNAL void BrotliBuildMetaBlock(MemoryManager* m,
|
|
80
80
|
ContextType literal_context_mode,
|
81
81
|
MetaBlockSplit* mb);
|
82
82
|
|
83
|
-
/* Uses a fast greedy block splitter that tries to merge current block with the
|
84
|
-
last or the second last block and does not do any context modeling. */
|
85
|
-
BROTLI_INTERNAL void BrotliBuildMetaBlockGreedy(MemoryManager* m,
|
86
|
-
const uint8_t* ringbuffer,
|
87
|
-
size_t pos,
|
88
|
-
size_t mask,
|
89
|
-
const Command* commands,
|
90
|
-
size_t n_commands,
|
91
|
-
MetaBlockSplit* mb);
|
92
|
-
|
93
83
|
/* Uses a fast greedy block splitter that tries to merge current block with the
|
94
84
|
last or the second last block and uses a static context clustering which
|
95
85
|
is the same for all block types. */
|
96
|
-
BROTLI_INTERNAL void
|
86
|
+
BROTLI_INTERNAL void BrotliBuildMetaBlockGreedy(
|
97
87
|
MemoryManager* m, const uint8_t* ringbuffer, size_t pos, size_t mask,
|
98
88
|
uint8_t prev_byte, uint8_t prev_byte2, ContextType literal_context_mode,
|
99
89
|
size_t num_contexts, const uint32_t* static_context_map,
|
File without changes
|
@@ -12,8 +12,8 @@
|
|
12
12
|
#include <assert.h>
|
13
13
|
#include <string.h> /* memcpy */
|
14
14
|
|
15
|
-
#include
|
16
|
-
#include
|
15
|
+
#include <brotli/port.h>
|
16
|
+
#include <brotli/types.h>
|
17
17
|
|
18
18
|
#if defined OS_LINUX || defined OS_CYGWIN
|
19
19
|
#include <endian.h>
|
@@ -26,36 +26,37 @@
|
|
26
26
|
#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
27
27
|
#endif
|
28
28
|
|
29
|
-
/* define the macro
|
29
|
+
/* define the macro BROTLI_LITTLE_ENDIAN
|
30
30
|
using the above endian definitions from endian.h if
|
31
31
|
endian.h was included */
|
32
32
|
#ifdef __BYTE_ORDER
|
33
33
|
#if __BYTE_ORDER == __LITTLE_ENDIAN
|
34
|
-
#define
|
34
|
+
#define BROTLI_LITTLE_ENDIAN
|
35
35
|
#endif
|
36
36
|
|
37
37
|
#else
|
38
38
|
|
39
39
|
#if defined(__LITTLE_ENDIAN__)
|
40
|
-
#define
|
40
|
+
#define BROTLI_LITTLE_ENDIAN
|
41
41
|
#endif
|
42
42
|
#endif /* __BYTE_ORDER */
|
43
43
|
|
44
44
|
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
45
|
-
#define
|
45
|
+
#define BROTLI_LITTLE_ENDIAN
|
46
46
|
#endif
|
47
47
|
|
48
48
|
/* Enable little-endian optimization for x64 architecture on Windows. */
|
49
49
|
#if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
|
50
|
-
#define
|
50
|
+
#define BROTLI_LITTLE_ENDIAN
|
51
51
|
#endif
|
52
52
|
|
53
53
|
/* Portable handling of unaligned loads, stores, and copies.
|
54
54
|
On some platforms, like ARM, the copy functions can be more efficient
|
55
55
|
then a load and a store. */
|
56
56
|
|
57
|
-
#if defined(
|
58
|
-
|
57
|
+
#if defined(BROTLI_LITTLE_ENDIAN) && (\
|
58
|
+
defined(ARCH_PIII) || defined(ARCH_ATHLON) || \
|
59
|
+
defined(ARCH_K8) || defined(_ARCH_PPC))
|
59
60
|
|
60
61
|
/* x86 and x86-64 can perform unaligned loads/stores directly;
|
61
62
|
modern PowerPC hardware can also do unaligned integer loads and stores;
|
@@ -63,14 +64,12 @@
|
|
63
64
|
*/
|
64
65
|
|
65
66
|
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
|
66
|
-
#define
|
67
|
+
#define BROTLI_UNALIGNED_LOAD64LE(_p) (*(const uint64_t *)(_p))
|
67
68
|
|
68
|
-
#define
|
69
|
-
(*(uint32_t *)(_p) = (_val))
|
70
|
-
#define BROTLI_UNALIGNED_STORE64(_p, _val) \
|
69
|
+
#define BROTLI_UNALIGNED_STORE64LE(_p, _val) \
|
71
70
|
(*(uint64_t *)(_p) = (_val))
|
72
71
|
|
73
|
-
#elif defined(__arm__) && \
|
72
|
+
#elif defined(BROTLI_LITTLE_ENDIAN) && defined(__arm__) && \
|
74
73
|
!defined(__ARM_ARCH_5__) && \
|
75
74
|
!defined(__ARM_ARCH_5T__) && \
|
76
75
|
!defined(__ARM_ARCH_5TE__) && \
|
@@ -88,16 +87,14 @@
|
|
88
87
|
slowly (trip through kernel mode). */
|
89
88
|
|
90
89
|
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
|
91
|
-
#define BROTLI_UNALIGNED_STORE32(_p, _val) \
|
92
|
-
(*(uint32_t *)(_p) = (_val))
|
93
90
|
|
94
|
-
static BROTLI_INLINE uint64_t
|
91
|
+
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
|
95
92
|
uint64_t t;
|
96
93
|
memcpy(&t, p, sizeof t);
|
97
94
|
return t;
|
98
95
|
}
|
99
96
|
|
100
|
-
static BROTLI_INLINE void
|
97
|
+
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
|
101
98
|
memcpy(p, &v, sizeof v);
|
102
99
|
}
|
103
100
|
|
@@ -112,36 +109,55 @@ static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) {
|
|
112
109
|
return t;
|
113
110
|
}
|
114
111
|
|
115
|
-
|
112
|
+
#if defined(BROTLI_LITTLE_ENDIAN)
|
113
|
+
|
114
|
+
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
|
116
115
|
uint64_t t;
|
117
116
|
memcpy(&t, p, sizeof t);
|
118
117
|
return t;
|
119
118
|
}
|
120
119
|
|
121
|
-
static BROTLI_INLINE void
|
120
|
+
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
|
122
121
|
memcpy(p, &v, sizeof v);
|
123
122
|
}
|
124
123
|
|
125
|
-
|
126
|
-
|
124
|
+
#else /* BROTLI_LITTLE_ENDIAN */
|
125
|
+
|
126
|
+
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
|
127
|
+
const uint8_t* in = (const uint8_t*)p;
|
128
|
+
uint64_t value = (uint64_t)(in[0]);
|
129
|
+
value |= (uint64_t)(in[1]) << 8;
|
130
|
+
value |= (uint64_t)(in[2]) << 16;
|
131
|
+
value |= (uint64_t)(in[3]) << 24;
|
132
|
+
value |= (uint64_t)(in[4]) << 32;
|
133
|
+
value |= (uint64_t)(in[5]) << 40;
|
134
|
+
value |= (uint64_t)(in[6]) << 48;
|
135
|
+
value |= (uint64_t)(in[7]) << 56;
|
136
|
+
return value;
|
127
137
|
}
|
128
138
|
|
129
|
-
|
139
|
+
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
|
140
|
+
uint8_t* out = (uint8_t*)p;
|
141
|
+
out[0] = (uint8_t)v;
|
142
|
+
out[1] = (uint8_t)(v >> 8);
|
143
|
+
out[2] = (uint8_t)(v >> 16);
|
144
|
+
out[3] = (uint8_t)(v >> 24);
|
145
|
+
out[4] = (uint8_t)(v >> 32);
|
146
|
+
out[5] = (uint8_t)(v >> 40);
|
147
|
+
out[6] = (uint8_t)(v >> 48);
|
148
|
+
out[7] = (uint8_t)(v >> 56);
|
149
|
+
}
|
150
|
+
|
151
|
+
#endif /* BROTLI_LITTLE_ENDIAN */
|
130
152
|
|
131
|
-
#if !defined(__cplusplus) && !defined(c_plusplus) && __STDC_VERSION__ >= 199901L
|
132
|
-
#define BROTLI_RESTRICT restrict
|
133
|
-
#elif BROTLI_GCC_VERSION > 295 || defined(__llvm__)
|
134
|
-
#define BROTLI_RESTRICT __restrict
|
135
|
-
#else
|
136
|
-
#define BROTLI_RESTRICT
|
137
153
|
#endif
|
138
154
|
|
139
|
-
#define
|
155
|
+
#define TEMPLATE_(T) \
|
140
156
|
static BROTLI_INLINE T brotli_min_ ## T (T a, T b) { return a < b ? a : b; } \
|
141
157
|
static BROTLI_INLINE T brotli_max_ ## T (T a, T b) { return a > b ? a : b; }
|
142
|
-
|
143
|
-
|
144
|
-
#undef
|
158
|
+
TEMPLATE_(double) TEMPLATE_(float) TEMPLATE_(int)
|
159
|
+
TEMPLATE_(size_t) TEMPLATE_(uint32_t) TEMPLATE_(uint8_t)
|
160
|
+
#undef TEMPLATE_
|
145
161
|
#define BROTLI_MIN(T, A, B) (brotli_min_ ## T((A), (B)))
|
146
162
|
#define BROTLI_MAX(T, A, B) (brotli_max_ ## T((A), (B)))
|
147
163
|
|
@@ -11,14 +11,16 @@
|
|
11
11
|
#define BROTLI_ENC_PREFIX_H_
|
12
12
|
|
13
13
|
#include "../common/constants.h"
|
14
|
-
#include
|
15
|
-
#include
|
14
|
+
#include <brotli/port.h>
|
15
|
+
#include <brotli/types.h>
|
16
16
|
#include "./fast_log.h"
|
17
17
|
|
18
18
|
#if defined(__cplusplus) || defined(c_plusplus)
|
19
19
|
extern "C" {
|
20
20
|
#endif
|
21
21
|
|
22
|
+
/* Here distance_code is an intermediate code, i.e. one of the special codes or
|
23
|
+
the actual distance increased by BROTLI_NUM_DISTANCE_SHORT_CODES - 1. */
|
22
24
|
static BROTLI_INLINE void PrefixEncodeCopyDistance(size_t distance_code,
|
23
25
|
size_t num_direct_codes,
|
24
26
|
size_t postfix_bits,
|
@@ -10,14 +10,14 @@
|
|
10
10
|
#ifndef BROTLI_ENC_QUALITY_H_
|
11
11
|
#define BROTLI_ENC_QUALITY_H_
|
12
12
|
|
13
|
-
#include
|
13
|
+
#include <brotli/encode.h>
|
14
14
|
|
15
15
|
#define FAST_ONE_PASS_COMPRESSION_QUALITY 0
|
16
16
|
#define FAST_TWO_PASS_COMPRESSION_QUALITY 1
|
17
17
|
#define ZOPFLIFICATION_QUALITY 10
|
18
18
|
#define HQ_ZOPFLIFICATION_QUALITY 11
|
19
19
|
|
20
|
-
#define
|
20
|
+
#define MAX_QUALITY_FOR_STATIC_ENTROPY_CODES 2
|
21
21
|
#define MIN_QUALITY_FOR_BLOCK_SPLIT 4
|
22
22
|
#define MIN_QUALITY_FOR_OPTIMIZE_HISTOGRAMS 4
|
23
23
|
#define MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH 5
|
@@ -31,15 +31,26 @@
|
|
31
31
|
so we buffer at most this much literals and commands. */
|
32
32
|
#define MAX_NUM_DELAYED_SYMBOLS 0x2fff
|
33
33
|
|
34
|
+
typedef struct BrotliHasherParams {
|
35
|
+
int type;
|
36
|
+
int bucket_bits;
|
37
|
+
int block_bits;
|
38
|
+
int hash_len;
|
39
|
+
int num_last_distances_to_check;
|
40
|
+
} BrotliHasherParams;
|
41
|
+
|
34
42
|
/* Encoding parameters */
|
35
43
|
typedef struct BrotliEncoderParams {
|
36
44
|
BrotliEncoderMode mode;
|
37
45
|
int quality;
|
38
46
|
int lgwin;
|
39
47
|
int lgblock;
|
48
|
+
size_t size_hint;
|
49
|
+
BROTLI_BOOL disable_literal_context_modeling;
|
50
|
+
BrotliHasherParams hasher;
|
40
51
|
} BrotliEncoderParams;
|
41
52
|
|
42
|
-
/* Returns
|
53
|
+
/* Returns hash-table size for quality levels 0 and 1. */
|
43
54
|
static BROTLI_INLINE size_t MaxHashTableSize(int quality) {
|
44
55
|
return quality == FAST_ONE_PASS_COMPRESSION_QUALITY ? 1 << 15 : 1 << 17;
|
45
56
|
}
|
@@ -48,13 +59,16 @@ static BROTLI_INLINE size_t MaxHashTableSize(int quality) {
|
|
48
59
|
#define MAX_ZOPFLI_LEN_QUALITY_10 150
|
49
60
|
#define MAX_ZOPFLI_LEN_QUALITY_11 325
|
50
61
|
|
62
|
+
/* Do not thoroughly search when a long copy is found. */
|
63
|
+
#define BROTLI_LONG_COPY_QUICK_STEP 16384
|
64
|
+
|
51
65
|
static BROTLI_INLINE size_t MaxZopfliLen(const BrotliEncoderParams* params) {
|
52
66
|
return params->quality <= 10 ?
|
53
67
|
MAX_ZOPFLI_LEN_QUALITY_10 :
|
54
68
|
MAX_ZOPFLI_LEN_QUALITY_11;
|
55
69
|
}
|
56
70
|
|
57
|
-
/* Number of best candidates to evaluate to expand
|
71
|
+
/* Number of best candidates to evaluate to expand Zopfli chain. */
|
58
72
|
static BROTLI_INLINE size_t MaxZopfliCandidates(
|
59
73
|
const BrotliEncoderParams* params) {
|
60
74
|
return params->quality <= 10 ? 1 : 5;
|
@@ -63,10 +77,10 @@ static BROTLI_INLINE size_t MaxZopfliCandidates(
|
|
63
77
|
static BROTLI_INLINE void SanitizeParams(BrotliEncoderParams* params) {
|
64
78
|
params->quality = BROTLI_MIN(int, BROTLI_MAX_QUALITY,
|
65
79
|
BROTLI_MAX(int, BROTLI_MIN_QUALITY, params->quality));
|
66
|
-
if (params->lgwin <
|
67
|
-
params->lgwin =
|
68
|
-
} else if (params->lgwin >
|
69
|
-
params->lgwin =
|
80
|
+
if (params->lgwin < BROTLI_MIN_WINDOW_BITS) {
|
81
|
+
params->lgwin = BROTLI_MIN_WINDOW_BITS;
|
82
|
+
} else if (params->lgwin > BROTLI_MAX_WINDOW_BITS) {
|
83
|
+
params->lgwin = BROTLI_MAX_WINDOW_BITS;
|
70
84
|
}
|
71
85
|
}
|
72
86
|
|
@@ -84,8 +98,8 @@ static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
|
|
84
98
|
lgblock = BROTLI_MIN(int, 18, params->lgwin);
|
85
99
|
}
|
86
100
|
} else {
|
87
|
-
lgblock = BROTLI_MIN(int,
|
88
|
-
BROTLI_MAX(int,
|
101
|
+
lgblock = BROTLI_MIN(int, BROTLI_MAX_INPUT_BLOCK_BITS,
|
102
|
+
BROTLI_MAX(int, BROTLI_MIN_INPUT_BLOCK_BITS, lgblock));
|
89
103
|
}
|
90
104
|
return lgblock;
|
91
105
|
}
|
@@ -94,14 +108,15 @@ static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
|
|
94
108
|
Allocate at least lgwin + 1 bits for the ring buffer so that the newly
|
95
109
|
added block fits there completely and we still get lgwin bits and at least
|
96
110
|
read_block_size_bits + 1 bits because the copy tail length needs to be
|
97
|
-
smaller than
|
111
|
+
smaller than ring-buffer size. */
|
98
112
|
static BROTLI_INLINE int ComputeRbBits(const BrotliEncoderParams* params) {
|
99
113
|
return 1 + BROTLI_MAX(int, params->lgwin, params->lgblock);
|
100
114
|
}
|
101
115
|
|
102
116
|
static BROTLI_INLINE size_t MaxMetablockSize(
|
103
117
|
const BrotliEncoderParams* params) {
|
104
|
-
int bits =
|
118
|
+
int bits =
|
119
|
+
BROTLI_MIN(int, ComputeRbBits(params), BROTLI_MAX_INPUT_BLOCK_BITS);
|
105
120
|
return (size_t)1 << bits;
|
106
121
|
}
|
107
122
|
|
@@ -116,15 +131,30 @@ static BROTLI_INLINE size_t LiteralSpreeLengthForSparseSearch(
|
|
116
131
|
return params->quality < 9 ? 64 : 512;
|
117
132
|
}
|
118
133
|
|
119
|
-
static BROTLI_INLINE
|
134
|
+
static BROTLI_INLINE void ChooseHasher(const BrotliEncoderParams* params,
|
135
|
+
BrotliHasherParams* hparams) {
|
120
136
|
if (params->quality > 9) {
|
121
|
-
|
137
|
+
hparams->type = 10;
|
138
|
+
} else if (params->quality == 4 && params->size_hint >= (1 << 20)) {
|
139
|
+
hparams->type = 54;
|
122
140
|
} else if (params->quality < 5) {
|
123
|
-
|
141
|
+
hparams->type = params->quality;
|
124
142
|
} else if (params->lgwin <= 16) {
|
125
|
-
|
143
|
+
hparams->type = params->quality < 7 ? 40 : params->quality < 9 ? 41 : 42;
|
144
|
+
} else if (params->size_hint >= (1 << 20) && params->lgwin >= 19) {
|
145
|
+
hparams->type = 6;
|
146
|
+
hparams->block_bits = params->quality - 1;
|
147
|
+
hparams->bucket_bits = 15;
|
148
|
+
hparams->hash_len = 5;
|
149
|
+
hparams->num_last_distances_to_check =
|
150
|
+
params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16;
|
151
|
+
} else {
|
152
|
+
hparams->type = 5;
|
153
|
+
hparams->block_bits = params->quality - 1;
|
154
|
+
hparams->bucket_bits = params->quality < 7 ? 14 : 15;
|
155
|
+
hparams->num_last_distances_to_check =
|
156
|
+
params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16;
|
126
157
|
}
|
127
|
-
return params->quality;
|
128
158
|
}
|
129
159
|
|
130
160
|
#endif /* BROTLI_ENC_QUALITY_H_ */
|
@@ -11,7 +11,7 @@
|
|
11
11
|
|
12
12
|
#include <string.h> /* memcpy */
|
13
13
|
|
14
|
-
#include
|
14
|
+
#include <brotli/types.h>
|
15
15
|
#include "./memory.h"
|
16
16
|
#include "./port.h"
|
17
17
|
#include "./quality.h"
|
@@ -30,7 +30,7 @@ extern "C" {
|
|
30
30
|
buffer_[-1] == buffer_[(1 << window_bits) - 1] and
|
31
31
|
buffer_[-2] == buffer_[(1 << window_bits) - 2]. */
|
32
32
|
typedef struct RingBuffer {
|
33
|
-
/* Size of the
|
33
|
+
/* Size of the ring-buffer is (1 << window_bits) + tail_size_. */
|
34
34
|
const uint32_t size_;
|
35
35
|
const uint32_t mask_;
|
36
36
|
const uint32_t tail_size_;
|
@@ -42,7 +42,7 @@ typedef struct RingBuffer {
|
|
42
42
|
/* The actual ring buffer containing the copy of the last two bytes, the data,
|
43
43
|
and the copy of the beginning as a tail. */
|
44
44
|
uint8_t *data_;
|
45
|
-
/* The start of the
|
45
|
+
/* The start of the ring-buffer. */
|
46
46
|
uint8_t *buffer_;
|
47
47
|
} RingBuffer;
|
48
48
|
|
@@ -93,7 +93,7 @@ static BROTLI_INLINE void RingBufferInitBuffer(
|
|
93
93
|
static BROTLI_INLINE void RingBufferWriteTail(
|
94
94
|
const uint8_t *bytes, size_t n, RingBuffer* rb) {
|
95
95
|
const size_t masked_pos = rb->pos_ & rb->mask_;
|
96
|
-
if (
|
96
|
+
if (BROTLI_PREDICT_FALSE(masked_pos < rb->tail_size_)) {
|
97
97
|
/* Just fill the tail buffer with the beginning data. */
|
98
98
|
const size_t p = rb->size_ + masked_pos;
|
99
99
|
memcpy(&rb->buffer_[p], bytes,
|
@@ -106,7 +106,7 @@ static BROTLI_INLINE void RingBufferWrite(
|
|
106
106
|
MemoryManager* m, const uint8_t *bytes, size_t n, RingBuffer* rb) {
|
107
107
|
if (rb->pos_ == 0 && n < rb->tail_size_) {
|
108
108
|
/* Special case for the first write: to process the first block, we don't
|
109
|
-
need to allocate the whole
|
109
|
+
need to allocate the whole ring-buffer and we don't need the tail
|
110
110
|
either. However, we do this memory usage optimization only if the
|
111
111
|
first write is less than the tail size, which is also the input block
|
112
112
|
size, otherwise it is likely that other blocks will follow and we
|
@@ -131,7 +131,7 @@ static BROTLI_INLINE void RingBufferWrite(
|
|
131
131
|
/* The length of the writes is limited so that we do not need to worry
|
132
132
|
about a write */
|
133
133
|
RingBufferWriteTail(bytes, n, rb);
|
134
|
-
if (
|
134
|
+
if (BROTLI_PREDICT_TRUE(masked_pos + n <= rb->size_)) {
|
135
135
|
/* A single write fits. */
|
136
136
|
memcpy(&rb->buffer_[masked_pos], bytes, n);
|
137
137
|
} else {
|
@@ -33,23 +33,24 @@ static BROTLI_INLINE void AddMatch(size_t distance, size_t len, size_t len_code,
|
|
33
33
|
matches[len] = BROTLI_MIN(uint32_t, matches[len], match);
|
34
34
|
}
|
35
35
|
|
36
|
-
static BROTLI_INLINE size_t DictMatchLength(const
|
36
|
+
static BROTLI_INLINE size_t DictMatchLength(const BrotliDictionary* dictionary,
|
37
|
+
const uint8_t* data,
|
37
38
|
size_t id,
|
38
39
|
size_t len,
|
39
40
|
size_t maxlen) {
|
40
|
-
const size_t offset =
|
41
|
-
return FindMatchLengthWithLimit(&
|
41
|
+
const size_t offset = dictionary->offsets_by_length[len] + len * id;
|
42
|
+
return FindMatchLengthWithLimit(&dictionary->data[offset], data,
|
42
43
|
BROTLI_MIN(size_t, len, maxlen));
|
43
44
|
}
|
44
45
|
|
45
|
-
static BROTLI_INLINE BROTLI_BOOL IsMatch(
|
46
|
+
static BROTLI_INLINE BROTLI_BOOL IsMatch(const BrotliDictionary* dictionary,
|
46
47
|
DictWord w, const uint8_t* data, size_t max_length) {
|
47
48
|
if (w.len > max_length) {
|
48
49
|
return BROTLI_FALSE;
|
49
50
|
} else {
|
50
|
-
const size_t offset =
|
51
|
+
const size_t offset = dictionary->offsets_by_length[w.len] +
|
51
52
|
(size_t)w.len * (size_t)w.idx;
|
52
|
-
const uint8_t* dict = &
|
53
|
+
const uint8_t* dict = &dictionary->data[offset];
|
53
54
|
if (w.transform == 0) {
|
54
55
|
/* Match against base dictionary word. */
|
55
56
|
return
|
@@ -78,21 +79,22 @@ static BROTLI_INLINE BROTLI_BOOL IsMatch(
|
|
78
79
|
}
|
79
80
|
|
80
81
|
BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
81
|
-
const uint8_t* data, size_t min_length,
|
82
|
-
uint32_t* matches) {
|
82
|
+
const BrotliDictionary* dictionary, const uint8_t* data, size_t min_length,
|
83
|
+
size_t max_length, uint32_t* matches) {
|
83
84
|
BROTLI_BOOL has_found_match = BROTLI_FALSE;
|
84
85
|
{
|
85
86
|
size_t offset = kStaticDictionaryBuckets[Hash(data)];
|
86
87
|
BROTLI_BOOL end = !offset;
|
87
88
|
while (!end) {
|
88
89
|
DictWord w = kStaticDictionaryWords[offset++];
|
89
|
-
const size_t l = w.len &
|
90
|
-
const size_t n = (size_t)1 <<
|
90
|
+
const size_t l = w.len & 0x1F;
|
91
|
+
const size_t n = (size_t)1 << dictionary->size_bits_by_length[l];
|
91
92
|
const size_t id = w.idx;
|
92
93
|
end = !!(w.len & 0x80);
|
93
94
|
w.len = (uint8_t)l;
|
94
95
|
if (w.transform == 0) {
|
95
|
-
const size_t matchlen =
|
96
|
+
const size_t matchlen =
|
97
|
+
DictMatchLength(dictionary, data, id, l, max_length);
|
96
98
|
const uint8_t* s;
|
97
99
|
size_t minlen;
|
98
100
|
size_t maxlen;
|
@@ -276,7 +278,7 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
276
278
|
const BROTLI_BOOL is_all_caps =
|
277
279
|
TO_BROTLI_BOOL(w.transform != kUppercaseFirst);
|
278
280
|
const uint8_t* s;
|
279
|
-
if (!IsMatch(w, data, max_length)) {
|
281
|
+
if (!IsMatch(dictionary, w, data, max_length)) {
|
280
282
|
continue;
|
281
283
|
}
|
282
284
|
/* Transform "" + kUppercase{First,All} + "" */
|
@@ -325,14 +327,14 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
325
327
|
BROTLI_BOOL end = !offset;
|
326
328
|
while (!end) {
|
327
329
|
DictWord w = kStaticDictionaryWords[offset++];
|
328
|
-
const size_t l = w.len &
|
329
|
-
const size_t n = (size_t)1 <<
|
330
|
+
const size_t l = w.len & 0x1F;
|
331
|
+
const size_t n = (size_t)1 << dictionary->size_bits_by_length[l];
|
330
332
|
const size_t id = w.idx;
|
331
333
|
end = !!(w.len & 0x80);
|
332
334
|
w.len = (uint8_t)l;
|
333
335
|
if (w.transform == 0) {
|
334
336
|
const uint8_t* s;
|
335
|
-
if (!IsMatch(w, &data[1], max_length - 1)) {
|
337
|
+
if (!IsMatch(dictionary, w, &data[1], max_length - 1)) {
|
336
338
|
continue;
|
337
339
|
}
|
338
340
|
/* Transforms " " + kIdentity + "" and "." + kIdentity + "" */
|
@@ -373,7 +375,7 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
373
375
|
const BROTLI_BOOL is_all_caps =
|
374
376
|
TO_BROTLI_BOOL(w.transform != kUppercaseFirst);
|
375
377
|
const uint8_t* s;
|
376
|
-
if (!IsMatch(w, &data[1], max_length - 1)) {
|
378
|
+
if (!IsMatch(dictionary, w, &data[1], max_length - 1)) {
|
377
379
|
continue;
|
378
380
|
}
|
379
381
|
/* Transforms " " + kUppercase{First,All} + "" */
|
@@ -417,12 +419,13 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
417
419
|
BROTLI_BOOL end = !offset;
|
418
420
|
while (!end) {
|
419
421
|
DictWord w = kStaticDictionaryWords[offset++];
|
420
|
-
const size_t l = w.len &
|
421
|
-
const size_t n = (size_t)1 <<
|
422
|
+
const size_t l = w.len & 0x1F;
|
423
|
+
const size_t n = (size_t)1 << dictionary->size_bits_by_length[l];
|
422
424
|
const size_t id = w.idx;
|
423
425
|
end = !!(w.len & 0x80);
|
424
426
|
w.len = (uint8_t)l;
|
425
|
-
if (w.transform == 0 &&
|
427
|
+
if (w.transform == 0 &&
|
428
|
+
IsMatch(dictionary, w, &data[2], max_length - 2)) {
|
426
429
|
if (data[0] == 0xc2) {
|
427
430
|
AddMatch(id + 102 * n, l + 2, l, matches);
|
428
431
|
has_found_match = BROTLI_TRUE;
|
@@ -445,12 +448,13 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
|
445
448
|
BROTLI_BOOL end = !offset;
|
446
449
|
while (!end) {
|
447
450
|
DictWord w = kStaticDictionaryWords[offset++];
|
448
|
-
const size_t l = w.len &
|
449
|
-
const size_t n = (size_t)1 <<
|
451
|
+
const size_t l = w.len & 0x1F;
|
452
|
+
const size_t n = (size_t)1 << dictionary->size_bits_by_length[l];
|
450
453
|
const size_t id = w.idx;
|
451
454
|
end = !!(w.len & 0x80);
|
452
455
|
w.len = (uint8_t)l;
|
453
|
-
if (w.transform == 0 &&
|
456
|
+
if (w.transform == 0 &&
|
457
|
+
IsMatch(dictionary, w, &data[5], max_length - 5)) {
|
454
458
|
AddMatch(id + (data[0] == ' ' ? 41 : 72) * n, l + 5, l, matches);
|
455
459
|
has_found_match = BROTLI_TRUE;
|
456
460
|
if (l + 5 < max_length) {
|