brotli 0.1.8 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +7 -3
- data/brotli.gemspec +1 -1
- data/ext/brotli/brotli.c +4 -4
- data/ext/brotli/brotli.h +2 -2
- data/ext/brotli/extconf.rb +9 -16
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/{common → c/common}/constants.h +11 -1
- data/vendor/brotli/c/common/dictionary.bin +432 -0
- data/vendor/brotli/c/common/dictionary.c +5905 -0
- data/vendor/brotli/c/common/dictionary.h +64 -0
- data/vendor/brotli/c/common/version.h +19 -0
- data/vendor/brotli/{dec → c/dec}/bit_reader.c +2 -2
- data/vendor/brotli/{dec → c/dec}/bit_reader.h +11 -34
- data/vendor/brotli/{dec → c/dec}/context.h +1 -1
- data/vendor/brotli/{dec → c/dec}/decode.c +389 -356
- data/vendor/brotli/{dec → c/dec}/huffman.c +24 -23
- data/vendor/brotli/{dec → c/dec}/huffman.h +1 -1
- data/vendor/brotli/{dec → c/dec}/port.h +19 -10
- data/vendor/brotli/{dec → c/dec}/prefix.h +1 -1
- data/vendor/brotli/{dec → c/dec}/state.c +23 -19
- data/vendor/brotli/{dec → c/dec}/state.h +18 -17
- data/vendor/brotli/{dec → c/dec}/transform.h +2 -2
- data/vendor/brotli/c/enc/backward_references.c +134 -0
- data/vendor/brotli/c/enc/backward_references.h +39 -0
- data/vendor/brotli/{enc/backward_references.c → c/enc/backward_references_hq.c} +144 -232
- data/vendor/brotli/{enc/backward_references.h → c/enc/backward_references_hq.h} +28 -31
- data/vendor/brotli/{enc → c/enc}/backward_references_inc.h +37 -31
- data/vendor/brotli/{enc → c/enc}/bit_cost.c +1 -1
- data/vendor/brotli/{enc → c/enc}/bit_cost.h +1 -1
- data/vendor/brotli/{enc → c/enc}/bit_cost_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/block_encoder_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/block_splitter.c +2 -4
- data/vendor/brotli/{enc → c/enc}/block_splitter.h +1 -1
- data/vendor/brotli/{enc → c/enc}/block_splitter_inc.h +6 -7
- data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.c +22 -26
- data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.h +1 -5
- data/vendor/brotli/{enc → c/enc}/cluster.c +1 -1
- data/vendor/brotli/{enc → c/enc}/cluster.h +1 -1
- data/vendor/brotli/{enc → c/enc}/cluster_inc.h +2 -0
- data/vendor/brotli/{enc → c/enc}/command.h +34 -17
- data/vendor/brotli/{enc → c/enc}/compress_fragment.c +97 -53
- data/vendor/brotli/{enc → c/enc}/compress_fragment.h +5 -2
- data/vendor/brotli/{enc → c/enc}/compress_fragment_two_pass.c +106 -51
- data/vendor/brotli/{enc → c/enc}/compress_fragment_two_pass.h +5 -2
- data/vendor/brotli/{enc → c/enc}/context.h +3 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +1120 -0
- data/vendor/brotli/c/enc/dictionary_hash.h +24 -0
- data/vendor/brotli/{enc → c/enc}/encode.c +442 -240
- data/vendor/brotli/{enc → c/enc}/entropy_encode.c +9 -9
- data/vendor/brotli/{enc → c/enc}/entropy_encode.h +4 -4
- data/vendor/brotli/{enc → c/enc}/entropy_encode_static.h +4 -4
- data/vendor/brotli/{enc → c/enc}/fast_log.h +3 -3
- data/vendor/brotli/{enc → c/enc}/find_match_length.h +8 -8
- data/vendor/brotli/c/enc/hash.h +446 -0
- data/vendor/brotli/{enc → c/enc}/hash_forgetful_chain_inc.h +72 -68
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +266 -0
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +258 -0
- data/vendor/brotli/{enc → c/enc}/hash_longest_match_quickly_inc.h +81 -77
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +326 -0
- data/vendor/brotli/{enc → c/enc}/histogram.c +4 -2
- data/vendor/brotli/{enc → c/enc}/histogram.h +1 -1
- data/vendor/brotli/{enc → c/enc}/histogram_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/literal_cost.c +4 -7
- data/vendor/brotli/{enc → c/enc}/literal_cost.h +2 -2
- data/vendor/brotli/{enc → c/enc}/memory.c +1 -1
- data/vendor/brotli/{enc → c/enc}/memory.h +3 -2
- data/vendor/brotli/{enc → c/enc}/metablock.c +136 -123
- data/vendor/brotli/{enc → c/enc}/metablock.h +2 -12
- data/vendor/brotli/{enc → c/enc}/metablock_inc.h +0 -0
- data/vendor/brotli/{enc → c/enc}/port.h +49 -33
- data/vendor/brotli/{enc → c/enc}/prefix.h +4 -2
- data/vendor/brotli/{enc → c/enc}/quality.h +47 -17
- data/vendor/brotli/{enc → c/enc}/ringbuffer.h +6 -6
- data/vendor/brotli/{enc → c/enc}/static_dict.c +26 -22
- data/vendor/brotli/{enc → c/enc}/static_dict.h +3 -1
- data/vendor/brotli/c/enc/static_dict_lut.h +5864 -0
- data/vendor/brotli/{enc → c/enc}/utf8_util.c +1 -1
- data/vendor/brotli/{enc → c/enc}/utf8_util.h +2 -2
- data/vendor/brotli/{enc → c/enc}/write_bits.h +3 -3
- data/vendor/brotli/c/include/brotli/decode.h +339 -0
- data/vendor/brotli/c/include/brotli/encode.h +402 -0
- data/vendor/brotli/c/include/brotli/port.h +146 -0
- data/vendor/brotli/c/include/brotli/types.h +90 -0
- metadata +80 -79
- data/vendor/brotli/common/dictionary.c +0 -9474
- data/vendor/brotli/common/dictionary.h +0 -29
- data/vendor/brotli/common/port.h +0 -107
- data/vendor/brotli/common/types.h +0 -58
- data/vendor/brotli/dec/decode.h +0 -188
- data/vendor/brotli/enc/compressor.cc +0 -139
- data/vendor/brotli/enc/compressor.h +0 -161
- data/vendor/brotli/enc/dictionary_hash.h +0 -4121
- data/vendor/brotli/enc/encode.h +0 -221
- data/vendor/brotli/enc/encode_parallel.cc +0 -289
- data/vendor/brotli/enc/encode_parallel.h +0 -27
- data/vendor/brotli/enc/hash.h +0 -717
- data/vendor/brotli/enc/hash_longest_match_inc.h +0 -241
- data/vendor/brotli/enc/static_dict_lut.h +0 -11241
- data/vendor/brotli/enc/streams.cc +0 -114
- data/vendor/brotli/enc/streams.h +0 -121
@@ -0,0 +1,24 @@
|
|
1
|
+
/* Copyright 2015 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
/* Hash table on the 4-byte prefixes of static dictionary words. */
|
8
|
+
|
9
|
+
#ifndef BROTLI_ENC_DICTIONARY_HASH_H_
|
10
|
+
#define BROTLI_ENC_DICTIONARY_HASH_H_
|
11
|
+
|
12
|
+
#include <brotli/types.h>
|
13
|
+
|
14
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
15
|
+
extern "C" {
|
16
|
+
#endif
|
17
|
+
|
18
|
+
extern const uint16_t kStaticDictionaryHash[32768];
|
19
|
+
|
20
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
21
|
+
} /* extern "C" */
|
22
|
+
#endif
|
23
|
+
|
24
|
+
#endif /* BROTLI_ENC_DICTIONARY_HASH_H_ */
|
@@ -6,12 +6,14 @@
|
|
6
6
|
|
7
7
|
/* Implementation of Brotli compressor. */
|
8
8
|
|
9
|
-
#include
|
9
|
+
#include <brotli/encode.h>
|
10
10
|
|
11
11
|
#include <stdlib.h> /* free, malloc */
|
12
12
|
#include <string.h> /* memcpy, memset */
|
13
13
|
|
14
|
+
#include "../common/version.h"
|
14
15
|
#include "./backward_references.h"
|
16
|
+
#include "./backward_references_hq.h"
|
15
17
|
#include "./bit_cost.h"
|
16
18
|
#include "./brotli_bit_stream.h"
|
17
19
|
#include "./compress_fragment.h"
|
@@ -43,7 +45,11 @@ typedef enum BrotliEncoderStreamState {
|
|
43
45
|
performed before getting back to default state. */
|
44
46
|
BROTLI_STREAM_FLUSH_REQUESTED = 1,
|
45
47
|
/* Last metablock was produced; no more input is acceptable. */
|
46
|
-
BROTLI_STREAM_FINISHED = 2
|
48
|
+
BROTLI_STREAM_FINISHED = 2,
|
49
|
+
/* Flushing compressed block and writing meta-data block header. */
|
50
|
+
BROTLI_STREAM_METADATA_HEAD = 3,
|
51
|
+
/* Writing metadata block body. */
|
52
|
+
BROTLI_STREAM_METADATA_BODY = 4
|
47
53
|
} BrotliEncoderStreamState;
|
48
54
|
|
49
55
|
typedef struct BrotliEncoderStateStruct {
|
@@ -51,7 +57,7 @@ typedef struct BrotliEncoderStateStruct {
|
|
51
57
|
|
52
58
|
MemoryManager memory_manager_;
|
53
59
|
|
54
|
-
|
60
|
+
HasherHandle hasher_;
|
55
61
|
uint64_t input_pos_;
|
56
62
|
RingBuffer ringbuffer_;
|
57
63
|
size_t cmd_alloc_size_;
|
@@ -61,7 +67,7 @@ typedef struct BrotliEncoderStateStruct {
|
|
61
67
|
size_t last_insert_len_;
|
62
68
|
uint64_t last_flush_pos_;
|
63
69
|
uint64_t last_processed_pos_;
|
64
|
-
int dist_cache_[
|
70
|
+
int dist_cache_[BROTLI_NUM_DISTANCE_SHORT_CODES];
|
65
71
|
int saved_dist_cache_[4];
|
66
72
|
uint8_t last_byte_;
|
67
73
|
uint8_t last_byte_bits_;
|
@@ -94,7 +100,12 @@ typedef struct BrotliEncoderStateStruct {
|
|
94
100
|
uint8_t* next_out_;
|
95
101
|
size_t available_out_;
|
96
102
|
size_t total_out_;
|
97
|
-
|
103
|
+
/* Temporary buffer for padding flush bits or metadata block header / body. */
|
104
|
+
union {
|
105
|
+
uint64_t u64[2];
|
106
|
+
uint8_t u8[16];
|
107
|
+
} tiny_buf_;
|
108
|
+
uint32_t remaining_metadata_bytes_;
|
98
109
|
BrotliEncoderStreamState stream_state_;
|
99
110
|
|
100
111
|
BROTLI_BOOL is_last_block_emitted_;
|
@@ -103,7 +114,7 @@ typedef struct BrotliEncoderStateStruct {
|
|
103
114
|
|
104
115
|
static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s);
|
105
116
|
|
106
|
-
size_t
|
117
|
+
static size_t InputBlockSize(BrotliEncoderState* s) {
|
107
118
|
if (!EnsureInitialized(s)) return 0;
|
108
119
|
return (size_t)1 << s->params.lgblock;
|
109
120
|
}
|
@@ -114,7 +125,7 @@ static uint64_t UnprocessedInputSize(BrotliEncoderState* s) {
|
|
114
125
|
|
115
126
|
static size_t RemainingInputBlockSize(BrotliEncoderState* s) {
|
116
127
|
const uint64_t delta = UnprocessedInputSize(s);
|
117
|
-
size_t block_size =
|
128
|
+
size_t block_size = InputBlockSize(s);
|
118
129
|
if (delta >= block_size) return 0;
|
119
130
|
return block_size - (size_t)delta;
|
120
131
|
}
|
@@ -123,7 +134,7 @@ BROTLI_BOOL BrotliEncoderSetParameter(
|
|
123
134
|
BrotliEncoderState* state, BrotliEncoderParameter p, uint32_t value) {
|
124
135
|
/* Changing parameters on the fly is not implemented yet. */
|
125
136
|
if (state->is_initialized_) return BROTLI_FALSE;
|
126
|
-
/* TODO: Validate/clamp
|
137
|
+
/* TODO: Validate/clamp parameters here. */
|
127
138
|
switch (p) {
|
128
139
|
case BROTLI_PARAM_MODE:
|
129
140
|
state->params.mode = (BrotliEncoderMode)value;
|
@@ -141,6 +152,15 @@ BROTLI_BOOL BrotliEncoderSetParameter(
|
|
141
152
|
state->params.lgblock = (int)value;
|
142
153
|
return BROTLI_TRUE;
|
143
154
|
|
155
|
+
case BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING:
|
156
|
+
if ((value != 0) && (value != 1)) return BROTLI_FALSE;
|
157
|
+
state->params.disable_literal_context_modeling = TO_BROTLI_BOOL(!!value);
|
158
|
+
return BROTLI_TRUE;
|
159
|
+
|
160
|
+
case BROTLI_PARAM_SIZE_HINT:
|
161
|
+
state->params.size_hint = value;
|
162
|
+
return BROTLI_TRUE;
|
163
|
+
|
144
164
|
default: return BROTLI_FALSE;
|
145
165
|
}
|
146
166
|
}
|
@@ -156,7 +176,7 @@ static void RecomputeDistancePrefixes(Command* cmds,
|
|
156
176
|
for (i = 0; i < num_commands; ++i) {
|
157
177
|
Command* cmd = &cmds[i];
|
158
178
|
if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
|
159
|
-
PrefixEncodeCopyDistance(
|
179
|
+
PrefixEncodeCopyDistance(CommandRestoreDistanceCode(cmd),
|
160
180
|
num_direct_distance_codes,
|
161
181
|
distance_postfix_bits,
|
162
182
|
&cmd->dist_prefix_,
|
@@ -165,13 +185,13 @@ static void RecomputeDistancePrefixes(Command* cmds,
|
|
165
185
|
}
|
166
186
|
}
|
167
187
|
|
168
|
-
/* Wraps 64-bit input position to 32-bit
|
188
|
+
/* Wraps 64-bit input position to 32-bit ring-buffer position preserving
|
169
189
|
"not-a-first-lap" feature. */
|
170
190
|
static uint32_t WrapPosition(uint64_t position) {
|
171
191
|
uint32_t result = (uint32_t)position;
|
172
192
|
uint64_t gb = position >> 30;
|
173
193
|
if (gb > 2) {
|
174
|
-
/* Wrap every 2GiB; The first 3GB are
|
194
|
+
/* Wrap every 2GiB; The first 3GB are continuous. */
|
175
195
|
result = (result & ((1u << 30) - 1)) | ((uint32_t)((gb - 1) & 1) + 1) << 30;
|
176
196
|
}
|
177
197
|
return result;
|
@@ -207,6 +227,12 @@ static int* GetHashTable(BrotliEncoderState* s, int quality,
|
|
207
227
|
size_t htsize = HashTableSize(max_table_size, input_size);
|
208
228
|
int* table;
|
209
229
|
assert(max_table_size >= 256);
|
230
|
+
if (quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
|
231
|
+
/* Only odd shifts are supported by fast-one-pass. */
|
232
|
+
if ((htsize & 0xAAAAA) == 0) {
|
233
|
+
htsize <<= 1;
|
234
|
+
}
|
235
|
+
}
|
210
236
|
|
211
237
|
if (htsize <= sizeof(s->small_table_) / sizeof(s->small_table_[0])) {
|
212
238
|
table = s->small_table_;
|
@@ -290,8 +316,8 @@ static void InitCommandPrefixCodes(uint8_t cmd_depths[128],
|
|
290
316
|
|
291
317
|
/* Decide about the context map based on the ability of the prediction
|
292
318
|
ability of the previous byte UTF8-prefix on the next byte. The
|
293
|
-
prediction ability is calculated as
|
294
|
-
|
319
|
+
prediction ability is calculated as Shannon entropy. Here we need
|
320
|
+
Shannon entropy instead of 'BitsEntropy' since the prefix will be
|
295
321
|
encoded with the remaining 6 bits of the following byte, and
|
296
322
|
BitsEntropy will assume that symbol to be stored alone using Huffman
|
297
323
|
coding. */
|
@@ -314,18 +340,13 @@ static void ChooseContextMap(int quality,
|
|
314
340
|
|
315
341
|
uint32_t monogram_histo[3] = { 0 };
|
316
342
|
uint32_t two_prefix_histo[6] = { 0 };
|
317
|
-
size_t total
|
343
|
+
size_t total;
|
318
344
|
size_t i;
|
319
345
|
size_t dummy;
|
320
346
|
double entropy[4];
|
321
347
|
for (i = 0; i < 9; ++i) {
|
322
|
-
size_t j = i;
|
323
|
-
total += bigram_histo[i];
|
324
348
|
monogram_histo[i % 3] += bigram_histo[i];
|
325
|
-
|
326
|
-
j -= 6;
|
327
|
-
}
|
328
|
-
two_prefix_histo[j] += bigram_histo[i];
|
349
|
+
two_prefix_histo[i % 6] += bigram_histo[i];
|
329
350
|
}
|
330
351
|
entropy[1] = ShannonEntropy(monogram_histo, 3, &dummy);
|
331
352
|
entropy[2] = (ShannonEntropy(two_prefix_histo, 3, &dummy) +
|
@@ -335,6 +356,7 @@ static void ChooseContextMap(int quality,
|
|
335
356
|
entropy[3] += ShannonEntropy(bigram_histo + 3 * i, 3, &dummy);
|
336
357
|
}
|
337
358
|
|
359
|
+
total = monogram_histo[0] + monogram_histo[1] + monogram_histo[2];
|
338
360
|
assert(total != 0);
|
339
361
|
entropy[0] = 1.0 / (double)total;
|
340
362
|
entropy[1] *= entropy[0];
|
@@ -359,14 +381,99 @@ static void ChooseContextMap(int quality,
|
|
359
381
|
}
|
360
382
|
}
|
361
383
|
|
384
|
+
/* Decide if we want to use a more complex static context map containing 13
|
385
|
+
context values, based on the entropy reduction of histograms over the
|
386
|
+
first 5 bits of literals. */
|
387
|
+
static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
|
388
|
+
size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
|
389
|
+
size_t* num_literal_contexts, const uint32_t** literal_context_map) {
|
390
|
+
static const uint32_t kStaticContextMapComplexUTF8[64] = {
|
391
|
+
11, 11, 12, 12, /* 0 special */
|
392
|
+
0, 0, 0, 0, /* 4 lf */
|
393
|
+
1, 1, 9, 9, /* 8 space */
|
394
|
+
2, 2, 2, 2, /* !, first after space/lf and after something else. */
|
395
|
+
1, 1, 1, 1, /* " */
|
396
|
+
8, 3, 3, 3, /* % */
|
397
|
+
1, 1, 1, 1, /* ({[ */
|
398
|
+
2, 2, 2, 2, /* }]) */
|
399
|
+
8, 4, 4, 4, /* :; */
|
400
|
+
8, 7, 4, 4, /* . */
|
401
|
+
8, 0, 0, 0, /* > */
|
402
|
+
3, 3, 3, 3, /* [0..9] */
|
403
|
+
5, 5, 10, 5, /* [A-Z] */
|
404
|
+
5, 5, 10, 5,
|
405
|
+
6, 6, 6, 6, /* [a-z] */
|
406
|
+
6, 6, 6, 6,
|
407
|
+
};
|
408
|
+
BROTLI_UNUSED(quality);
|
409
|
+
/* Try the more complex static context map only for long data. */
|
410
|
+
if (size_hint < (1 << 20)) {
|
411
|
+
return BROTLI_FALSE;
|
412
|
+
} else {
|
413
|
+
const size_t end_pos = start_pos + length;
|
414
|
+
/* To make entropy calculations faster and to fit on the stack, we collect
|
415
|
+
histograms over the 5 most significant bits of literals. One histogram
|
416
|
+
without context and 13 additional histograms for each context value. */
|
417
|
+
uint32_t combined_histo[32] = { 0 };
|
418
|
+
uint32_t context_histo[13][32] = { { 0 } };
|
419
|
+
uint32_t total = 0;
|
420
|
+
double entropy[3];
|
421
|
+
size_t dummy;
|
422
|
+
size_t i;
|
423
|
+
for (; start_pos + 64 <= end_pos; start_pos += 4096) {
|
424
|
+
const size_t stride_end_pos = start_pos + 64;
|
425
|
+
uint8_t prev2 = input[start_pos & mask];
|
426
|
+
uint8_t prev1 = input[(start_pos + 1) & mask];
|
427
|
+
size_t pos;
|
428
|
+
/* To make the analysis of the data faster we only examine 64 byte long
|
429
|
+
strides at every 4kB intervals. */
|
430
|
+
for (pos = start_pos + 2; pos < stride_end_pos; ++pos) {
|
431
|
+
const uint8_t literal = input[pos & mask];
|
432
|
+
const uint8_t context = (uint8_t)kStaticContextMapComplexUTF8[
|
433
|
+
Context(prev1, prev2, CONTEXT_UTF8)];
|
434
|
+
++total;
|
435
|
+
++combined_histo[literal >> 3];
|
436
|
+
++context_histo[context][literal >> 3];
|
437
|
+
prev2 = prev1;
|
438
|
+
prev1 = literal;
|
439
|
+
}
|
440
|
+
}
|
441
|
+
entropy[1] = ShannonEntropy(combined_histo, 32, &dummy);
|
442
|
+
entropy[2] = 0;
|
443
|
+
for (i = 0; i < 13; ++i) {
|
444
|
+
entropy[2] += ShannonEntropy(&context_histo[i][0], 32, &dummy);
|
445
|
+
}
|
446
|
+
entropy[0] = 1.0 / (double)total;
|
447
|
+
entropy[1] *= entropy[0];
|
448
|
+
entropy[2] *= entropy[0];
|
449
|
+
/* The triggering heuristics below were tuned by compressing the individual
|
450
|
+
files of the silesia corpus. If we skip this kind of context modeling
|
451
|
+
for not very well compressible input (i.e. entropy using context modeling
|
452
|
+
is 60% of maximal entropy) or if expected savings by symbol are less
|
453
|
+
than 0.2 bits, then in every case when it triggers, the final compression
|
454
|
+
ratio is improved. Note however that this heuristics might be too strict
|
455
|
+
for some cases and could be tuned further. */
|
456
|
+
if (entropy[2] > 3.0 || entropy[1] - entropy[2] < 0.2) {
|
457
|
+
return BROTLI_FALSE;
|
458
|
+
} else {
|
459
|
+
*num_literal_contexts = 13;
|
460
|
+
*literal_context_map = kStaticContextMapComplexUTF8;
|
461
|
+
return BROTLI_TRUE;
|
462
|
+
}
|
463
|
+
}
|
464
|
+
}
|
465
|
+
|
362
466
|
static void DecideOverLiteralContextModeling(const uint8_t* input,
|
363
|
-
size_t start_pos, size_t length, size_t mask, int quality,
|
364
|
-
|
365
|
-
const uint32_t** literal_context_map) {
|
467
|
+
size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
|
468
|
+
size_t* num_literal_contexts, const uint32_t** literal_context_map) {
|
366
469
|
if (quality < MIN_QUALITY_FOR_CONTEXT_MODELING || length < 64) {
|
367
470
|
return;
|
471
|
+
} else if (ShouldUseComplexStaticContextMap(
|
472
|
+
input, start_pos, length, mask, quality, size_hint,
|
473
|
+
num_literal_contexts, literal_context_map)) {
|
474
|
+
/* Context map was already set, nothing else to do. */
|
368
475
|
} else {
|
369
|
-
/* Gather
|
476
|
+
/* Gather bi-gram data of the UTF8 byte prefixes. To make the analysis of
|
370
477
|
UTF8 data faster we only examine 64 byte long strides at every 4kB
|
371
478
|
intervals. */
|
372
479
|
const size_t end_pos = start_pos + length;
|
@@ -382,7 +489,6 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
|
|
382
489
|
prev = lut[literal >> 6] * 3;
|
383
490
|
}
|
384
491
|
}
|
385
|
-
*literal_context_mode = CONTEXT_UTF8;
|
386
492
|
ChooseContextMap(quality, &bigram_prefix_histo[0], num_literal_contexts,
|
387
493
|
literal_context_map);
|
388
494
|
}
|
@@ -464,7 +570,7 @@ static void WriteMetaBlockInternal(MemoryManager* m,
|
|
464
570
|
num_direct_distance_codes,
|
465
571
|
distance_postfix_bits);
|
466
572
|
}
|
467
|
-
if (params->quality <=
|
573
|
+
if (params->quality <= MAX_QUALITY_FOR_STATIC_ENTROPY_CODES) {
|
468
574
|
BrotliStoreMetaBlockFast(m, data, wrapped_last_flush_pos,
|
469
575
|
bytes, mask, is_last,
|
470
576
|
commands, num_commands,
|
@@ -483,28 +589,16 @@ static void WriteMetaBlockInternal(MemoryManager* m,
|
|
483
589
|
if (params->quality < MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING) {
|
484
590
|
size_t num_literal_contexts = 1;
|
485
591
|
const uint32_t* literal_context_map = NULL;
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
&literal_context_map);
|
492
|
-
if (literal_context_map == NULL) {
|
493
|
-
BrotliBuildMetaBlockGreedy(m, data, wrapped_last_flush_pos, mask,
|
494
|
-
commands, num_commands, &mb);
|
495
|
-
if (BROTLI_IS_OOM(m)) return;
|
496
|
-
} else {
|
497
|
-
BrotliBuildMetaBlockGreedyWithContexts(m, data,
|
498
|
-
wrapped_last_flush_pos,
|
499
|
-
mask,
|
500
|
-
prev_byte, prev_byte2,
|
501
|
-
literal_context_mode,
|
502
|
-
num_literal_contexts,
|
503
|
-
literal_context_map,
|
504
|
-
commands, num_commands,
|
505
|
-
&mb);
|
506
|
-
if (BROTLI_IS_OOM(m)) return;
|
592
|
+
if (!params->disable_literal_context_modeling) {
|
593
|
+
DecideOverLiteralContextModeling(
|
594
|
+
data, wrapped_last_flush_pos, bytes, mask, params->quality,
|
595
|
+
params->size_hint, &num_literal_contexts,
|
596
|
+
&literal_context_map);
|
507
597
|
}
|
598
|
+
BrotliBuildMetaBlockGreedy(m, data, wrapped_last_flush_pos, mask,
|
599
|
+
prev_byte, prev_byte2, literal_context_mode, num_literal_contexts,
|
600
|
+
literal_context_map, commands, num_commands, &mb);
|
601
|
+
if (BROTLI_IS_OOM(m)) return;
|
508
602
|
} else {
|
509
603
|
if (!BrotliIsMostlyUTF8(data, wrapped_last_flush_pos, mask, bytes,
|
510
604
|
kMinUTF8Ratio)) {
|
@@ -552,30 +646,40 @@ static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s) {
|
|
552
646
|
SanitizeParams(&s->params);
|
553
647
|
s->params.lgblock = ComputeLgBlock(&s->params);
|
554
648
|
|
649
|
+
s->remaining_metadata_bytes_ = BROTLI_UINT32_MAX;
|
650
|
+
|
555
651
|
RingBufferSetup(&s->params, &s->ringbuffer_);
|
556
652
|
|
557
653
|
/* Initialize last byte with stream header. */
|
558
|
-
|
654
|
+
{
|
655
|
+
int lgwin = s->params.lgwin;
|
656
|
+
if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
|
657
|
+
s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
|
658
|
+
lgwin = BROTLI_MAX(int, lgwin, 18);
|
659
|
+
}
|
660
|
+
EncodeWindowBits(lgwin, &s->last_byte_, &s->last_byte_bits_);
|
661
|
+
}
|
559
662
|
|
560
663
|
if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
|
561
664
|
InitCommandPrefixCodes(s->cmd_depths_, s->cmd_bits_,
|
562
665
|
s->cmd_code_, &s->cmd_code_numbits_);
|
563
666
|
}
|
564
667
|
|
565
|
-
/* Initialize hashers. */
|
566
|
-
HashersSetup(&s->memory_manager_, &s->hashers_, ChooseHasher(&s->params));
|
567
|
-
if (BROTLI_IS_OOM(&s->memory_manager_)) return BROTLI_FALSE;
|
568
|
-
|
569
668
|
s->is_initialized_ = BROTLI_TRUE;
|
570
669
|
return BROTLI_TRUE;
|
571
670
|
}
|
572
671
|
|
573
|
-
static void
|
574
|
-
|
575
|
-
|
576
|
-
|
577
|
-
|
672
|
+
static void BrotliEncoderInitParams(BrotliEncoderParams* params) {
|
673
|
+
params->mode = BROTLI_DEFAULT_MODE;
|
674
|
+
params->quality = BROTLI_DEFAULT_QUALITY;
|
675
|
+
params->lgwin = BROTLI_DEFAULT_WINDOW;
|
676
|
+
params->lgblock = 0;
|
677
|
+
params->size_hint = 0;
|
678
|
+
params->disable_literal_context_modeling = BROTLI_FALSE;
|
679
|
+
}
|
578
680
|
|
681
|
+
static void BrotliEncoderInitState(BrotliEncoderState* s) {
|
682
|
+
BrotliEncoderInitParams(&s->params);
|
579
683
|
s->input_pos_ = 0;
|
580
684
|
s->num_commands_ = 0;
|
581
685
|
s->num_literals_ = 0;
|
@@ -586,6 +690,7 @@ static void BrotliEncoderInitState(BrotliEncoderState* s) {
|
|
586
690
|
s->prev_byte2_ = 0;
|
587
691
|
s->storage_size_ = 0;
|
588
692
|
s->storage_ = 0;
|
693
|
+
s->hasher_ = NULL;
|
589
694
|
s->large_table_ = NULL;
|
590
695
|
s->large_table_size_ = 0;
|
591
696
|
s->cmd_code_numbits_ = 0;
|
@@ -598,8 +703,6 @@ static void BrotliEncoderInitState(BrotliEncoderState* s) {
|
|
598
703
|
s->is_last_block_emitted_ = BROTLI_FALSE;
|
599
704
|
s->is_initialized_ = BROTLI_FALSE;
|
600
705
|
|
601
|
-
InitHashers(&s->hashers_);
|
602
|
-
|
603
706
|
RingBufferInit(&s->ringbuffer_);
|
604
707
|
|
605
708
|
s->commands_ = 0;
|
@@ -612,7 +715,7 @@ static void BrotliEncoderInitState(BrotliEncoderState* s) {
|
|
612
715
|
s->dist_cache_[3] = 16;
|
613
716
|
/* Save the state of the distance cache in case we need to restore it for
|
614
717
|
emitting an uncompressed block. */
|
615
|
-
memcpy(s->saved_dist_cache_, s->dist_cache_, sizeof(s->
|
718
|
+
memcpy(s->saved_dist_cache_, s->dist_cache_, sizeof(s->saved_dist_cache_));
|
616
719
|
}
|
617
720
|
|
618
721
|
BrotliEncoderState* BrotliEncoderCreateInstance(brotli_alloc_func alloc_func,
|
@@ -643,7 +746,7 @@ static void BrotliEncoderCleanupState(BrotliEncoderState* s) {
|
|
643
746
|
BROTLI_FREE(m, s->storage_);
|
644
747
|
BROTLI_FREE(m, s->commands_);
|
645
748
|
RingBufferFree(m, &s->ringbuffer_);
|
646
|
-
|
749
|
+
DestroyHasher(m, &s->hasher_);
|
647
750
|
BROTLI_FREE(m, s->large_table_);
|
648
751
|
BROTLI_FREE(m, s->command_buf_);
|
649
752
|
BROTLI_FREE(m, s->literal_buf_);
|
@@ -662,9 +765,16 @@ void BrotliEncoderDestroyInstance(BrotliEncoderState* state) {
|
|
662
765
|
}
|
663
766
|
}
|
664
767
|
|
665
|
-
|
666
|
-
|
667
|
-
|
768
|
+
/*
|
769
|
+
Copies the given input data to the internal ring buffer of the compressor.
|
770
|
+
No processing of the data occurs at this time and this function can be
|
771
|
+
called multiple times before calling WriteBrotliData() to process the
|
772
|
+
accumulated input. At most input_block_size() bytes of input data can be
|
773
|
+
copied to the ring buffer, otherwise the next WriteBrotliData() will fail.
|
774
|
+
*/
|
775
|
+
static void CopyInputToRingBuffer(BrotliEncoderState* s,
|
776
|
+
const size_t input_size,
|
777
|
+
const uint8_t* input_buffer) {
|
668
778
|
RingBuffer* ringbuffer_ = &s->ringbuffer_;
|
669
779
|
MemoryManager* m = &s->memory_manager_;
|
670
780
|
if (!EnsureInitialized(s)) return;
|
@@ -689,8 +799,8 @@ void BrotliEncoderCopyInputToRingBuffer(BrotliEncoderState* s,
|
|
689
799
|
reading new bytes from the input. However, at the last few indexes of
|
690
800
|
the ring buffer, there are not enough bytes to build full-length
|
691
801
|
substrings from. Since the hash table always contains full-length
|
692
|
-
substrings, we erase with dummy
|
693
|
-
substrings will contain
|
802
|
+
substrings, we erase with dummy zeros here to make sure that those
|
803
|
+
substrings will contain zeros at the end instead of uninitialized
|
694
804
|
data.
|
695
805
|
|
696
806
|
Please note that erasing is not necessary (because the
|
@@ -699,55 +809,25 @@ void BrotliEncoderCopyInputToRingBuffer(BrotliEncoderState* s,
|
|
699
809
|
skip erasing if we have already gone around at least once in
|
700
810
|
the ring buffer.
|
701
811
|
|
702
|
-
Only clear during the first round of
|
703
|
-
subsequent rounds data in the
|
812
|
+
Only clear during the first round of ring-buffer writes. On
|
813
|
+
subsequent rounds data in the ring-buffer would be affected. */
|
704
814
|
if (ringbuffer_->pos_ <= ringbuffer_->mask_) {
|
705
815
|
/* This is the first time when the ring buffer is being written.
|
706
816
|
We clear 7 bytes just after the bytes that have been copied from
|
707
817
|
the input buffer.
|
708
818
|
|
709
|
-
The
|
819
|
+
The ring-buffer has a "tail" that holds a copy of the beginning,
|
710
820
|
but only once the ring buffer has been fully written once, i.e.,
|
711
821
|
pos <= mask. For the first time, we need to write values
|
712
822
|
in this tail (where index may be larger than mask), so that
|
713
|
-
we have exactly defined behavior and don't read
|
823
|
+
we have exactly defined behavior and don't read uninitialized
|
714
824
|
memory. Due to performance reasons, hashing reads data using a
|
715
825
|
LOAD64, which can go 7 bytes beyond the bytes written in the
|
716
|
-
|
826
|
+
ring-buffer. */
|
717
827
|
memset(ringbuffer_->buffer_ + ringbuffer_->pos_, 0, 7);
|
718
828
|
}
|
719
829
|
}
|
720
830
|
|
721
|
-
void BrotliEncoderSetCustomDictionary(BrotliEncoderState* s, size_t size,
|
722
|
-
const uint8_t* dict) {
|
723
|
-
size_t max_dict_size = MaxBackwardLimit(s->params.lgwin);
|
724
|
-
size_t dict_size = size;
|
725
|
-
MemoryManager* m = &s->memory_manager_;
|
726
|
-
|
727
|
-
if (!EnsureInitialized(s)) return;
|
728
|
-
|
729
|
-
if (dict_size == 0 ||
|
730
|
-
s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
|
731
|
-
s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
|
732
|
-
return;
|
733
|
-
}
|
734
|
-
if (size > max_dict_size) {
|
735
|
-
dict += size - max_dict_size;
|
736
|
-
dict_size = max_dict_size;
|
737
|
-
}
|
738
|
-
BrotliEncoderCopyInputToRingBuffer(s, dict_size, dict);
|
739
|
-
s->last_flush_pos_ = dict_size;
|
740
|
-
s->last_processed_pos_ = dict_size;
|
741
|
-
if (dict_size > 0) {
|
742
|
-
s->prev_byte_ = dict[dict_size - 1];
|
743
|
-
}
|
744
|
-
if (dict_size > 1) {
|
745
|
-
s->prev_byte2_ = dict[dict_size - 2];
|
746
|
-
}
|
747
|
-
HashersPrependCustomDictionary(m, &s->hashers_, &s->params, dict_size, dict);
|
748
|
-
if (BROTLI_IS_OOM(m)) return;
|
749
|
-
}
|
750
|
-
|
751
831
|
/* Marks all input as processed.
|
752
832
|
Returns true if position wrapping occurs. */
|
753
833
|
static BROTLI_BOOL UpdateLastProcessedPos(BrotliEncoderState* s) {
|
@@ -757,7 +837,19 @@ static BROTLI_BOOL UpdateLastProcessedPos(BrotliEncoderState* s) {
|
|
757
837
|
return TO_BROTLI_BOOL(wrapped_input_pos < wrapped_last_processed_pos);
|
758
838
|
}
|
759
839
|
|
760
|
-
|
840
|
+
/*
|
841
|
+
Processes the accumulated input data and sets |*out_size| to the length of
|
842
|
+
the new output meta-block, or to zero if no new output meta-block has been
|
843
|
+
created (in this case the processed input data is buffered internally).
|
844
|
+
If |*out_size| is positive, |*output| points to the start of the output
|
845
|
+
data. If |is_last| or |force_flush| is BROTLI_TRUE, an output meta-block is
|
846
|
+
always created. However, until |is_last| is BROTLI_TRUE encoder may retain up
|
847
|
+
to 7 bits of the last byte of output. To force encoder to dump the remaining
|
848
|
+
bits use WriteMetadata() to append an empty meta-data block.
|
849
|
+
Returns BROTLI_FALSE if the size of the input data is larger than
|
850
|
+
input_block_size().
|
851
|
+
*/
|
852
|
+
static BROTLI_BOOL EncodeData(
|
761
853
|
BrotliEncoderState* s, const BROTLI_BOOL is_last,
|
762
854
|
const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output) {
|
763
855
|
const uint64_t delta = UnprocessedInputSize(s);
|
@@ -767,6 +859,7 @@ BROTLI_BOOL BrotliEncoderWriteData(
|
|
767
859
|
uint8_t* data;
|
768
860
|
uint32_t mask;
|
769
861
|
MemoryManager* m = &s->memory_manager_;
|
862
|
+
const BrotliDictionary* dictionary = BrotliGetDictionary();
|
770
863
|
|
771
864
|
if (!EnsureInitialized(s)) return BROTLI_FALSE;
|
772
865
|
data = s->ringbuffer_.buffer_;
|
@@ -776,7 +869,7 @@ BROTLI_BOOL BrotliEncoderWriteData(
|
|
776
869
|
if (s->is_last_block_emitted_) return BROTLI_FALSE;
|
777
870
|
if (is_last) s->is_last_block_emitted_ = BROTLI_TRUE;
|
778
871
|
|
779
|
-
if (delta >
|
872
|
+
if (delta > InputBlockSize(s)) {
|
780
873
|
return BROTLI_FALSE;
|
781
874
|
}
|
782
875
|
if (s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY &&
|
@@ -801,7 +894,7 @@ BROTLI_BOOL BrotliEncoderWriteData(
|
|
801
894
|
*out_size = 0;
|
802
895
|
return BROTLI_TRUE;
|
803
896
|
}
|
804
|
-
storage = GetBrotliStorage(s, 2 * bytes +
|
897
|
+
storage = GetBrotliStorage(s, 2 * bytes + 502);
|
805
898
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
806
899
|
storage[0] = s->last_byte_;
|
807
900
|
table = GetHashTable(s, s->params.quality, bytes, &table_size);
|
@@ -838,7 +931,7 @@ BROTLI_BOOL BrotliEncoderWriteData(
|
|
838
931
|
if (newsize > s->cmd_alloc_size_) {
|
839
932
|
Command* new_commands;
|
840
933
|
/* Reserve a bit more memory to allow merging with a next block
|
841
|
-
without
|
934
|
+
without reallocation: that would impact speed. */
|
842
935
|
newsize += (bytes / 4) + 16;
|
843
936
|
s->cmd_alloc_size_ = newsize;
|
844
937
|
new_commands = BROTLI_ALLOC(m, Command, newsize);
|
@@ -851,17 +944,31 @@ BROTLI_BOOL BrotliEncoderWriteData(
|
|
851
944
|
}
|
852
945
|
}
|
853
946
|
|
854
|
-
|
855
|
-
|
856
|
-
&s->params,
|
857
|
-
&s->hashers_,
|
858
|
-
s->dist_cache_,
|
859
|
-
&s->last_insert_len_,
|
860
|
-
&s->commands_[s->num_commands_],
|
861
|
-
&s->num_commands_,
|
862
|
-
&s->num_literals_);
|
947
|
+
InitOrStitchToPreviousBlock(m, &s->hasher_, data, mask, &s->params,
|
948
|
+
wrapped_last_processed_pos, bytes, is_last);
|
863
949
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
864
950
|
|
951
|
+
if (s->params.quality == ZOPFLIFICATION_QUALITY) {
|
952
|
+
assert(s->params.hasher.type == 10);
|
953
|
+
BrotliCreateZopfliBackwardReferences(
|
954
|
+
m, dictionary, bytes, wrapped_last_processed_pos, data, mask,
|
955
|
+
&s->params, s->hasher_, s->dist_cache_, &s->last_insert_len_,
|
956
|
+
&s->commands_[s->num_commands_], &s->num_commands_, &s->num_literals_);
|
957
|
+
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
958
|
+
} else if (s->params.quality == HQ_ZOPFLIFICATION_QUALITY) {
|
959
|
+
assert(s->params.hasher.type == 10);
|
960
|
+
BrotliCreateHqZopfliBackwardReferences(
|
961
|
+
m, dictionary, bytes, wrapped_last_processed_pos, data, mask,
|
962
|
+
&s->params, s->hasher_, s->dist_cache_, &s->last_insert_len_,
|
963
|
+
&s->commands_[s->num_commands_], &s->num_commands_, &s->num_literals_);
|
964
|
+
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
965
|
+
} else {
|
966
|
+
BrotliCreateBackwardReferences(
|
967
|
+
dictionary, bytes, wrapped_last_processed_pos, data, mask,
|
968
|
+
&s->params, s->hasher_, s->dist_cache_, &s->last_insert_len_,
|
969
|
+
&s->commands_[s->num_commands_], &s->num_commands_, &s->num_literals_);
|
970
|
+
}
|
971
|
+
|
865
972
|
{
|
866
973
|
const size_t max_length = MaxMetablockSize(&s->params);
|
867
974
|
const size_t max_literals = max_length / 8;
|
@@ -870,7 +977,7 @@ BROTLI_BOOL BrotliEncoderWriteData(
|
|
870
977
|
/* If maximal possible additional block doesn't fit metablock, flush now. */
|
871
978
|
/* TODO: Postpone decision until next block arrives? */
|
872
979
|
const BROTLI_BOOL next_input_fits_metablock = TO_BROTLI_BOOL(
|
873
|
-
processed_bytes +
|
980
|
+
processed_bytes + InputBlockSize(s) <= max_length);
|
874
981
|
/* If block splitting is not used, then flush as soon as there is some
|
875
982
|
amount of commands / literals produced. */
|
876
983
|
const BROTLI_BOOL should_flush = TO_BROTLI_BOOL(
|
@@ -882,7 +989,7 @@ BROTLI_BOOL BrotliEncoderWriteData(
|
|
882
989
|
s->num_commands_ < max_commands) {
|
883
990
|
/* Merge with next input block. Everything will happen later. */
|
884
991
|
if (UpdateLastProcessedPos(s)) {
|
885
|
-
|
992
|
+
HasherReset(s->hasher_);
|
886
993
|
}
|
887
994
|
*out_size = 0;
|
888
995
|
return BROTLI_TRUE;
|
@@ -908,7 +1015,7 @@ BROTLI_BOOL BrotliEncoderWriteData(
|
|
908
1015
|
{
|
909
1016
|
const uint32_t metablock_size =
|
910
1017
|
(uint32_t)(s->input_pos_ - s->last_flush_pos_);
|
911
|
-
uint8_t* storage = GetBrotliStorage(s, 2 * metablock_size +
|
1018
|
+
uint8_t* storage = GetBrotliStorage(s, 2 * metablock_size + 502);
|
912
1019
|
size_t storage_ix = s->last_byte_bits_;
|
913
1020
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
914
1021
|
storage[0] = s->last_byte_;
|
@@ -922,7 +1029,7 @@ BROTLI_BOOL BrotliEncoderWriteData(
|
|
922
1029
|
s->last_byte_bits_ = storage_ix & 7u;
|
923
1030
|
s->last_flush_pos_ = s->input_pos_;
|
924
1031
|
if (UpdateLastProcessedPos(s)) {
|
925
|
-
|
1032
|
+
HasherReset(s->hasher_);
|
926
1033
|
}
|
927
1034
|
if (s->last_flush_pos_ > 0) {
|
928
1035
|
s->prev_byte_ = data[((uint32_t)s->last_flush_pos_ - 1) & mask];
|
@@ -934,77 +1041,38 @@ BROTLI_BOOL BrotliEncoderWriteData(
|
|
934
1041
|
s->num_literals_ = 0;
|
935
1042
|
/* Save the state of the distance cache in case we need to restore it for
|
936
1043
|
emitting an uncompressed block. */
|
937
|
-
memcpy(s->saved_dist_cache_, s->dist_cache_, sizeof(s->
|
1044
|
+
memcpy(s->saved_dist_cache_, s->dist_cache_, sizeof(s->saved_dist_cache_));
|
938
1045
|
*output = &storage[0];
|
939
1046
|
*out_size = storage_ix >> 3;
|
940
1047
|
return BROTLI_TRUE;
|
941
1048
|
}
|
942
1049
|
}
|
943
1050
|
|
944
|
-
|
945
|
-
|
946
|
-
|
947
|
-
|
948
|
-
|
949
|
-
|
950
|
-
if (!EnsureInitialized(s)) return BROTLI_FALSE;
|
951
|
-
BrotliEncoderCopyInputToRingBuffer(s, input_size, input_buffer);
|
952
|
-
result = BrotliEncoderWriteData(
|
953
|
-
s, is_last, /* force_flush */ BROTLI_TRUE, &out_size, &output);
|
954
|
-
if (!result || out_size > *encoded_size) {
|
955
|
-
return BROTLI_FALSE;
|
956
|
-
}
|
957
|
-
if (out_size > 0) {
|
958
|
-
memcpy(encoded_buffer, output, out_size);
|
959
|
-
}
|
960
|
-
*encoded_size = out_size;
|
961
|
-
return BROTLI_TRUE;
|
962
|
-
}
|
963
|
-
|
964
|
-
BROTLI_BOOL BrotliEncoderWriteMetadata(
|
965
|
-
BrotliEncoderState* s, const size_t input_size, const uint8_t* input_buffer,
|
966
|
-
const BROTLI_BOOL is_last, size_t* encoded_size, uint8_t* encoded_buffer) {
|
967
|
-
uint64_t hdr_buffer_data[2];
|
968
|
-
uint8_t* hdr_buffer = (uint8_t*)&hdr_buffer_data[0];
|
1051
|
+
/* Dumps remaining output bits and metadata header to |header|.
|
1052
|
+
Returns number of produced bytes.
|
1053
|
+
REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long.
|
1054
|
+
REQUIRED: |block_size| <= (1 << 24). */
|
1055
|
+
static size_t WriteMetadataHeader(
|
1056
|
+
BrotliEncoderState* s, const size_t block_size, uint8_t* header) {
|
969
1057
|
size_t storage_ix;
|
970
|
-
if (!EnsureInitialized(s)) return BROTLI_FALSE;
|
971
|
-
if (input_size > (1 << 24) || input_size + 6 > *encoded_size) {
|
972
|
-
return BROTLI_FALSE;
|
973
|
-
}
|
974
1058
|
storage_ix = s->last_byte_bits_;
|
975
|
-
|
976
|
-
BrotliWriteBits(1, 0, &storage_ix, hdr_buffer);
|
977
|
-
BrotliWriteBits(2, 3, &storage_ix, hdr_buffer);
|
978
|
-
BrotliWriteBits(1, 0, &storage_ix, hdr_buffer);
|
979
|
-
if (input_size == 0) {
|
980
|
-
BrotliWriteBits(2, 0, &storage_ix, hdr_buffer);
|
981
|
-
*encoded_size = (storage_ix + 7u) >> 3;
|
982
|
-
memcpy(encoded_buffer, hdr_buffer, *encoded_size);
|
983
|
-
} else {
|
984
|
-
uint32_t nbits = (input_size == 1) ? 0 :
|
985
|
-
(Log2FloorNonZero((uint32_t)input_size - 1) + 1);
|
986
|
-
uint32_t nbytes = (nbits + 7) / 8;
|
987
|
-
size_t hdr_size;
|
988
|
-
BrotliWriteBits(2, nbytes, &storage_ix, hdr_buffer);
|
989
|
-
BrotliWriteBits(8 * nbytes, input_size - 1, &storage_ix, hdr_buffer);
|
990
|
-
hdr_size = (storage_ix + 7u) >> 3;
|
991
|
-
memcpy(encoded_buffer, hdr_buffer, hdr_size);
|
992
|
-
memcpy(&encoded_buffer[hdr_size], input_buffer, input_size);
|
993
|
-
*encoded_size = hdr_size + input_size;
|
994
|
-
}
|
995
|
-
if (is_last) {
|
996
|
-
encoded_buffer[(*encoded_size)++] = 3;
|
997
|
-
}
|
1059
|
+
header[0] = s->last_byte_;
|
998
1060
|
s->last_byte_ = 0;
|
999
1061
|
s->last_byte_bits_ = 0;
|
1000
|
-
return BROTLI_TRUE;
|
1001
|
-
}
|
1002
1062
|
|
1003
|
-
|
1004
|
-
|
1005
|
-
|
1006
|
-
|
1007
|
-
|
1063
|
+
BrotliWriteBits(1, 0, &storage_ix, header);
|
1064
|
+
BrotliWriteBits(2, 3, &storage_ix, header);
|
1065
|
+
BrotliWriteBits(1, 0, &storage_ix, header);
|
1066
|
+
if (block_size == 0) {
|
1067
|
+
BrotliWriteBits(2, 0, &storage_ix, header);
|
1068
|
+
} else {
|
1069
|
+
uint32_t nbits = (block_size == 1) ? 0 :
|
1070
|
+
(Log2FloorNonZero((uint32_t)block_size - 1) + 1);
|
1071
|
+
uint32_t nbytes = (nbits + 7) / 8;
|
1072
|
+
BrotliWriteBits(2, nbytes, &storage_ix, header);
|
1073
|
+
BrotliWriteBits(8 * nbytes, block_size - 1, &storage_ix, header);
|
1074
|
+
}
|
1075
|
+
return (storage_ix + 7u) >> 3;
|
1008
1076
|
}
|
1009
1077
|
|
1010
1078
|
static BROTLI_BOOL BrotliCompressBufferQuality10(
|
@@ -1014,7 +1082,7 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1014
1082
|
MemoryManager* m = &memory_manager;
|
1015
1083
|
|
1016
1084
|
const size_t mask = BROTLI_SIZE_MAX >> 1;
|
1017
|
-
const size_t max_backward_limit =
|
1085
|
+
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(lgwin);
|
1018
1086
|
int dist_cache[4] = { 4, 11, 15, 16 };
|
1019
1087
|
int saved_dist_cache[4] = { 4, 11, 15, 16 };
|
1020
1088
|
BROTLI_BOOL ok = BROTLI_TRUE;
|
@@ -1022,12 +1090,13 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1022
1090
|
size_t total_out_size = 0;
|
1023
1091
|
uint8_t last_byte;
|
1024
1092
|
uint8_t last_byte_bits;
|
1025
|
-
|
1093
|
+
HasherHandle hasher = NULL;
|
1026
1094
|
|
1027
1095
|
const size_t hasher_eff_size =
|
1028
|
-
BROTLI_MIN(size_t, input_size, max_backward_limit +
|
1096
|
+
BROTLI_MIN(size_t, input_size, max_backward_limit + BROTLI_WINDOW_GAP);
|
1029
1097
|
|
1030
1098
|
BrotliEncoderParams params;
|
1099
|
+
const BrotliDictionary* dictionary = BrotliGetDictionary();
|
1031
1100
|
|
1032
1101
|
const int lgmetablock = BROTLI_MIN(int, 24, lgwin + 1);
|
1033
1102
|
size_t max_block_size;
|
@@ -1038,10 +1107,9 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1038
1107
|
uint8_t prev_byte = 0;
|
1039
1108
|
uint8_t prev_byte2 = 0;
|
1040
1109
|
|
1041
|
-
params
|
1110
|
+
BrotliEncoderInitParams(¶ms);
|
1042
1111
|
params.quality = 10;
|
1043
1112
|
params.lgwin = lgwin;
|
1044
|
-
params.lgblock = 0;
|
1045
1113
|
SanitizeParams(¶ms);
|
1046
1114
|
params.lgblock = ComputeLgBlock(¶ms);
|
1047
1115
|
max_block_size = (size_t)1 << params.lgblock;
|
@@ -1050,10 +1118,8 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1050
1118
|
|
1051
1119
|
assert(input_size <= mask + 1);
|
1052
1120
|
EncodeWindowBits(lgwin, &last_byte, &last_byte_bits);
|
1053
|
-
hasher
|
1054
|
-
|
1055
|
-
InitializeH10(hasher);
|
1056
|
-
InitH10(m, hasher, input_buffer, ¶ms, 0, hasher_eff_size, 1);
|
1121
|
+
InitOrStitchToPreviousBlock(m, &hasher, input_buffer, mask, ¶ms,
|
1122
|
+
0, hasher_eff_size, BROTLI_TRUE);
|
1057
1123
|
if (BROTLI_IS_OOM(m)) goto oom;
|
1058
1124
|
|
1059
1125
|
while (ok && metablock_start < input_size) {
|
@@ -1083,14 +1149,14 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1083
1149
|
StitchToPreviousBlockH10(hasher, block_size, block_start,
|
1084
1150
|
input_buffer, mask);
|
1085
1151
|
path_size = BrotliZopfliComputeShortestPath(
|
1086
|
-
m, block_size, block_start, input_buffer, mask, ¶ms,
|
1152
|
+
m, dictionary, block_size, block_start, input_buffer, mask, ¶ms,
|
1087
1153
|
max_backward_limit, dist_cache, hasher, nodes);
|
1088
1154
|
if (BROTLI_IS_OOM(m)) goto oom;
|
1089
1155
|
/* We allocate a command buffer in the first iteration of this loop that
|
1090
1156
|
will be likely big enough for the whole metablock, so that for most
|
1091
1157
|
inputs we will not have to reallocate in later iterations. We do the
|
1092
1158
|
allocation here and not before the loop, because if the input is small,
|
1093
|
-
this will be allocated after the
|
1159
|
+
this will be allocated after the Zopfli cost model is freed, so this
|
1094
1160
|
will not increase peak memory usage.
|
1095
1161
|
TODO: If the first allocation is too small, increase command
|
1096
1162
|
buffer size exponentially. */
|
@@ -1108,7 +1174,8 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1108
1174
|
}
|
1109
1175
|
BrotliZopfliCreateCommands(block_size, block_start, max_backward_limit,
|
1110
1176
|
&nodes[0], dist_cache, &last_insert_len,
|
1111
|
-
&commands[num_commands],
|
1177
|
+
¶ms, &commands[num_commands],
|
1178
|
+
&num_literals);
|
1112
1179
|
num_commands += path_size;
|
1113
1180
|
block_start += block_size;
|
1114
1181
|
metablock_size += block_size;
|
@@ -1165,7 +1232,7 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1165
1232
|
BrotliOptimizeHistograms(num_direct_distance_codes,
|
1166
1233
|
distance_postfix_bits,
|
1167
1234
|
&mb);
|
1168
|
-
storage = BROTLI_ALLOC(m, uint8_t, 2 * metablock_size +
|
1235
|
+
storage = BROTLI_ALLOC(m, uint8_t, 2 * metablock_size + 502);
|
1169
1236
|
if (BROTLI_IS_OOM(m)) goto oom;
|
1170
1237
|
storage[0] = last_byte;
|
1171
1238
|
BrotliStoreMetaBlock(m, input_buffer, metablock_start, metablock_size,
|
@@ -1213,8 +1280,7 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1213
1280
|
}
|
1214
1281
|
|
1215
1282
|
*encoded_size = total_out_size;
|
1216
|
-
|
1217
|
-
BROTLI_FREE(m, hasher);
|
1283
|
+
DestroyHasher(m, &hasher);
|
1218
1284
|
return ok;
|
1219
1285
|
|
1220
1286
|
oom:
|
@@ -1290,7 +1356,8 @@ BROTLI_BOOL BrotliEncoderCompress(
|
|
1290
1356
|
}
|
1291
1357
|
if (quality == 10) {
|
1292
1358
|
/* TODO: Implement this direct path for all quality levels. */
|
1293
|
-
const int lg_win = BROTLI_MIN(int,
|
1359
|
+
const int lg_win = BROTLI_MIN(int, BROTLI_MAX_WINDOW_BITS,
|
1360
|
+
BROTLI_MAX(int, 16, lgwin));
|
1294
1361
|
int ok = BrotliCompressBufferQuality10(lg_win, input_size, input_buffer,
|
1295
1362
|
encoded_size, encoded_buffer);
|
1296
1363
|
if (!ok || (max_out_size && *encoded_size > max_out_size)) {
|
@@ -1312,6 +1379,7 @@ BROTLI_BOOL BrotliEncoderCompress(
|
|
1312
1379
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)quality);
|
1313
1380
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)lgwin);
|
1314
1381
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_MODE, (uint32_t)mode);
|
1382
|
+
BrotliEncoderSetParameter(s, BROTLI_PARAM_SIZE_HINT, (uint32_t)input_size);
|
1315
1383
|
result = BrotliEncoderCompressStream(s, BROTLI_OPERATION_FINISH,
|
1316
1384
|
&available_in, &next_in, &available_out, &next_out, &total_out);
|
1317
1385
|
if (!BrotliEncoderIsFinished(s)) result = 0;
|
@@ -1336,15 +1404,57 @@ fallback:
|
|
1336
1404
|
static void InjectBytePaddingBlock(BrotliEncoderState* s) {
|
1337
1405
|
uint32_t seal = s->last_byte_;
|
1338
1406
|
size_t seal_bits = s->last_byte_bits_;
|
1407
|
+
uint8_t* destination;
|
1339
1408
|
s->last_byte_ = 0;
|
1340
1409
|
s->last_byte_bits_ = 0;
|
1341
|
-
/* is_last = 0, data_nibbles = 11,
|
1410
|
+
/* is_last = 0, data_nibbles = 11, reserved = 0, meta_nibbles = 00 */
|
1342
1411
|
seal |= 0x6u << seal_bits;
|
1343
1412
|
seal_bits += 6;
|
1344
|
-
|
1345
|
-
|
1346
|
-
s->next_out_
|
1347
|
-
|
1413
|
+
/* If we have already created storage, then append to it.
|
1414
|
+
Storage is valid until next block is being compressed. */
|
1415
|
+
if (s->next_out_) {
|
1416
|
+
destination = s->next_out_ + s->available_out_;
|
1417
|
+
} else {
|
1418
|
+
destination = s->tiny_buf_.u8;
|
1419
|
+
s->next_out_ = destination;
|
1420
|
+
}
|
1421
|
+
destination[0] = (uint8_t)seal;
|
1422
|
+
if (seal_bits > 8) destination[1] = (uint8_t)(seal >> 8);
|
1423
|
+
s->available_out_ += (seal_bits + 7) >> 3;
|
1424
|
+
}
|
1425
|
+
|
1426
|
+
/* Injects padding bits or pushes compressed data to output.
|
1427
|
+
Returns false if nothing is done. */
|
1428
|
+
static BROTLI_BOOL InjectFlushOrPushOutput(BrotliEncoderState* s,
|
1429
|
+
size_t* available_out, uint8_t** next_out, size_t* total_out) {
|
1430
|
+
if (s->stream_state_ == BROTLI_STREAM_FLUSH_REQUESTED &&
|
1431
|
+
s->last_byte_bits_ != 0) {
|
1432
|
+
InjectBytePaddingBlock(s);
|
1433
|
+
return BROTLI_TRUE;
|
1434
|
+
}
|
1435
|
+
|
1436
|
+
if (s->available_out_ != 0 && *available_out != 0) {
|
1437
|
+
size_t copy_output_size =
|
1438
|
+
BROTLI_MIN(size_t, s->available_out_, *available_out);
|
1439
|
+
memcpy(*next_out, s->next_out_, copy_output_size);
|
1440
|
+
*next_out += copy_output_size;
|
1441
|
+
*available_out -= copy_output_size;
|
1442
|
+
s->next_out_ += copy_output_size;
|
1443
|
+
s->available_out_ -= copy_output_size;
|
1444
|
+
s->total_out_ += copy_output_size;
|
1445
|
+
if (total_out) *total_out = s->total_out_;
|
1446
|
+
return BROTLI_TRUE;
|
1447
|
+
}
|
1448
|
+
|
1449
|
+
return BROTLI_FALSE;
|
1450
|
+
}
|
1451
|
+
|
1452
|
+
static void CheckFlushComplete(BrotliEncoderState* s) {
|
1453
|
+
if (s->stream_state_ == BROTLI_STREAM_FLUSH_REQUESTED &&
|
1454
|
+
s->available_out_ == 0) {
|
1455
|
+
s->stream_state_ = BROTLI_STREAM_PROCESSING;
|
1456
|
+
s->next_out_ = 0;
|
1457
|
+
}
|
1348
1458
|
}
|
1349
1459
|
|
1350
1460
|
static BROTLI_BOOL BrotliEncoderCompressStreamFast(
|
@@ -1384,24 +1494,7 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
|
|
1384
1494
|
}
|
1385
1495
|
|
1386
1496
|
while (BROTLI_TRUE) {
|
1387
|
-
if (s
|
1388
|
-
s->stream_state_ == BROTLI_STREAM_FLUSH_REQUESTED) {
|
1389
|
-
s->stream_state_ = BROTLI_STREAM_PROCESSING;
|
1390
|
-
if (s->last_byte_bits_ == 0) break;
|
1391
|
-
InjectBytePaddingBlock(s);
|
1392
|
-
continue;
|
1393
|
-
}
|
1394
|
-
|
1395
|
-
if (s->available_out_ != 0 && *available_out != 0) {
|
1396
|
-
size_t copy_output_size =
|
1397
|
-
BROTLI_MIN(size_t, s->available_out_, *available_out);
|
1398
|
-
memcpy(*next_out, s->next_out_, copy_output_size);
|
1399
|
-
*next_out += copy_output_size;
|
1400
|
-
*available_out -= copy_output_size;
|
1401
|
-
s->next_out_ += copy_output_size;
|
1402
|
-
s->available_out_ -= copy_output_size;
|
1403
|
-
s->total_out_ += copy_output_size;
|
1404
|
-
if (total_out) *total_out = s->total_out_;
|
1497
|
+
if (InjectFlushOrPushOutput(s, available_out, next_out, total_out)) {
|
1405
1498
|
continue;
|
1406
1499
|
}
|
1407
1500
|
|
@@ -1416,7 +1509,7 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
|
|
1416
1509
|
(*available_in == block_size) && (op == BROTLI_OPERATION_FINISH);
|
1417
1510
|
BROTLI_BOOL force_flush =
|
1418
1511
|
(*available_in == block_size) && (op == BROTLI_OPERATION_FLUSH);
|
1419
|
-
size_t max_out_size = 2 * block_size +
|
1512
|
+
size_t max_out_size = 2 * block_size + 502;
|
1420
1513
|
BROTLI_BOOL inplace = BROTLI_TRUE;
|
1421
1514
|
uint8_t* storage = NULL;
|
1422
1515
|
size_t storage_ix = s->last_byte_bits_;
|
@@ -1430,7 +1523,7 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
|
|
1430
1523
|
if (max_out_size <= *available_out) {
|
1431
1524
|
storage = *next_out;
|
1432
1525
|
} else {
|
1433
|
-
inplace =
|
1526
|
+
inplace = BROTLI_FALSE;
|
1434
1527
|
storage = GetBrotliStorage(s, max_out_size);
|
1435
1528
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1436
1529
|
}
|
@@ -1475,15 +1568,116 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
|
|
1475
1568
|
}
|
1476
1569
|
BROTLI_FREE(m, tmp_command_buf);
|
1477
1570
|
BROTLI_FREE(m, tmp_literal_buf);
|
1571
|
+
CheckFlushComplete(s);
|
1572
|
+
return BROTLI_TRUE;
|
1573
|
+
}
|
1574
|
+
|
1575
|
+
static BROTLI_BOOL ProcessMetadata(
|
1576
|
+
BrotliEncoderState* s, size_t* available_in, const uint8_t** next_in,
|
1577
|
+
size_t* available_out, uint8_t** next_out, size_t* total_out) {
|
1578
|
+
if (*available_in > (1u << 24)) return BROTLI_FALSE;
|
1579
|
+
/* Switch to metadata block workflow, if required. */
|
1580
|
+
if (s->stream_state_ == BROTLI_STREAM_PROCESSING) {
|
1581
|
+
s->remaining_metadata_bytes_ = (uint32_t)*available_in;
|
1582
|
+
s->stream_state_ = BROTLI_STREAM_METADATA_HEAD;
|
1583
|
+
}
|
1584
|
+
if (s->stream_state_ != BROTLI_STREAM_METADATA_HEAD &&
|
1585
|
+
s->stream_state_ != BROTLI_STREAM_METADATA_BODY) {
|
1586
|
+
return BROTLI_FALSE;
|
1587
|
+
}
|
1588
|
+
|
1589
|
+
while (BROTLI_TRUE) {
|
1590
|
+
if (InjectFlushOrPushOutput(s, available_out, next_out, total_out)) {
|
1591
|
+
continue;
|
1592
|
+
}
|
1593
|
+
if (s->available_out_ != 0) break;
|
1594
|
+
|
1595
|
+
if (s->input_pos_ != s->last_flush_pos_) {
|
1596
|
+
BROTLI_BOOL result = EncodeData(s, BROTLI_FALSE, BROTLI_TRUE,
|
1597
|
+
&s->available_out_, &s->next_out_);
|
1598
|
+
if (!result) return BROTLI_FALSE;
|
1599
|
+
continue;
|
1600
|
+
}
|
1601
|
+
|
1602
|
+
if (s->stream_state_ == BROTLI_STREAM_METADATA_HEAD) {
|
1603
|
+
s->next_out_ = s->tiny_buf_.u8;
|
1604
|
+
s->available_out_ =
|
1605
|
+
WriteMetadataHeader(s, s->remaining_metadata_bytes_, s->next_out_);
|
1606
|
+
s->stream_state_ = BROTLI_STREAM_METADATA_BODY;
|
1607
|
+
continue;
|
1608
|
+
} else {
|
1609
|
+
/* Exit workflow only when there is no more input and no more output.
|
1610
|
+
Otherwise client may continue producing empty metadata blocks. */
|
1611
|
+
if (s->remaining_metadata_bytes_ == 0) {
|
1612
|
+
s->remaining_metadata_bytes_ = BROTLI_UINT32_MAX;
|
1613
|
+
s->stream_state_ = BROTLI_STREAM_PROCESSING;
|
1614
|
+
break;
|
1615
|
+
}
|
1616
|
+
if (*available_out) {
|
1617
|
+
/* Directly copy input to output. */
|
1618
|
+
uint32_t copy = (uint32_t)BROTLI_MIN(
|
1619
|
+
size_t, s->remaining_metadata_bytes_, *available_out);
|
1620
|
+
memcpy(*next_out, *next_in, copy);
|
1621
|
+
*next_in += copy;
|
1622
|
+
*available_in -= copy;
|
1623
|
+
s->remaining_metadata_bytes_ -= copy;
|
1624
|
+
*next_out += copy;
|
1625
|
+
*available_out -= copy;
|
1626
|
+
} else {
|
1627
|
+
/* This guarantees progress in "TakeOutput" workflow. */
|
1628
|
+
uint32_t copy = BROTLI_MIN(uint32_t, s->remaining_metadata_bytes_, 16);
|
1629
|
+
s->next_out_ = s->tiny_buf_.u8;
|
1630
|
+
memcpy(s->next_out_, *next_in, copy);
|
1631
|
+
*next_in += copy;
|
1632
|
+
*available_in -= copy;
|
1633
|
+
s->remaining_metadata_bytes_ -= copy;
|
1634
|
+
s->available_out_ = copy;
|
1635
|
+
}
|
1636
|
+
continue;
|
1637
|
+
}
|
1638
|
+
}
|
1639
|
+
|
1478
1640
|
return BROTLI_TRUE;
|
1479
1641
|
}
|
1480
1642
|
|
1643
|
+
static void UpdateSizeHint(BrotliEncoderState* s, size_t available_in) {
|
1644
|
+
if (s->params.size_hint == 0) {
|
1645
|
+
uint64_t delta = UnprocessedInputSize(s);
|
1646
|
+
uint64_t tail = available_in;
|
1647
|
+
uint32_t limit = 1u << 30;
|
1648
|
+
uint32_t total;
|
1649
|
+
if ((delta >= limit) || (tail >= limit) || ((delta + tail) >= limit)) {
|
1650
|
+
total = limit;
|
1651
|
+
} else {
|
1652
|
+
total = (uint32_t)(delta + tail);
|
1653
|
+
}
|
1654
|
+
s->params.size_hint = total;
|
1655
|
+
}
|
1656
|
+
}
|
1657
|
+
|
1481
1658
|
BROTLI_BOOL BrotliEncoderCompressStream(
|
1482
1659
|
BrotliEncoderState* s, BrotliEncoderOperation op, size_t* available_in,
|
1483
1660
|
const uint8_t** next_in, size_t* available_out,uint8_t** next_out,
|
1484
1661
|
size_t* total_out) {
|
1485
1662
|
if (!EnsureInitialized(s)) return BROTLI_FALSE;
|
1486
1663
|
|
1664
|
+
/* Unfinished metadata block; check requirements. */
|
1665
|
+
if (s->remaining_metadata_bytes_ != BROTLI_UINT32_MAX) {
|
1666
|
+
if (*available_in != s->remaining_metadata_bytes_) return BROTLI_FALSE;
|
1667
|
+
if (op != BROTLI_OPERATION_EMIT_METADATA) return BROTLI_FALSE;
|
1668
|
+
}
|
1669
|
+
|
1670
|
+
if (op == BROTLI_OPERATION_EMIT_METADATA) {
|
1671
|
+
UpdateSizeHint(s, 0); /* First data metablock might be emitted here. */
|
1672
|
+
return ProcessMetadata(
|
1673
|
+
s, available_in, next_in, available_out, next_out, total_out);
|
1674
|
+
}
|
1675
|
+
|
1676
|
+
if (s->stream_state_ == BROTLI_STREAM_METADATA_HEAD ||
|
1677
|
+
s->stream_state_ == BROTLI_STREAM_METADATA_BODY) {
|
1678
|
+
return BROTLI_FALSE;
|
1679
|
+
}
|
1680
|
+
|
1487
1681
|
if (s->stream_state_ != BROTLI_STREAM_PROCESSING && *available_in != 0) {
|
1488
1682
|
return BROTLI_FALSE;
|
1489
1683
|
}
|
@@ -1498,34 +1692,17 @@ BROTLI_BOOL BrotliEncoderCompressStream(
|
|
1498
1692
|
if (remaining_block_size != 0 && *available_in != 0) {
|
1499
1693
|
size_t copy_input_size =
|
1500
1694
|
BROTLI_MIN(size_t, remaining_block_size, *available_in);
|
1501
|
-
|
1695
|
+
CopyInputToRingBuffer(s, copy_input_size, *next_in);
|
1502
1696
|
*next_in += copy_input_size;
|
1503
1697
|
*available_in -= copy_input_size;
|
1504
1698
|
continue;
|
1505
1699
|
}
|
1506
1700
|
|
1507
|
-
if (s
|
1508
|
-
s->stream_state_ == BROTLI_STREAM_FLUSH_REQUESTED) {
|
1509
|
-
s->stream_state_ = BROTLI_STREAM_PROCESSING;
|
1510
|
-
if (s->last_byte_bits_ == 0) break;
|
1511
|
-
InjectBytePaddingBlock(s);
|
1701
|
+
if (InjectFlushOrPushOutput(s, available_out, next_out, total_out)) {
|
1512
1702
|
continue;
|
1513
1703
|
}
|
1514
1704
|
|
1515
|
-
|
1516
|
-
size_t copy_output_size =
|
1517
|
-
BROTLI_MIN(size_t, s->available_out_, *available_out);
|
1518
|
-
memcpy(*next_out, s->next_out_, copy_output_size);
|
1519
|
-
*next_out += copy_output_size;
|
1520
|
-
*available_out -= copy_output_size;
|
1521
|
-
s->next_out_ += copy_output_size;
|
1522
|
-
s->available_out_ -= copy_output_size;
|
1523
|
-
s->total_out_ += copy_output_size;
|
1524
|
-
if (total_out) *total_out = s->total_out_;
|
1525
|
-
continue;
|
1526
|
-
}
|
1527
|
-
|
1528
|
-
/* Compress data only when internal outpuf buffer is empty, stream is not
|
1705
|
+
/* Compress data only when internal output buffer is empty, stream is not
|
1529
1706
|
finished and there is no pending flush request. */
|
1530
1707
|
if (s->available_out_ == 0 &&
|
1531
1708
|
s->stream_state_ == BROTLI_STREAM_PROCESSING) {
|
@@ -1534,7 +1711,9 @@ BROTLI_BOOL BrotliEncoderCompressStream(
|
|
1534
1711
|
(*available_in == 0) && op == BROTLI_OPERATION_FINISH);
|
1535
1712
|
BROTLI_BOOL force_flush = TO_BROTLI_BOOL(
|
1536
1713
|
(*available_in == 0) && op == BROTLI_OPERATION_FLUSH);
|
1537
|
-
BROTLI_BOOL result
|
1714
|
+
BROTLI_BOOL result;
|
1715
|
+
UpdateSizeHint(s, *available_in);
|
1716
|
+
result = EncodeData(s, is_last, force_flush,
|
1538
1717
|
&s->available_out_, &s->next_out_);
|
1539
1718
|
if (!result) return BROTLI_FALSE;
|
1540
1719
|
if (force_flush) s->stream_state_ = BROTLI_STREAM_FLUSH_REQUESTED;
|
@@ -1544,6 +1723,7 @@ BROTLI_BOOL BrotliEncoderCompressStream(
|
|
1544
1723
|
}
|
1545
1724
|
break;
|
1546
1725
|
}
|
1726
|
+
CheckFlushComplete(s);
|
1547
1727
|
return BROTLI_TRUE;
|
1548
1728
|
}
|
1549
1729
|
|
@@ -1556,6 +1736,28 @@ BROTLI_BOOL BrotliEncoderHasMoreOutput(BrotliEncoderState* s) {
|
|
1556
1736
|
return TO_BROTLI_BOOL(s->available_out_ != 0);
|
1557
1737
|
}
|
1558
1738
|
|
1739
|
+
const uint8_t* BrotliEncoderTakeOutput(BrotliEncoderState* s, size_t* size) {
|
1740
|
+
size_t consumed_size = s->available_out_;
|
1741
|
+
uint8_t* result = s->next_out_;
|
1742
|
+
if (*size) {
|
1743
|
+
consumed_size = BROTLI_MIN(size_t, *size, s->available_out_);
|
1744
|
+
}
|
1745
|
+
if (consumed_size) {
|
1746
|
+
s->next_out_ += consumed_size;
|
1747
|
+
s->available_out_ -= consumed_size;
|
1748
|
+
s->total_out_ += consumed_size;
|
1749
|
+
CheckFlushComplete(s);
|
1750
|
+
*size = consumed_size;
|
1751
|
+
} else {
|
1752
|
+
*size = 0;
|
1753
|
+
result = 0;
|
1754
|
+
}
|
1755
|
+
return result;
|
1756
|
+
}
|
1757
|
+
|
1758
|
+
uint32_t BrotliEncoderVersion(void) {
|
1759
|
+
return BROTLI_VERSION;
|
1760
|
+
}
|
1559
1761
|
|
1560
1762
|
#if defined(__cplusplus) || defined(c_plusplus)
|
1561
1763
|
} /* extern "C" */
|