brotli 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +6 -3
- data/.github/workflows/publish.yml +7 -17
- data/.gitmodules +1 -1
- data/README.md +2 -2
- data/ext/brotli/brotli.c +1 -0
- data/ext/brotli/extconf.rb +6 -0
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +4 -1
- data/vendor/brotli/c/common/constants.c +1 -1
- data/vendor/brotli/c/common/constants.h +2 -1
- data/vendor/brotli/c/common/context.c +1 -1
- data/vendor/brotli/c/common/dictionary.c +5 -3
- data/vendor/brotli/c/common/platform.c +2 -1
- data/vendor/brotli/c/common/platform.h +60 -113
- data/vendor/brotli/c/common/shared_dictionary.c +521 -0
- data/vendor/brotli/c/common/shared_dictionary_internal.h +75 -0
- data/vendor/brotli/c/common/transform.c +1 -1
- data/vendor/brotli/c/common/version.h +31 -6
- data/vendor/brotli/c/dec/bit_reader.c +10 -8
- data/vendor/brotli/c/dec/bit_reader.h +172 -100
- data/vendor/brotli/c/dec/decode.c +467 -200
- data/vendor/brotli/c/dec/huffman.c +7 -4
- data/vendor/brotli/c/dec/huffman.h +2 -1
- data/vendor/brotli/c/dec/prefix.h +2 -1
- data/vendor/brotli/c/dec/state.c +33 -9
- data/vendor/brotli/c/dec/state.h +70 -35
- data/vendor/brotli/c/enc/backward_references.c +81 -19
- data/vendor/brotli/c/enc/backward_references.h +5 -4
- data/vendor/brotli/c/enc/backward_references_hq.c +148 -52
- data/vendor/brotli/c/enc/backward_references_hq.h +6 -5
- data/vendor/brotli/c/enc/backward_references_inc.h +31 -5
- data/vendor/brotli/c/enc/bit_cost.c +8 -7
- data/vendor/brotli/c/enc/bit_cost.h +5 -4
- data/vendor/brotli/c/enc/block_splitter.c +37 -14
- data/vendor/brotli/c/enc/block_splitter.h +5 -4
- data/vendor/brotli/c/enc/block_splitter_inc.h +86 -45
- data/vendor/brotli/c/enc/brotli_bit_stream.c +132 -110
- data/vendor/brotli/c/enc/brotli_bit_stream.h +11 -6
- data/vendor/brotli/c/enc/cluster.c +10 -9
- data/vendor/brotli/c/enc/cluster.h +7 -6
- data/vendor/brotli/c/enc/cluster_inc.h +25 -20
- data/vendor/brotli/c/enc/command.c +1 -1
- data/vendor/brotli/c/enc/command.h +5 -4
- data/vendor/brotli/c/enc/compound_dictionary.c +207 -0
- data/vendor/brotli/c/enc/compound_dictionary.h +74 -0
- data/vendor/brotli/c/enc/compress_fragment.c +93 -83
- data/vendor/brotli/c/enc/compress_fragment.h +32 -7
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +99 -87
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +21 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +3 -1
- data/vendor/brotli/c/enc/encode.c +473 -404
- data/vendor/brotli/c/enc/encoder_dict.c +611 -4
- data/vendor/brotli/c/enc/encoder_dict.h +117 -3
- data/vendor/brotli/c/enc/entropy_encode.c +3 -2
- data/vendor/brotli/c/enc/entropy_encode.h +2 -1
- data/vendor/brotli/c/enc/entropy_encode_static.h +5 -2
- data/vendor/brotli/c/enc/fast_log.c +1 -1
- data/vendor/brotli/c/enc/fast_log.h +2 -1
- data/vendor/brotli/c/enc/find_match_length.h +15 -22
- data/vendor/brotli/c/enc/hash.h +285 -45
- data/vendor/brotli/c/enc/hash_composite_inc.h +26 -11
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +20 -18
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +34 -39
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +6 -10
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +4 -4
- data/vendor/brotli/c/enc/hash_rolling_inc.h +4 -4
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +6 -5
- data/vendor/brotli/c/enc/histogram.c +4 -4
- data/vendor/brotli/c/enc/histogram.h +7 -6
- data/vendor/brotli/c/enc/literal_cost.c +20 -15
- data/vendor/brotli/c/enc/literal_cost.h +4 -2
- data/vendor/brotli/c/enc/memory.c +29 -5
- data/vendor/brotli/c/enc/memory.h +19 -2
- data/vendor/brotli/c/enc/metablock.c +72 -58
- data/vendor/brotli/c/enc/metablock.h +9 -8
- data/vendor/brotli/c/enc/metablock_inc.h +8 -6
- data/vendor/brotli/c/enc/params.h +4 -3
- data/vendor/brotli/c/enc/prefix.h +3 -2
- data/vendor/brotli/c/enc/quality.h +40 -3
- data/vendor/brotli/c/enc/ringbuffer.h +4 -3
- data/vendor/brotli/c/enc/state.h +104 -0
- data/vendor/brotli/c/enc/static_dict.c +60 -4
- data/vendor/brotli/c/enc/static_dict.h +3 -2
- data/vendor/brotli/c/enc/static_dict_lut.h +2 -0
- data/vendor/brotli/c/enc/utf8_util.c +1 -1
- data/vendor/brotli/c/enc/utf8_util.h +2 -1
- data/vendor/brotli/c/enc/write_bits.h +2 -1
- data/vendor/brotli/c/include/brotli/decode.h +67 -2
- data/vendor/brotli/c/include/brotli/encode.h +55 -2
- data/vendor/brotli/c/include/brotli/port.h +28 -11
- data/vendor/brotli/c/include/brotli/shared_dictionary.h +100 -0
- metadata +9 -3
@@ -7,6 +7,8 @@
|
|
7
7
|
/* Implementation of Brotli compressor. */
|
8
8
|
|
9
9
|
#include <brotli/encode.h>
|
10
|
+
#include <brotli/shared_dictionary.h>
|
11
|
+
#include <brotli/types.h>
|
10
12
|
|
11
13
|
#include <stdlib.h> /* free, malloc */
|
12
14
|
#include <string.h> /* memcpy, memset */
|
@@ -15,24 +17,26 @@
|
|
15
17
|
#include "../common/context.h"
|
16
18
|
#include "../common/platform.h"
|
17
19
|
#include "../common/version.h"
|
18
|
-
#include "
|
19
|
-
#include "
|
20
|
-
#include "
|
21
|
-
#include "
|
22
|
-
#include "
|
23
|
-
#include "
|
24
|
-
#include "
|
25
|
-
#include "
|
26
|
-
#include "
|
27
|
-
#include "
|
28
|
-
#include "
|
29
|
-
#include "
|
30
|
-
#include "
|
31
|
-
#include "
|
32
|
-
#include "
|
33
|
-
#include "
|
34
|
-
#include "
|
35
|
-
#include "
|
20
|
+
#include "backward_references.h"
|
21
|
+
#include "backward_references_hq.h"
|
22
|
+
#include "bit_cost.h"
|
23
|
+
#include "brotli_bit_stream.h"
|
24
|
+
#include "compress_fragment.h"
|
25
|
+
#include "compress_fragment_two_pass.h"
|
26
|
+
#include "dictionary_hash.h"
|
27
|
+
#include "encoder_dict.h"
|
28
|
+
#include "entropy_encode.h"
|
29
|
+
#include "fast_log.h"
|
30
|
+
#include "hash.h"
|
31
|
+
#include "histogram.h"
|
32
|
+
#include "memory.h"
|
33
|
+
#include "metablock.h"
|
34
|
+
#include "prefix.h"
|
35
|
+
#include "state.h"
|
36
|
+
#include "quality.h"
|
37
|
+
#include "ringbuffer.h"
|
38
|
+
#include "utf8_util.h"
|
39
|
+
#include "write_bits.h"
|
36
40
|
|
37
41
|
#if defined(__cplusplus) || defined(c_plusplus)
|
38
42
|
extern "C" {
|
@@ -40,94 +44,6 @@ extern "C" {
|
|
40
44
|
|
41
45
|
#define COPY_ARRAY(dst, src) memcpy(dst, src, sizeof(src));
|
42
46
|
|
43
|
-
typedef enum BrotliEncoderStreamState {
|
44
|
-
/* Default state. */
|
45
|
-
BROTLI_STREAM_PROCESSING = 0,
|
46
|
-
/* Intermediate state; after next block is emitted, byte-padding should be
|
47
|
-
performed before getting back to default state. */
|
48
|
-
BROTLI_STREAM_FLUSH_REQUESTED = 1,
|
49
|
-
/* Last metablock was produced; no more input is acceptable. */
|
50
|
-
BROTLI_STREAM_FINISHED = 2,
|
51
|
-
/* Flushing compressed block and writing meta-data block header. */
|
52
|
-
BROTLI_STREAM_METADATA_HEAD = 3,
|
53
|
-
/* Writing metadata block body. */
|
54
|
-
BROTLI_STREAM_METADATA_BODY = 4
|
55
|
-
} BrotliEncoderStreamState;
|
56
|
-
|
57
|
-
typedef enum BrotliEncoderFlintState {
|
58
|
-
BROTLI_FLINT_NEEDS_2_BYTES = 2,
|
59
|
-
BROTLI_FLINT_NEEDS_1_BYTE = 1,
|
60
|
-
BROTLI_FLINT_WAITING_FOR_PROCESSING = 0,
|
61
|
-
BROTLI_FLINT_WAITING_FOR_FLUSHING = -1,
|
62
|
-
BROTLI_FLINT_DONE = -2
|
63
|
-
} BrotliEncoderFlintState;
|
64
|
-
|
65
|
-
typedef struct BrotliEncoderStateStruct {
|
66
|
-
BrotliEncoderParams params;
|
67
|
-
|
68
|
-
MemoryManager memory_manager_;
|
69
|
-
|
70
|
-
uint64_t input_pos_;
|
71
|
-
RingBuffer ringbuffer_;
|
72
|
-
size_t cmd_alloc_size_;
|
73
|
-
Command* commands_;
|
74
|
-
size_t num_commands_;
|
75
|
-
size_t num_literals_;
|
76
|
-
size_t last_insert_len_;
|
77
|
-
uint64_t last_flush_pos_;
|
78
|
-
uint64_t last_processed_pos_;
|
79
|
-
int dist_cache_[BROTLI_NUM_DISTANCE_SHORT_CODES];
|
80
|
-
int saved_dist_cache_[4];
|
81
|
-
uint16_t last_bytes_;
|
82
|
-
uint8_t last_bytes_bits_;
|
83
|
-
/* "Flint" is a tiny uncompressed block emitted before the continuation
|
84
|
-
block to unwire literal context from previous data. Despite being int8_t,
|
85
|
-
field is actually BrotliEncoderFlintState enum. */
|
86
|
-
int8_t flint_;
|
87
|
-
uint8_t prev_byte_;
|
88
|
-
uint8_t prev_byte2_;
|
89
|
-
size_t storage_size_;
|
90
|
-
uint8_t* storage_;
|
91
|
-
|
92
|
-
Hasher hasher_;
|
93
|
-
|
94
|
-
/* Hash table for FAST_ONE_PASS_COMPRESSION_QUALITY mode. */
|
95
|
-
int small_table_[1 << 10]; /* 4KiB */
|
96
|
-
int* large_table_; /* Allocated only when needed */
|
97
|
-
size_t large_table_size_;
|
98
|
-
/* Command and distance prefix codes (each 64 symbols, stored back-to-back)
|
99
|
-
used for the next block in FAST_ONE_PASS_COMPRESSION_QUALITY. The command
|
100
|
-
prefix code is over a smaller alphabet with the following 64 symbols:
|
101
|
-
0 - 15: insert length code 0, copy length code 0 - 15, same distance
|
102
|
-
16 - 39: insert length code 0, copy length code 0 - 23
|
103
|
-
40 - 63: insert length code 0 - 23, copy length code 0
|
104
|
-
Note that symbols 16 and 40 represent the same code in the full alphabet,
|
105
|
-
but we do not use either of them in FAST_ONE_PASS_COMPRESSION_QUALITY. */
|
106
|
-
uint8_t cmd_depths_[128];
|
107
|
-
uint16_t cmd_bits_[128];
|
108
|
-
/* The compressed form of the command and distance prefix codes for the next
|
109
|
-
block in FAST_ONE_PASS_COMPRESSION_QUALITY. */
|
110
|
-
uint8_t cmd_code_[512];
|
111
|
-
size_t cmd_code_numbits_;
|
112
|
-
/* Command and literal buffers for FAST_TWO_PASS_COMPRESSION_QUALITY. */
|
113
|
-
uint32_t* command_buf_;
|
114
|
-
uint8_t* literal_buf_;
|
115
|
-
|
116
|
-
uint8_t* next_out_;
|
117
|
-
size_t available_out_;
|
118
|
-
size_t total_out_;
|
119
|
-
/* Temporary buffer for padding flush bits or metadata block header / body. */
|
120
|
-
union {
|
121
|
-
uint64_t u64[2];
|
122
|
-
uint8_t u8[16];
|
123
|
-
} tiny_buf_;
|
124
|
-
uint32_t remaining_metadata_bytes_;
|
125
|
-
BrotliEncoderStreamState stream_state_;
|
126
|
-
|
127
|
-
BROTLI_BOOL is_last_block_emitted_;
|
128
|
-
BROTLI_BOOL is_initialized_;
|
129
|
-
} BrotliEncoderStateStruct;
|
130
|
-
|
131
47
|
static size_t InputBlockSize(BrotliEncoderState* s) {
|
132
48
|
return (size_t)1 << s->params.lgblock;
|
133
49
|
}
|
@@ -147,7 +63,7 @@ BROTLI_BOOL BrotliEncoderSetParameter(
|
|
147
63
|
BrotliEncoderState* state, BrotliEncoderParameter p, uint32_t value) {
|
148
64
|
/* Changing parameters on the fly is not implemented yet. */
|
149
65
|
if (state->is_initialized_) return BROTLI_FALSE;
|
150
|
-
/* TODO: Validate/clamp parameters here. */
|
66
|
+
/* TODO(eustas): Validate/clamp parameters here. */
|
151
67
|
switch (p) {
|
152
68
|
case BROTLI_PARAM_MODE:
|
153
69
|
state->params.mode = (BrotliEncoderMode)value;
|
@@ -283,11 +199,9 @@ static void EncodeWindowBits(int lgwin, BROTLI_BOOL large_window,
|
|
283
199
|
}
|
284
200
|
}
|
285
201
|
|
202
|
+
/* TODO(eustas): move to compress_fragment.c? */
|
286
203
|
/* Initializes the command and distance prefix codes for the first block. */
|
287
|
-
static void InitCommandPrefixCodes(
|
288
|
-
uint16_t cmd_bits[128],
|
289
|
-
uint8_t cmd_code[512],
|
290
|
-
size_t* cmd_code_numbits) {
|
204
|
+
static void InitCommandPrefixCodes(BrotliOnePassArena* s) {
|
291
205
|
static const uint8_t kDefaultCommandDepths[128] = {
|
292
206
|
0, 4, 4, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8,
|
293
207
|
0, 0, 0, 4, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7,
|
@@ -320,13 +234,13 @@ static void InitCommandPrefixCodes(uint8_t cmd_depths[128],
|
|
320
234
|
0x46, 0xe1, 0xb0, 0xd0, 0x4e, 0xb2, 0xf7, 0x04, 0x00,
|
321
235
|
};
|
322
236
|
static const size_t kDefaultCommandCodeNumBits = 448;
|
323
|
-
COPY_ARRAY(
|
324
|
-
COPY_ARRAY(cmd_bits, kDefaultCommandBits);
|
237
|
+
COPY_ARRAY(s->cmd_depth, kDefaultCommandDepths);
|
238
|
+
COPY_ARRAY(s->cmd_bits, kDefaultCommandBits);
|
325
239
|
|
326
240
|
/* Initialize the pre-compressed form of the command and distance prefix
|
327
241
|
codes. */
|
328
|
-
COPY_ARRAY(cmd_code, kDefaultCommandCode);
|
329
|
-
|
242
|
+
COPY_ARRAY(s->cmd_code, kDefaultCommandCode);
|
243
|
+
s->cmd_code_numbits = kDefaultCommandCodeNumBits;
|
330
244
|
}
|
331
245
|
|
332
246
|
/* Decide about the context map based on the ability of the prediction
|
@@ -401,7 +315,8 @@ static void ChooseContextMap(int quality,
|
|
401
315
|
first 5 bits of literals. */
|
402
316
|
static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
|
403
317
|
size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
|
404
|
-
size_t* num_literal_contexts, const uint32_t** literal_context_map
|
318
|
+
size_t* num_literal_contexts, const uint32_t** literal_context_map,
|
319
|
+
uint32_t* arena) {
|
405
320
|
static const uint32_t kStaticContextMapComplexUTF8[64] = {
|
406
321
|
11, 11, 12, 12, /* 0 special */
|
407
322
|
0, 0, 0, 0, /* 4 lf */
|
@@ -426,16 +341,17 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
|
|
426
341
|
return BROTLI_FALSE;
|
427
342
|
} else {
|
428
343
|
const size_t end_pos = start_pos + length;
|
429
|
-
/* To make entropy calculations faster
|
430
|
-
|
344
|
+
/* To make entropy calculations faster, we collect histograms
|
345
|
+
over the 5 most significant bits of literals. One histogram
|
431
346
|
without context and 13 additional histograms for each context value. */
|
432
|
-
uint32_t combined_histo
|
433
|
-
uint32_t context_histo
|
347
|
+
uint32_t* BROTLI_RESTRICT const combined_histo = arena;
|
348
|
+
uint32_t* BROTLI_RESTRICT const context_histo = arena + 32;
|
434
349
|
uint32_t total = 0;
|
435
350
|
double entropy[3];
|
436
351
|
size_t dummy;
|
437
352
|
size_t i;
|
438
353
|
ContextLut utf8_lut = BROTLI_CONTEXT_LUT(CONTEXT_UTF8);
|
354
|
+
memset(arena, 0, sizeof(arena[0]) * 32 * 14);
|
439
355
|
for (; start_pos + 64 <= end_pos; start_pos += 4096) {
|
440
356
|
const size_t stride_end_pos = start_pos + 64;
|
441
357
|
uint8_t prev2 = input[start_pos & mask];
|
@@ -449,7 +365,7 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
|
|
449
365
|
BROTLI_CONTEXT(prev1, prev2, utf8_lut)];
|
450
366
|
++total;
|
451
367
|
++combined_histo[literal >> 3];
|
452
|
-
++context_histo[context
|
368
|
+
++context_histo[(context << 5) + (literal >> 3)];
|
453
369
|
prev2 = prev1;
|
454
370
|
prev1 = literal;
|
455
371
|
}
|
@@ -457,7 +373,7 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
|
|
457
373
|
entropy[1] = ShannonEntropy(combined_histo, 32, &dummy);
|
458
374
|
entropy[2] = 0;
|
459
375
|
for (i = 0; i < 13; ++i) {
|
460
|
-
entropy[2] += ShannonEntropy(
|
376
|
+
entropy[2] += ShannonEntropy(context_histo + (i << 5), 32, &dummy);
|
461
377
|
}
|
462
378
|
entropy[0] = 1.0 / (double)total;
|
463
379
|
entropy[1] *= entropy[0];
|
@@ -481,19 +397,21 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
|
|
481
397
|
|
482
398
|
static void DecideOverLiteralContextModeling(const uint8_t* input,
|
483
399
|
size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
|
484
|
-
size_t* num_literal_contexts, const uint32_t** literal_context_map
|
400
|
+
size_t* num_literal_contexts, const uint32_t** literal_context_map,
|
401
|
+
uint32_t* arena) {
|
485
402
|
if (quality < MIN_QUALITY_FOR_CONTEXT_MODELING || length < 64) {
|
486
403
|
return;
|
487
404
|
} else if (ShouldUseComplexStaticContextMap(
|
488
405
|
input, start_pos, length, mask, quality, size_hint,
|
489
|
-
num_literal_contexts, literal_context_map)) {
|
406
|
+
num_literal_contexts, literal_context_map, arena)) {
|
490
407
|
/* Context map was already set, nothing else to do. */
|
491
408
|
} else {
|
492
409
|
/* Gather bi-gram data of the UTF8 byte prefixes. To make the analysis of
|
493
410
|
UTF8 data faster we only examine 64 byte long strides at every 4kB
|
494
411
|
intervals. */
|
495
412
|
const size_t end_pos = start_pos + length;
|
496
|
-
uint32_t bigram_prefix_histo
|
413
|
+
uint32_t* BROTLI_RESTRICT const bigram_prefix_histo = arena;
|
414
|
+
memset(bigram_prefix_histo, 0, sizeof(arena[0]) * 9);
|
497
415
|
for (; start_pos + 64 <= end_pos; start_pos += 4096) {
|
498
416
|
static const int lut[4] = { 0, 0, 1, 2 };
|
499
417
|
const size_t stride_end_pos = start_pos + 64;
|
@@ -513,7 +431,7 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
|
|
513
431
|
static BROTLI_BOOL ShouldCompress(
|
514
432
|
const uint8_t* data, const size_t mask, const uint64_t last_flush_pos,
|
515
433
|
const size_t bytes, const size_t num_literals, const size_t num_commands) {
|
516
|
-
/* TODO: find more precise minimal block overhead. */
|
434
|
+
/* TODO(eustas): find more precise minimal block overhead. */
|
517
435
|
if (bytes <= 2) return BROTLI_FALSE;
|
518
436
|
if (num_commands < (bytes >> 8) + 2) {
|
519
437
|
if ((double)num_literals > 0.99 * (double)bytes) {
|
@@ -613,10 +531,14 @@ static void WriteMetaBlockInternal(MemoryManager* m,
|
|
613
531
|
size_t num_literal_contexts = 1;
|
614
532
|
const uint32_t* literal_context_map = NULL;
|
615
533
|
if (!params->disable_literal_context_modeling) {
|
534
|
+
/* TODO(eustas): pull to higher level and reuse. */
|
535
|
+
uint32_t* arena = BROTLI_ALLOC(m, uint32_t, 14 * 32);
|
536
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
|
616
537
|
DecideOverLiteralContextModeling(
|
617
538
|
data, wrapped_last_flush_pos, bytes, mask, params->quality,
|
618
539
|
params->size_hint, &num_literal_contexts,
|
619
|
-
&literal_context_map);
|
540
|
+
&literal_context_map, arena);
|
541
|
+
BROTLI_FREE(m, arena);
|
620
542
|
}
|
621
543
|
BrotliBuildMetaBlockGreedy(m, data, wrapped_last_flush_pos, mask,
|
622
544
|
prev_byte, prev_byte2, literal_context_lut, num_literal_contexts,
|
@@ -681,12 +603,13 @@ static void ChooseDistanceParams(BrotliEncoderParams* params) {
|
|
681
603
|
}
|
682
604
|
}
|
683
605
|
|
684
|
-
BrotliInitDistanceParams(
|
685
|
-
|
606
|
+
BrotliInitDistanceParams(¶ms->dist, distance_postfix_bits,
|
607
|
+
num_direct_distance_codes, params->large_window);
|
686
608
|
}
|
687
609
|
|
688
610
|
static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s) {
|
689
|
-
|
611
|
+
MemoryManager* m = &s->memory_manager_;
|
612
|
+
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
690
613
|
if (s->is_initialized_) return BROTLI_TRUE;
|
691
614
|
|
692
615
|
s->last_bytes_bits_ = 0;
|
@@ -728,8 +651,12 @@ static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s) {
|
|
728
651
|
}
|
729
652
|
|
730
653
|
if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
|
731
|
-
|
732
|
-
|
654
|
+
s->one_pass_arena_ = BROTLI_ALLOC(m, BrotliOnePassArena, 1);
|
655
|
+
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
656
|
+
InitCommandPrefixCodes(s->one_pass_arena_);
|
657
|
+
} else if (s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
|
658
|
+
s->two_pass_arena_ = BROTLI_ALLOC(m, BrotliTwoPassArena, 1);
|
659
|
+
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
733
660
|
}
|
734
661
|
|
735
662
|
s->is_initialized_ = BROTLI_TRUE;
|
@@ -745,7 +672,7 @@ static void BrotliEncoderInitParams(BrotliEncoderParams* params) {
|
|
745
672
|
params->stream_offset = 0;
|
746
673
|
params->size_hint = 0;
|
747
674
|
params->disable_literal_context_modeling = BROTLI_FALSE;
|
748
|
-
|
675
|
+
BrotliInitSharedEncoderDictionary(¶ms->dictionary);
|
749
676
|
params->dist.distance_postfix_bits = 0;
|
750
677
|
params->dist.num_direct_distance_codes = 0;
|
751
678
|
params->dist.alphabet_size_max =
|
@@ -754,6 +681,11 @@ static void BrotliEncoderInitParams(BrotliEncoderParams* params) {
|
|
754
681
|
params->dist.max_distance = BROTLI_MAX_DISTANCE;
|
755
682
|
}
|
756
683
|
|
684
|
+
static void BrotliEncoderCleanupParams(MemoryManager* m,
|
685
|
+
BrotliEncoderParams* params) {
|
686
|
+
BrotliCleanupSharedEncoderDictionary(m, ¶ms->dictionary);
|
687
|
+
}
|
688
|
+
|
757
689
|
static void BrotliEncoderInitState(BrotliEncoderState* s) {
|
758
690
|
BrotliEncoderInitParams(&s->params);
|
759
691
|
s->input_pos_ = 0;
|
@@ -769,9 +701,11 @@ static void BrotliEncoderInitState(BrotliEncoderState* s) {
|
|
769
701
|
HasherInit(&s->hasher_);
|
770
702
|
s->large_table_ = NULL;
|
771
703
|
s->large_table_size_ = 0;
|
772
|
-
s->
|
704
|
+
s->one_pass_arena_ = NULL;
|
705
|
+
s->two_pass_arena_ = NULL;
|
773
706
|
s->command_buf_ = NULL;
|
774
707
|
s->literal_buf_ = NULL;
|
708
|
+
s->total_in_ = 0;
|
775
709
|
s->next_out_ = NULL;
|
776
710
|
s->available_out_ = 0;
|
777
711
|
s->total_out_ = 0;
|
@@ -796,13 +730,9 @@ static void BrotliEncoderInitState(BrotliEncoderState* s) {
|
|
796
730
|
|
797
731
|
BrotliEncoderState* BrotliEncoderCreateInstance(
|
798
732
|
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
|
799
|
-
BrotliEncoderState* state =
|
800
|
-
|
801
|
-
|
802
|
-
} else if (alloc_func && free_func) {
|
803
|
-
state = (BrotliEncoderState*)alloc_func(opaque, sizeof(BrotliEncoderState));
|
804
|
-
}
|
805
|
-
if (state == 0) {
|
733
|
+
BrotliEncoderState* state = (BrotliEncoderState*)BrotliBootstrapAlloc(
|
734
|
+
sizeof(BrotliEncoderState), alloc_func, free_func, opaque);
|
735
|
+
if (state == NULL) {
|
806
736
|
/* BROTLI_DUMP(); */
|
807
737
|
return 0;
|
808
738
|
}
|
@@ -812,19 +742,36 @@ BrotliEncoderState* BrotliEncoderCreateInstance(
|
|
812
742
|
return state;
|
813
743
|
}
|
814
744
|
|
745
|
+
#ifdef BROTLI_REPORTING
|
746
|
+
/* When BROTLI_REPORTING is defined extra reporting module have to be linked. */
|
747
|
+
void BrotliEncoderOnFinish(const BrotliEncoderState* s);
|
748
|
+
#define BROTLI_ENCODER_ON_FINISH(s) BrotliEncoderOnFinish(s);
|
749
|
+
#else
|
750
|
+
#if !defined(BROTLI_ENCODER_ON_FINISH)
|
751
|
+
#define BROTLI_ENCODER_ON_FINISH(s) (void)(s);
|
752
|
+
#endif
|
753
|
+
#endif
|
754
|
+
|
815
755
|
static void BrotliEncoderCleanupState(BrotliEncoderState* s) {
|
816
756
|
MemoryManager* m = &s->memory_manager_;
|
757
|
+
|
758
|
+
BROTLI_ENCODER_ON_FINISH(s);
|
759
|
+
|
817
760
|
if (BROTLI_IS_OOM(m)) {
|
818
761
|
BrotliWipeOutMemoryManager(m);
|
819
762
|
return;
|
820
763
|
}
|
764
|
+
|
821
765
|
BROTLI_FREE(m, s->storage_);
|
822
766
|
BROTLI_FREE(m, s->commands_);
|
823
767
|
RingBufferFree(m, &s->ringbuffer_);
|
824
768
|
DestroyHasher(m, &s->hasher_);
|
825
769
|
BROTLI_FREE(m, s->large_table_);
|
770
|
+
BROTLI_FREE(m, s->one_pass_arena_);
|
771
|
+
BROTLI_FREE(m, s->two_pass_arena_);
|
826
772
|
BROTLI_FREE(m, s->command_buf_);
|
827
773
|
BROTLI_FREE(m, s->literal_buf_);
|
774
|
+
BrotliEncoderCleanupParams(m, &s->params);
|
828
775
|
}
|
829
776
|
|
830
777
|
/* Deinitializes and frees BrotliEncoderState instance. */
|
@@ -832,11 +779,8 @@ void BrotliEncoderDestroyInstance(BrotliEncoderState* state) {
|
|
832
779
|
if (!state) {
|
833
780
|
return;
|
834
781
|
} else {
|
835
|
-
MemoryManager* m = &state->memory_manager_;
|
836
|
-
brotli_free_func free_func = m->free_func;
|
837
|
-
void* opaque = m->opaque;
|
838
782
|
BrotliEncoderCleanupState(state);
|
839
|
-
|
783
|
+
BrotliBootstrapFree(state, &state->memory_manager_);
|
840
784
|
}
|
841
785
|
}
|
842
786
|
|
@@ -925,6 +869,8 @@ static void ExtendLastCommand(BrotliEncoderState* s, uint32_t* bytes,
|
|
925
869
|
uint64_t cmd_dist = (uint64_t)s->dist_cache_[0];
|
926
870
|
uint32_t distance_code = CommandRestoreDistanceCode(last_command,
|
927
871
|
&s->params.dist);
|
872
|
+
const CompoundDictionary* dict = &s->params.dictionary.compound;
|
873
|
+
size_t compound_dictionary_size = dict->total_size;
|
928
874
|
if (distance_code < BROTLI_NUM_DISTANCE_SHORT_CODES ||
|
929
875
|
distance_code - (BROTLI_NUM_DISTANCE_SHORT_CODES - 1) == cmd_dist) {
|
930
876
|
if (cmd_dist <= max_distance) {
|
@@ -935,6 +881,38 @@ static void ExtendLastCommand(BrotliEncoderState* s, uint32_t* bytes,
|
|
935
881
|
(*wrapped_last_processed_pos)++;
|
936
882
|
}
|
937
883
|
} else {
|
884
|
+
if ((cmd_dist - max_distance - 1) < compound_dictionary_size &&
|
885
|
+
last_copy_len < cmd_dist - max_distance) {
|
886
|
+
size_t address =
|
887
|
+
compound_dictionary_size - (size_t)(cmd_dist - max_distance) +
|
888
|
+
(size_t)last_copy_len;
|
889
|
+
size_t br_index = 0;
|
890
|
+
size_t br_offset;
|
891
|
+
const uint8_t* chunk;
|
892
|
+
size_t chunk_length;
|
893
|
+
while (address >= dict->chunk_offsets[br_index + 1]) br_index++;
|
894
|
+
br_offset = address - dict->chunk_offsets[br_index];
|
895
|
+
chunk = dict->chunk_source[br_index];
|
896
|
+
chunk_length =
|
897
|
+
dict->chunk_offsets[br_index + 1] - dict->chunk_offsets[br_index];
|
898
|
+
while (*bytes != 0 && data[*wrapped_last_processed_pos & mask] ==
|
899
|
+
chunk[br_offset]) {
|
900
|
+
last_command->copy_len_++;
|
901
|
+
(*bytes)--;
|
902
|
+
(*wrapped_last_processed_pos)++;
|
903
|
+
if (++br_offset == chunk_length) {
|
904
|
+
br_index++;
|
905
|
+
br_offset = 0;
|
906
|
+
if (br_index != dict->num_chunks) {
|
907
|
+
chunk = dict->chunk_source[br_index];
|
908
|
+
chunk_length = dict->chunk_offsets[br_index + 1] -
|
909
|
+
dict->chunk_offsets[br_index];
|
910
|
+
} else {
|
911
|
+
break;
|
912
|
+
}
|
913
|
+
}
|
914
|
+
}
|
915
|
+
}
|
938
916
|
}
|
939
917
|
/* The copy length is at most the metablock size, and thus expressible. */
|
940
918
|
GetLengthCode(last_command->insert_len_,
|
@@ -952,8 +930,8 @@ static void ExtendLastCommand(BrotliEncoderState* s, uint32_t* bytes,
|
|
952
930
|
If |*out_size| is positive, |*output| points to the start of the output
|
953
931
|
data. If |is_last| or |force_flush| is BROTLI_TRUE, an output meta-block is
|
954
932
|
always created. However, until |is_last| is BROTLI_TRUE encoder may retain up
|
955
|
-
to 7 bits of the last byte of output.
|
956
|
-
|
933
|
+
to 7 bits of the last byte of output. Byte-alignment could be enforced by
|
934
|
+
emitting an empty meta-data block.
|
957
935
|
Returns BROTLI_FALSE if the size of the input data is larger than
|
958
936
|
input_block_size().
|
959
937
|
*/
|
@@ -968,10 +946,39 @@ static BROTLI_BOOL EncodeData(
|
|
968
946
|
MemoryManager* m = &s->memory_manager_;
|
969
947
|
ContextType literal_context_mode;
|
970
948
|
ContextLut literal_context_lut;
|
949
|
+
BROTLI_BOOL fast_compress =
|
950
|
+
s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
|
951
|
+
s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY;
|
971
952
|
|
972
953
|
data = s->ringbuffer_.buffer_;
|
973
954
|
mask = s->ringbuffer_.mask_;
|
974
955
|
|
956
|
+
if (delta == 0) { /* No new input; still might want to flush or finish. */
|
957
|
+
if (!data) { /* No input has been processed so far. */
|
958
|
+
if (is_last) { /* Emit complete finalized stream. */
|
959
|
+
BROTLI_DCHECK(s->last_bytes_bits_ <= 14);
|
960
|
+
s->last_bytes_ |= (uint16_t)(3u << s->last_bytes_bits_);
|
961
|
+
s->last_bytes_bits_ = (uint8_t)(s->last_bytes_bits_ + 2u);
|
962
|
+
s->tiny_buf_.u8[0] = (uint8_t)s->last_bytes_;
|
963
|
+
s->tiny_buf_.u8[1] = (uint8_t)(s->last_bytes_ >> 8);
|
964
|
+
*output = s->tiny_buf_.u8;
|
965
|
+
*out_size = (s->last_bytes_bits_ + 7u) >> 3u;
|
966
|
+
return BROTLI_TRUE;
|
967
|
+
} else { /* No data, not last -> no-op. */
|
968
|
+
*out_size = 0;
|
969
|
+
return BROTLI_TRUE;
|
970
|
+
}
|
971
|
+
} else {
|
972
|
+
/* Fast compress performs flush every block -> flush is no-op. */
|
973
|
+
if (!is_last && (!force_flush || fast_compress)) { /* Another no-op. */
|
974
|
+
*out_size = 0;
|
975
|
+
return BROTLI_TRUE;
|
976
|
+
}
|
977
|
+
}
|
978
|
+
}
|
979
|
+
BROTLI_DCHECK(data);
|
980
|
+
|
981
|
+
if (s->params.quality > s->params.dictionary.max_quality) return BROTLI_FALSE;
|
975
982
|
/* Adding more blocks after "last" block is forbidden. */
|
976
983
|
if (s->is_last_block_emitted_) return BROTLI_FALSE;
|
977
984
|
if (is_last) s->is_last_block_emitted_ = BROTLI_TRUE;
|
@@ -991,19 +998,12 @@ static BROTLI_BOOL EncodeData(
|
|
991
998
|
}
|
992
999
|
}
|
993
1000
|
|
994
|
-
if (
|
995
|
-
s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
|
1001
|
+
if (fast_compress) {
|
996
1002
|
uint8_t* storage;
|
997
1003
|
size_t storage_ix = s->last_bytes_bits_;
|
998
1004
|
size_t table_size;
|
999
1005
|
int* table;
|
1000
1006
|
|
1001
|
-
if (delta == 0 && !is_last) {
|
1002
|
-
/* We have no new input data and we don't have to finish the stream, so
|
1003
|
-
nothing to do. */
|
1004
|
-
*out_size = 0;
|
1005
|
-
return BROTLI_TRUE;
|
1006
|
-
}
|
1007
1007
|
storage = GetBrotliStorage(s, 2 * bytes + 503);
|
1008
1008
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1009
1009
|
storage[0] = (uint8_t)s->last_bytes_;
|
@@ -1012,16 +1012,14 @@ static BROTLI_BOOL EncodeData(
|
|
1012
1012
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1013
1013
|
if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
|
1014
1014
|
BrotliCompressFragmentFast(
|
1015
|
-
|
1015
|
+
s->one_pass_arena_, &data[wrapped_last_processed_pos & mask],
|
1016
1016
|
bytes, is_last,
|
1017
1017
|
table, table_size,
|
1018
|
-
s->cmd_depths_, s->cmd_bits_,
|
1019
|
-
&s->cmd_code_numbits_, s->cmd_code_,
|
1020
1018
|
&storage_ix, storage);
|
1021
1019
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1022
1020
|
} else {
|
1023
1021
|
BrotliCompressFragmentTwoPass(
|
1024
|
-
|
1022
|
+
s->two_pass_arena_, &data[wrapped_last_processed_pos & mask],
|
1025
1023
|
bytes, is_last,
|
1026
1024
|
s->command_buf_, s->literal_buf_,
|
1027
1025
|
table, table_size,
|
@@ -1099,7 +1097,7 @@ static BROTLI_BOOL EncodeData(
|
|
1099
1097
|
const size_t max_commands = max_length / 8;
|
1100
1098
|
const size_t processed_bytes = (size_t)(s->input_pos_ - s->last_flush_pos_);
|
1101
1099
|
/* If maximal possible additional block doesn't fit metablock, flush now. */
|
1102
|
-
/* TODO: Postpone decision until next block arrives? */
|
1100
|
+
/* TODO(eustas): Postpone decision until next block arrives? */
|
1103
1101
|
const BROTLI_BOOL next_input_fits_metablock = TO_BROTLI_BOOL(
|
1104
1102
|
processed_bytes + InputBlockSize(s) <= max_length);
|
1105
1103
|
/* If block splitting is not used, then flush as soon as there is some
|
@@ -1192,7 +1190,7 @@ static size_t WriteMetadataHeader(
|
|
1192
1190
|
if (block_size == 0) {
|
1193
1191
|
BrotliWriteBits(2, 0, &storage_ix, header);
|
1194
1192
|
} else {
|
1195
|
-
uint32_t nbits = (block_size == 1) ?
|
1193
|
+
uint32_t nbits = (block_size == 1) ? 1 :
|
1196
1194
|
(Log2FloorNonZero((uint32_t)block_size - 1) + 1);
|
1197
1195
|
uint32_t nbytes = (nbits + 7) / 8;
|
1198
1196
|
BrotliWriteBits(2, nbytes, &storage_ix, header);
|
@@ -1201,228 +1199,6 @@ static size_t WriteMetadataHeader(
|
|
1201
1199
|
return (storage_ix + 7u) >> 3;
|
1202
1200
|
}
|
1203
1201
|
|
1204
|
-
static BROTLI_BOOL BrotliCompressBufferQuality10(
|
1205
|
-
int lgwin, size_t input_size, const uint8_t* input_buffer,
|
1206
|
-
size_t* encoded_size, uint8_t* encoded_buffer) {
|
1207
|
-
MemoryManager memory_manager;
|
1208
|
-
MemoryManager* m = &memory_manager;
|
1209
|
-
|
1210
|
-
const size_t mask = BROTLI_SIZE_MAX >> 1;
|
1211
|
-
int dist_cache[4] = { 4, 11, 15, 16 };
|
1212
|
-
int saved_dist_cache[4] = { 4, 11, 15, 16 };
|
1213
|
-
BROTLI_BOOL ok = BROTLI_TRUE;
|
1214
|
-
const size_t max_out_size = *encoded_size;
|
1215
|
-
size_t total_out_size = 0;
|
1216
|
-
uint16_t last_bytes;
|
1217
|
-
uint8_t last_bytes_bits;
|
1218
|
-
|
1219
|
-
const size_t hasher_eff_size = BROTLI_MIN(size_t,
|
1220
|
-
input_size, BROTLI_MAX_BACKWARD_LIMIT(lgwin) + BROTLI_WINDOW_GAP);
|
1221
|
-
|
1222
|
-
BrotliEncoderParams params;
|
1223
|
-
|
1224
|
-
const int lgmetablock = BROTLI_MIN(int, 24, lgwin + 1);
|
1225
|
-
size_t max_block_size;
|
1226
|
-
const size_t max_metablock_size = (size_t)1 << lgmetablock;
|
1227
|
-
const size_t max_literals_per_metablock = max_metablock_size / 8;
|
1228
|
-
const size_t max_commands_per_metablock = max_metablock_size / 8;
|
1229
|
-
size_t metablock_start = 0;
|
1230
|
-
uint8_t prev_byte = 0;
|
1231
|
-
uint8_t prev_byte2 = 0;
|
1232
|
-
|
1233
|
-
Hasher hasher;
|
1234
|
-
HasherInit(&hasher);
|
1235
|
-
|
1236
|
-
BrotliEncoderInitParams(¶ms);
|
1237
|
-
params.quality = 10;
|
1238
|
-
params.lgwin = lgwin;
|
1239
|
-
if (lgwin > BROTLI_MAX_WINDOW_BITS) {
|
1240
|
-
params.large_window = BROTLI_TRUE;
|
1241
|
-
}
|
1242
|
-
SanitizeParams(¶ms);
|
1243
|
-
params.lgblock = ComputeLgBlock(¶ms);
|
1244
|
-
ChooseDistanceParams(¶ms);
|
1245
|
-
max_block_size = (size_t)1 << params.lgblock;
|
1246
|
-
|
1247
|
-
BrotliInitMemoryManager(m, 0, 0, 0);
|
1248
|
-
|
1249
|
-
BROTLI_DCHECK(input_size <= mask + 1);
|
1250
|
-
EncodeWindowBits(lgwin, params.large_window, &last_bytes, &last_bytes_bits);
|
1251
|
-
InitOrStitchToPreviousBlock(m, &hasher, input_buffer, mask, ¶ms,
|
1252
|
-
0, hasher_eff_size, BROTLI_TRUE);
|
1253
|
-
if (BROTLI_IS_OOM(m)) goto oom;
|
1254
|
-
|
1255
|
-
while (ok && metablock_start < input_size) {
|
1256
|
-
const size_t metablock_end =
|
1257
|
-
BROTLI_MIN(size_t, input_size, metablock_start + max_metablock_size);
|
1258
|
-
const size_t expected_num_commands =
|
1259
|
-
(metablock_end - metablock_start) / 12 + 16;
|
1260
|
-
Command* commands = 0;
|
1261
|
-
size_t num_commands = 0;
|
1262
|
-
size_t last_insert_len = 0;
|
1263
|
-
size_t num_literals = 0;
|
1264
|
-
size_t metablock_size = 0;
|
1265
|
-
size_t cmd_alloc_size = 0;
|
1266
|
-
BROTLI_BOOL is_last;
|
1267
|
-
uint8_t* storage;
|
1268
|
-
size_t storage_ix;
|
1269
|
-
|
1270
|
-
ContextType literal_context_mode = ChooseContextMode(¶ms,
|
1271
|
-
input_buffer, metablock_start, mask, metablock_end - metablock_start);
|
1272
|
-
ContextLut literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
|
1273
|
-
|
1274
|
-
size_t block_start;
|
1275
|
-
for (block_start = metablock_start; block_start < metablock_end; ) {
|
1276
|
-
size_t block_size =
|
1277
|
-
BROTLI_MIN(size_t, metablock_end - block_start, max_block_size);
|
1278
|
-
ZopfliNode* nodes = BROTLI_ALLOC(m, ZopfliNode, block_size + 1);
|
1279
|
-
size_t path_size;
|
1280
|
-
size_t new_cmd_alloc_size;
|
1281
|
-
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(nodes)) goto oom;
|
1282
|
-
BrotliInitZopfliNodes(nodes, block_size + 1);
|
1283
|
-
StitchToPreviousBlockH10(&hasher.privat._H10, block_size, block_start,
|
1284
|
-
input_buffer, mask);
|
1285
|
-
path_size = BrotliZopfliComputeShortestPath(m, block_size, block_start,
|
1286
|
-
input_buffer, mask, literal_context_lut, ¶ms, dist_cache, &hasher,
|
1287
|
-
nodes);
|
1288
|
-
if (BROTLI_IS_OOM(m)) goto oom;
|
1289
|
-
/* We allocate a command buffer in the first iteration of this loop that
|
1290
|
-
will be likely big enough for the whole metablock, so that for most
|
1291
|
-
inputs we will not have to reallocate in later iterations. We do the
|
1292
|
-
allocation here and not before the loop, because if the input is small,
|
1293
|
-
this will be allocated after the Zopfli cost model is freed, so this
|
1294
|
-
will not increase peak memory usage.
|
1295
|
-
TODO: If the first allocation is too small, increase command
|
1296
|
-
buffer size exponentially. */
|
1297
|
-
new_cmd_alloc_size = BROTLI_MAX(size_t, expected_num_commands,
|
1298
|
-
num_commands + path_size + 1);
|
1299
|
-
if (cmd_alloc_size != new_cmd_alloc_size) {
|
1300
|
-
Command* new_commands = BROTLI_ALLOC(m, Command, new_cmd_alloc_size);
|
1301
|
-
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_commands)) goto oom;
|
1302
|
-
cmd_alloc_size = new_cmd_alloc_size;
|
1303
|
-
if (commands) {
|
1304
|
-
memcpy(new_commands, commands, sizeof(Command) * num_commands);
|
1305
|
-
BROTLI_FREE(m, commands);
|
1306
|
-
}
|
1307
|
-
commands = new_commands;
|
1308
|
-
}
|
1309
|
-
BrotliZopfliCreateCommands(block_size, block_start, &nodes[0], dist_cache,
|
1310
|
-
&last_insert_len, ¶ms, &commands[num_commands], &num_literals);
|
1311
|
-
num_commands += path_size;
|
1312
|
-
block_start += block_size;
|
1313
|
-
metablock_size += block_size;
|
1314
|
-
BROTLI_FREE(m, nodes);
|
1315
|
-
if (num_literals > max_literals_per_metablock ||
|
1316
|
-
num_commands > max_commands_per_metablock) {
|
1317
|
-
break;
|
1318
|
-
}
|
1319
|
-
}
|
1320
|
-
|
1321
|
-
if (last_insert_len > 0) {
|
1322
|
-
InitInsertCommand(&commands[num_commands++], last_insert_len);
|
1323
|
-
num_literals += last_insert_len;
|
1324
|
-
}
|
1325
|
-
|
1326
|
-
is_last = TO_BROTLI_BOOL(metablock_start + metablock_size == input_size);
|
1327
|
-
storage = NULL;
|
1328
|
-
storage_ix = last_bytes_bits;
|
1329
|
-
|
1330
|
-
if (metablock_size == 0) {
|
1331
|
-
/* Write the ISLAST and ISEMPTY bits. */
|
1332
|
-
storage = BROTLI_ALLOC(m, uint8_t, 16);
|
1333
|
-
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(storage)) goto oom;
|
1334
|
-
storage[0] = (uint8_t)last_bytes;
|
1335
|
-
storage[1] = (uint8_t)(last_bytes >> 8);
|
1336
|
-
BrotliWriteBits(2, 3, &storage_ix, storage);
|
1337
|
-
storage_ix = (storage_ix + 7u) & ~7u;
|
1338
|
-
} else if (!ShouldCompress(input_buffer, mask, metablock_start,
|
1339
|
-
metablock_size, num_literals, num_commands)) {
|
1340
|
-
/* Restore the distance cache, as its last update by
|
1341
|
-
CreateBackwardReferences is now unused. */
|
1342
|
-
memcpy(dist_cache, saved_dist_cache, 4 * sizeof(dist_cache[0]));
|
1343
|
-
storage = BROTLI_ALLOC(m, uint8_t, metablock_size + 16);
|
1344
|
-
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(storage)) goto oom;
|
1345
|
-
storage[0] = (uint8_t)last_bytes;
|
1346
|
-
storage[1] = (uint8_t)(last_bytes >> 8);
|
1347
|
-
BrotliStoreUncompressedMetaBlock(is_last, input_buffer,
|
1348
|
-
metablock_start, mask, metablock_size,
|
1349
|
-
&storage_ix, storage);
|
1350
|
-
} else {
|
1351
|
-
MetaBlockSplit mb;
|
1352
|
-
BrotliEncoderParams block_params = params;
|
1353
|
-
InitMetaBlockSplit(&mb);
|
1354
|
-
BrotliBuildMetaBlock(m, input_buffer, metablock_start, mask,
|
1355
|
-
&block_params,
|
1356
|
-
prev_byte, prev_byte2,
|
1357
|
-
commands, num_commands,
|
1358
|
-
literal_context_mode,
|
1359
|
-
&mb);
|
1360
|
-
if (BROTLI_IS_OOM(m)) goto oom;
|
1361
|
-
{
|
1362
|
-
/* The number of distance symbols effectively used for distance
|
1363
|
-
histograms. It might be less than distance alphabet size
|
1364
|
-
for "Large Window Brotli" (32-bit). */
|
1365
|
-
BrotliOptimizeHistograms(block_params.dist.alphabet_size_limit, &mb);
|
1366
|
-
}
|
1367
|
-
storage = BROTLI_ALLOC(m, uint8_t, 2 * metablock_size + 503);
|
1368
|
-
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(storage)) goto oom;
|
1369
|
-
storage[0] = (uint8_t)last_bytes;
|
1370
|
-
storage[1] = (uint8_t)(last_bytes >> 8);
|
1371
|
-
BrotliStoreMetaBlock(m, input_buffer, metablock_start, metablock_size,
|
1372
|
-
mask, prev_byte, prev_byte2,
|
1373
|
-
is_last,
|
1374
|
-
&block_params,
|
1375
|
-
literal_context_mode,
|
1376
|
-
commands, num_commands,
|
1377
|
-
&mb,
|
1378
|
-
&storage_ix, storage);
|
1379
|
-
if (BROTLI_IS_OOM(m)) goto oom;
|
1380
|
-
if (metablock_size + 4 < (storage_ix >> 3)) {
|
1381
|
-
/* Restore the distance cache and last byte. */
|
1382
|
-
memcpy(dist_cache, saved_dist_cache, 4 * sizeof(dist_cache[0]));
|
1383
|
-
storage[0] = (uint8_t)last_bytes;
|
1384
|
-
storage[1] = (uint8_t)(last_bytes >> 8);
|
1385
|
-
storage_ix = last_bytes_bits;
|
1386
|
-
BrotliStoreUncompressedMetaBlock(is_last, input_buffer,
|
1387
|
-
metablock_start, mask,
|
1388
|
-
metablock_size, &storage_ix, storage);
|
1389
|
-
}
|
1390
|
-
DestroyMetaBlockSplit(m, &mb);
|
1391
|
-
}
|
1392
|
-
last_bytes = (uint16_t)(storage[storage_ix >> 3]);
|
1393
|
-
last_bytes_bits = storage_ix & 7u;
|
1394
|
-
metablock_start += metablock_size;
|
1395
|
-
if (metablock_start < input_size) {
|
1396
|
-
prev_byte = input_buffer[metablock_start - 1];
|
1397
|
-
prev_byte2 = input_buffer[metablock_start - 2];
|
1398
|
-
}
|
1399
|
-
/* Save the state of the distance cache in case we need to restore it for
|
1400
|
-
emitting an uncompressed block. */
|
1401
|
-
memcpy(saved_dist_cache, dist_cache, 4 * sizeof(dist_cache[0]));
|
1402
|
-
|
1403
|
-
{
|
1404
|
-
const size_t out_size = storage_ix >> 3;
|
1405
|
-
total_out_size += out_size;
|
1406
|
-
if (total_out_size <= max_out_size) {
|
1407
|
-
memcpy(encoded_buffer, storage, out_size);
|
1408
|
-
encoded_buffer += out_size;
|
1409
|
-
} else {
|
1410
|
-
ok = BROTLI_FALSE;
|
1411
|
-
}
|
1412
|
-
}
|
1413
|
-
BROTLI_FREE(m, storage);
|
1414
|
-
BROTLI_FREE(m, commands);
|
1415
|
-
}
|
1416
|
-
|
1417
|
-
*encoded_size = total_out_size;
|
1418
|
-
DestroyHasher(m, &hasher);
|
1419
|
-
return ok;
|
1420
|
-
|
1421
|
-
oom:
|
1422
|
-
BrotliWipeOutMemoryManager(m);
|
1423
|
-
return BROTLI_FALSE;
|
1424
|
-
}
|
1425
|
-
|
1426
1202
|
size_t BrotliEncoderMaxCompressedSize(size_t input_size) {
|
1427
1203
|
/* [window bits / empty metadata] + N * [uncompressed] + [last empty] */
|
1428
1204
|
size_t num_large_blocks = input_size >> 14;
|
@@ -1470,8 +1246,9 @@ static size_t MakeUncompressedStream(
|
|
1470
1246
|
|
1471
1247
|
BROTLI_BOOL BrotliEncoderCompress(
|
1472
1248
|
int quality, int lgwin, BrotliEncoderMode mode, size_t input_size,
|
1473
|
-
const uint8_t
|
1474
|
-
|
1249
|
+
const uint8_t input_buffer[BROTLI_ARRAY_PARAM(input_size)],
|
1250
|
+
size_t* encoded_size,
|
1251
|
+
uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(*encoded_size)]) {
|
1475
1252
|
BrotliEncoderState* s;
|
1476
1253
|
size_t out_size = *encoded_size;
|
1477
1254
|
const uint8_t* input_start = input_buffer;
|
@@ -1487,17 +1264,6 @@ BROTLI_BOOL BrotliEncoderCompress(
|
|
1487
1264
|
*encoded_buffer = 6;
|
1488
1265
|
return BROTLI_TRUE;
|
1489
1266
|
}
|
1490
|
-
if (quality == 10) {
|
1491
|
-
/* TODO: Implement this direct path for all quality levels. */
|
1492
|
-
const int lg_win = BROTLI_MIN(int, BROTLI_LARGE_MAX_WINDOW_BITS,
|
1493
|
-
BROTLI_MAX(int, 16, lgwin));
|
1494
|
-
int ok = BrotliCompressBufferQuality10(lg_win, input_size, input_buffer,
|
1495
|
-
encoded_size, encoded_buffer);
|
1496
|
-
if (!ok || (max_out_size && *encoded_size > max_out_size)) {
|
1497
|
-
goto fallback;
|
1498
|
-
}
|
1499
|
-
return BROTLI_TRUE;
|
1500
|
-
}
|
1501
1267
|
|
1502
1268
|
s = BrotliEncoderCreateInstance(0, 0, 0);
|
1503
1269
|
if (!s) {
|
@@ -1509,6 +1275,7 @@ BROTLI_BOOL BrotliEncoderCompress(
|
|
1509
1275
|
uint8_t* next_out = encoded_buffer;
|
1510
1276
|
size_t total_out = 0;
|
1511
1277
|
BROTLI_BOOL result = BROTLI_FALSE;
|
1278
|
+
/* TODO(eustas): check that parameters are sane. */
|
1512
1279
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)quality);
|
1513
1280
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)lgwin);
|
1514
1281
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_MODE, (uint32_t)mode);
|
@@ -1560,6 +1327,18 @@ static void InjectBytePaddingBlock(BrotliEncoderState* s) {
|
|
1560
1327
|
s->available_out_ += (seal_bits + 7) >> 3;
|
1561
1328
|
}
|
1562
1329
|
|
1330
|
+
/* Fills the |total_out|, if it is not NULL. */
|
1331
|
+
static void SetTotalOut(BrotliEncoderState* s, size_t* total_out) {
|
1332
|
+
if (total_out) {
|
1333
|
+
/* Saturating conversion uint64_t -> size_t */
|
1334
|
+
size_t result = (size_t)-1;
|
1335
|
+
if (s->total_out_ < result) {
|
1336
|
+
result = (size_t)s->total_out_;
|
1337
|
+
}
|
1338
|
+
*total_out = result;
|
1339
|
+
}
|
1340
|
+
}
|
1341
|
+
|
1563
1342
|
/* Injects padding bits or pushes compressed data to output.
|
1564
1343
|
Returns false if nothing is done. */
|
1565
1344
|
static BROTLI_BOOL InjectFlushOrPushOutput(BrotliEncoderState* s,
|
@@ -1579,7 +1358,7 @@ static BROTLI_BOOL InjectFlushOrPushOutput(BrotliEncoderState* s,
|
|
1579
1358
|
s->next_out_ += copy_output_size;
|
1580
1359
|
s->available_out_ -= copy_output_size;
|
1581
1360
|
s->total_out_ += copy_output_size;
|
1582
|
-
|
1361
|
+
SetTotalOut(s, total_out);
|
1583
1362
|
return BROTLI_TRUE;
|
1584
1363
|
}
|
1585
1364
|
|
@@ -1676,19 +1455,19 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
|
|
1676
1455
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1677
1456
|
|
1678
1457
|
if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
|
1679
|
-
BrotliCompressFragmentFast(
|
1680
|
-
|
1681
|
-
s->cmd_code_, &storage_ix, storage);
|
1458
|
+
BrotliCompressFragmentFast(s->one_pass_arena_, *next_in, block_size,
|
1459
|
+
is_last, table, table_size, &storage_ix, storage);
|
1682
1460
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1683
1461
|
} else {
|
1684
|
-
BrotliCompressFragmentTwoPass(
|
1685
|
-
command_buf, literal_buf, table, table_size,
|
1462
|
+
BrotliCompressFragmentTwoPass(s->two_pass_arena_, *next_in, block_size,
|
1463
|
+
is_last, command_buf, literal_buf, table, table_size,
|
1686
1464
|
&storage_ix, storage);
|
1687
1465
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1688
1466
|
}
|
1689
1467
|
if (block_size != 0) {
|
1690
1468
|
*next_in += block_size;
|
1691
1469
|
*available_in -= block_size;
|
1470
|
+
s->total_in_ += block_size;
|
1692
1471
|
}
|
1693
1472
|
if (inplace) {
|
1694
1473
|
size_t out_bytes = storage_ix >> 3;
|
@@ -1697,7 +1476,7 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
|
|
1697
1476
|
*next_out += out_bytes;
|
1698
1477
|
*available_out -= out_bytes;
|
1699
1478
|
s->total_out_ += out_bytes;
|
1700
|
-
|
1479
|
+
SetTotalOut(s, total_out);
|
1701
1480
|
} else {
|
1702
1481
|
size_t out_bytes = storage_ix >> 3;
|
1703
1482
|
s->next_out_ = storage;
|
@@ -1766,6 +1545,7 @@ static BROTLI_BOOL ProcessMetadata(
|
|
1766
1545
|
memcpy(*next_out, *next_in, copy);
|
1767
1546
|
*next_in += copy;
|
1768
1547
|
*available_in -= copy;
|
1548
|
+
s->total_in_ += copy; /* not actually data input, though */
|
1769
1549
|
s->remaining_metadata_bytes_ -= copy;
|
1770
1550
|
*next_out += copy;
|
1771
1551
|
*available_out -= copy;
|
@@ -1776,6 +1556,7 @@ static BROTLI_BOOL ProcessMetadata(
|
|
1776
1556
|
memcpy(s->next_out_, *next_in, copy);
|
1777
1557
|
*next_in += copy;
|
1778
1558
|
*available_in -= copy;
|
1559
|
+
s->total_in_ += copy; /* not actually data input, though */
|
1779
1560
|
s->remaining_metadata_bytes_ -= copy;
|
1780
1561
|
s->available_out_ = copy;
|
1781
1562
|
}
|
@@ -1803,7 +1584,7 @@ static void UpdateSizeHint(BrotliEncoderState* s, size_t available_in) {
|
|
1803
1584
|
|
1804
1585
|
BROTLI_BOOL BrotliEncoderCompressStream(
|
1805
1586
|
BrotliEncoderState* s, BrotliEncoderOperation op, size_t* available_in,
|
1806
|
-
const uint8_t** next_in, size_t* available_out,uint8_t** next_out,
|
1587
|
+
const uint8_t** next_in, size_t* available_out, uint8_t** next_out,
|
1807
1588
|
size_t* total_out) {
|
1808
1589
|
if (!EnsureInitialized(s)) return BROTLI_FALSE;
|
1809
1590
|
|
@@ -1845,6 +1626,7 @@ BROTLI_BOOL BrotliEncoderCompressStream(
|
|
1845
1626
|
CopyInputToRingBuffer(s, copy_input_size, *next_in);
|
1846
1627
|
*next_in += copy_input_size;
|
1847
1628
|
*available_in -= copy_input_size;
|
1629
|
+
s->total_in_ += copy_input_size;
|
1848
1630
|
if (s->flint_ > 0) s->flint_ = (int8_t)(s->flint_ - (int)copy_input_size);
|
1849
1631
|
continue;
|
1850
1632
|
}
|
@@ -1922,6 +1704,293 @@ uint32_t BrotliEncoderVersion(void) {
|
|
1922
1704
|
return BROTLI_VERSION;
|
1923
1705
|
}
|
1924
1706
|
|
1707
|
+
BrotliEncoderPreparedDictionary* BrotliEncoderPrepareDictionary(
|
1708
|
+
BrotliSharedDictionaryType type, size_t size,
|
1709
|
+
const uint8_t data[BROTLI_ARRAY_PARAM(size)], int quality,
|
1710
|
+
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
|
1711
|
+
ManagedDictionary* managed_dictionary = NULL;
|
1712
|
+
BROTLI_BOOL type_is_known = BROTLI_FALSE;
|
1713
|
+
type_is_known |= (type == BROTLI_SHARED_DICTIONARY_RAW);
|
1714
|
+
#if defined(BROTLI_EXPERIMENTAL)
|
1715
|
+
type_is_known |= (type == BROTLI_SHARED_DICTIONARY_SERIALIZED);
|
1716
|
+
#endif /* BROTLI_EXPERIMENTAL */
|
1717
|
+
if (!type_is_known) {
|
1718
|
+
return NULL;
|
1719
|
+
}
|
1720
|
+
managed_dictionary =
|
1721
|
+
BrotliCreateManagedDictionary(alloc_func, free_func, opaque);
|
1722
|
+
if (managed_dictionary == NULL) {
|
1723
|
+
return NULL;
|
1724
|
+
}
|
1725
|
+
if (type == BROTLI_SHARED_DICTIONARY_RAW) {
|
1726
|
+
managed_dictionary->dictionary = (uint32_t*)CreatePreparedDictionary(
|
1727
|
+
&managed_dictionary->memory_manager_, data, size);
|
1728
|
+
}
|
1729
|
+
#if defined(BROTLI_EXPERIMENTAL)
|
1730
|
+
if (type == BROTLI_SHARED_DICTIONARY_SERIALIZED) {
|
1731
|
+
SharedEncoderDictionary* dict = (SharedEncoderDictionary*)BrotliAllocate(
|
1732
|
+
&managed_dictionary->memory_manager_, sizeof(SharedEncoderDictionary));
|
1733
|
+
managed_dictionary->dictionary = (uint32_t*)dict;
|
1734
|
+
if (dict != NULL) {
|
1735
|
+
BROTLI_BOOL ok = BrotliInitCustomSharedEncoderDictionary(
|
1736
|
+
&managed_dictionary->memory_manager_, data, size, quality, dict);
|
1737
|
+
if (!ok) {
|
1738
|
+
BrotliFree(&managed_dictionary->memory_manager_, dict);
|
1739
|
+
managed_dictionary->dictionary = NULL;
|
1740
|
+
}
|
1741
|
+
}
|
1742
|
+
}
|
1743
|
+
#else /* BROTLI_EXPERIMENTAL */
|
1744
|
+
(void)quality;
|
1745
|
+
#endif /* BROTLI_EXPERIMENTAL */
|
1746
|
+
if (managed_dictionary->dictionary == NULL) {
|
1747
|
+
BrotliDestroyManagedDictionary(managed_dictionary);
|
1748
|
+
return NULL;
|
1749
|
+
}
|
1750
|
+
return (BrotliEncoderPreparedDictionary*)managed_dictionary;
|
1751
|
+
}
|
1752
|
+
|
1753
|
+
void BrotliEncoderDestroyPreparedDictionary(
|
1754
|
+
BrotliEncoderPreparedDictionary* dictionary) {
|
1755
|
+
ManagedDictionary* dict = (ManagedDictionary*)dictionary;
|
1756
|
+
if (!dictionary) return;
|
1757
|
+
/* First field of dictionary structs. */
|
1758
|
+
/* Only managed dictionaries are eligible for destruction by this method. */
|
1759
|
+
if (dict->magic != kManagedDictionaryMagic) {
|
1760
|
+
return;
|
1761
|
+
}
|
1762
|
+
if (dict->dictionary == NULL) {
|
1763
|
+
/* This should never ever happen. */
|
1764
|
+
} else if (*dict->dictionary == kLeanPreparedDictionaryMagic) {
|
1765
|
+
DestroyPreparedDictionary(
|
1766
|
+
&dict->memory_manager_, (PreparedDictionary*)dict->dictionary);
|
1767
|
+
} else if (*dict->dictionary == kSharedDictionaryMagic) {
|
1768
|
+
BrotliCleanupSharedEncoderDictionary(&dict->memory_manager_,
|
1769
|
+
(SharedEncoderDictionary*)dict->dictionary);
|
1770
|
+
BrotliFree(&dict->memory_manager_, dict->dictionary);
|
1771
|
+
} else {
|
1772
|
+
/* There is also kPreparedDictionaryMagic, but such instances should be
|
1773
|
+
* constructed and destroyed by different means. */
|
1774
|
+
}
|
1775
|
+
dict->dictionary = NULL;
|
1776
|
+
BrotliDestroyManagedDictionary(dict);
|
1777
|
+
}
|
1778
|
+
|
1779
|
+
BROTLI_BOOL BrotliEncoderAttachPreparedDictionary(BrotliEncoderState* state,
|
1780
|
+
const BrotliEncoderPreparedDictionary* dictionary) {
|
1781
|
+
/* First field of dictionary structs */
|
1782
|
+
const BrotliEncoderPreparedDictionary* dict = dictionary;
|
1783
|
+
uint32_t magic = *((const uint32_t*)dict);
|
1784
|
+
SharedEncoderDictionary* current = NULL;
|
1785
|
+
if (magic == kManagedDictionaryMagic) {
|
1786
|
+
/* Unwrap managed dictionary. */
|
1787
|
+
ManagedDictionary* managed_dictionary = (ManagedDictionary*)dict;
|
1788
|
+
magic = *managed_dictionary->dictionary;
|
1789
|
+
dict = (BrotliEncoderPreparedDictionary*)managed_dictionary->dictionary;
|
1790
|
+
}
|
1791
|
+
current = &state->params.dictionary;
|
1792
|
+
if (magic == kPreparedDictionaryMagic ||
|
1793
|
+
magic == kLeanPreparedDictionaryMagic) {
|
1794
|
+
const PreparedDictionary* prepared = (const PreparedDictionary*)dict;
|
1795
|
+
if (!AttachPreparedDictionary(¤t->compound, prepared)) {
|
1796
|
+
return BROTLI_FALSE;
|
1797
|
+
}
|
1798
|
+
} else if (magic == kSharedDictionaryMagic) {
|
1799
|
+
const SharedEncoderDictionary* attached =
|
1800
|
+
(const SharedEncoderDictionary*)dict;
|
1801
|
+
BROTLI_BOOL was_default = !current->contextual.context_based &&
|
1802
|
+
current->contextual.num_dictionaries == 1 &&
|
1803
|
+
current->contextual.dict[0]->hash_table_words ==
|
1804
|
+
kStaticDictionaryHashWords &&
|
1805
|
+
current->contextual.dict[0]->hash_table_lengths ==
|
1806
|
+
kStaticDictionaryHashLengths;
|
1807
|
+
BROTLI_BOOL new_default = !attached->contextual.context_based &&
|
1808
|
+
attached->contextual.num_dictionaries == 1 &&
|
1809
|
+
attached->contextual.dict[0]->hash_table_words ==
|
1810
|
+
kStaticDictionaryHashWords &&
|
1811
|
+
attached->contextual.dict[0]->hash_table_lengths ==
|
1812
|
+
kStaticDictionaryHashLengths;
|
1813
|
+
size_t i;
|
1814
|
+
if (state->is_initialized_) return BROTLI_FALSE;
|
1815
|
+
current->max_quality =
|
1816
|
+
BROTLI_MIN(int, current->max_quality, attached->max_quality);
|
1817
|
+
for (i = 0; i < attached->compound.num_chunks; i++) {
|
1818
|
+
if (!AttachPreparedDictionary(¤t->compound,
|
1819
|
+
attached->compound.chunks[i])) {
|
1820
|
+
return BROTLI_FALSE;
|
1821
|
+
}
|
1822
|
+
}
|
1823
|
+
if (!new_default) {
|
1824
|
+
if (!was_default) return BROTLI_FALSE;
|
1825
|
+
/* Copy by value, but then set num_instances_ to 0 because their memory
|
1826
|
+
is managed by attached, not by current */
|
1827
|
+
current->contextual = attached->contextual;
|
1828
|
+
current->contextual.num_instances_ = 0;
|
1829
|
+
}
|
1830
|
+
} else {
|
1831
|
+
return BROTLI_FALSE;
|
1832
|
+
}
|
1833
|
+
return BROTLI_TRUE;
|
1834
|
+
}
|
1835
|
+
|
1836
|
+
size_t BrotliEncoderEstimatePeakMemoryUsage(int quality, int lgwin,
|
1837
|
+
size_t input_size) {
|
1838
|
+
BrotliEncoderParams params;
|
1839
|
+
size_t memory_manager_slots = BROTLI_ENCODER_MEMORY_MANAGER_SLOTS;
|
1840
|
+
size_t memory_manager_size = memory_manager_slots * sizeof(void*);
|
1841
|
+
BrotliEncoderInitParams(¶ms);
|
1842
|
+
params.quality = quality;
|
1843
|
+
params.lgwin = lgwin;
|
1844
|
+
params.size_hint = input_size;
|
1845
|
+
params.large_window = lgwin > BROTLI_MAX_WINDOW_BITS;
|
1846
|
+
SanitizeParams(¶ms);
|
1847
|
+
params.lgblock = ComputeLgBlock(¶ms);
|
1848
|
+
ChooseHasher(¶ms, ¶ms.hasher);
|
1849
|
+
if (params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
|
1850
|
+
params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
|
1851
|
+
size_t state_size = sizeof(BrotliEncoderState);
|
1852
|
+
size_t block_size = BROTLI_MIN(size_t, input_size, (1ul << params.lgwin));
|
1853
|
+
size_t hash_table_size =
|
1854
|
+
HashTableSize(MaxHashTableSize(params.quality), block_size);
|
1855
|
+
size_t hash_size =
|
1856
|
+
(hash_table_size < (1u << 10)) ? 0 : sizeof(int) * hash_table_size;
|
1857
|
+
size_t cmdbuf_size = params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY ?
|
1858
|
+
5 * BROTLI_MIN(size_t, block_size, 1ul << 17) : 0;
|
1859
|
+
if (params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
|
1860
|
+
state_size += sizeof(BrotliOnePassArena);
|
1861
|
+
} else {
|
1862
|
+
state_size += sizeof(BrotliTwoPassArena);
|
1863
|
+
}
|
1864
|
+
return hash_size + cmdbuf_size + state_size;
|
1865
|
+
} else {
|
1866
|
+
size_t short_ringbuffer_size = (size_t)1 << params.lgblock;
|
1867
|
+
int ringbuffer_bits = ComputeRbBits(¶ms);
|
1868
|
+
size_t ringbuffer_size = input_size < short_ringbuffer_size ?
|
1869
|
+
input_size : (1u << ringbuffer_bits) + short_ringbuffer_size;
|
1870
|
+
size_t hash_size[4] = {0};
|
1871
|
+
size_t metablock_size =
|
1872
|
+
BROTLI_MIN(size_t, input_size, MaxMetablockSize(¶ms));
|
1873
|
+
size_t inputblock_size =
|
1874
|
+
BROTLI_MIN(size_t, input_size, (size_t)1 << params.lgblock);
|
1875
|
+
size_t cmdbuf_size = metablock_size * 2 + inputblock_size * 6;
|
1876
|
+
size_t outbuf_size = metablock_size * 2 + 503;
|
1877
|
+
size_t histogram_size = 0;
|
1878
|
+
HasherSize(¶ms, BROTLI_TRUE, input_size, hash_size);
|
1879
|
+
if (params.quality < MIN_QUALITY_FOR_BLOCK_SPLIT) {
|
1880
|
+
cmdbuf_size = BROTLI_MIN(size_t, cmdbuf_size,
|
1881
|
+
MAX_NUM_DELAYED_SYMBOLS * sizeof(Command) + inputblock_size * 12);
|
1882
|
+
}
|
1883
|
+
if (params.quality >= MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING) {
|
1884
|
+
/* Only a very rough estimation, based on enwik8. */
|
1885
|
+
histogram_size = 200 << 20;
|
1886
|
+
} else if (params.quality >= MIN_QUALITY_FOR_BLOCK_SPLIT) {
|
1887
|
+
size_t literal_histograms =
|
1888
|
+
BROTLI_MIN(size_t, metablock_size / 6144, 256);
|
1889
|
+
size_t command_histograms =
|
1890
|
+
BROTLI_MIN(size_t, metablock_size / 6144, 256);
|
1891
|
+
size_t distance_histograms =
|
1892
|
+
BROTLI_MIN(size_t, metablock_size / 6144, 256);
|
1893
|
+
histogram_size = literal_histograms * sizeof(HistogramLiteral) +
|
1894
|
+
command_histograms * sizeof(HistogramCommand) +
|
1895
|
+
distance_histograms * sizeof(HistogramDistance);
|
1896
|
+
}
|
1897
|
+
return (memory_manager_size + ringbuffer_size +
|
1898
|
+
hash_size[0] + hash_size[1] + hash_size[2] + hash_size[3] +
|
1899
|
+
cmdbuf_size +
|
1900
|
+
outbuf_size +
|
1901
|
+
histogram_size);
|
1902
|
+
}
|
1903
|
+
}
|
1904
|
+
size_t BrotliEncoderGetPreparedDictionarySize(
|
1905
|
+
const BrotliEncoderPreparedDictionary* prepared_dictionary) {
|
1906
|
+
/* First field of dictionary structs */
|
1907
|
+
const BrotliEncoderPreparedDictionary* prepared = prepared_dictionary;
|
1908
|
+
uint32_t magic = *((const uint32_t*)prepared);
|
1909
|
+
size_t overhead = 0;
|
1910
|
+
if (magic == kManagedDictionaryMagic) {
|
1911
|
+
const ManagedDictionary* managed = (const ManagedDictionary*)prepared;
|
1912
|
+
overhead = sizeof(ManagedDictionary);
|
1913
|
+
magic = *managed->dictionary;
|
1914
|
+
prepared = (const BrotliEncoderPreparedDictionary*)managed->dictionary;
|
1915
|
+
}
|
1916
|
+
|
1917
|
+
if (magic == kPreparedDictionaryMagic) {
|
1918
|
+
const PreparedDictionary* dictionary =
|
1919
|
+
(const PreparedDictionary*)prepared;
|
1920
|
+
/* Keep in sync with step 3 of CreatePreparedDictionary */
|
1921
|
+
return sizeof(PreparedDictionary) + dictionary->source_size +
|
1922
|
+
(sizeof(uint32_t) << dictionary->slot_bits) +
|
1923
|
+
(sizeof(uint16_t) << dictionary->bucket_bits) +
|
1924
|
+
(sizeof(uint32_t) * dictionary->num_items) + overhead;
|
1925
|
+
} else if (magic == kLeanPreparedDictionaryMagic) {
|
1926
|
+
const PreparedDictionary* dictionary =
|
1927
|
+
(const PreparedDictionary*)prepared;
|
1928
|
+
/* Keep in sync with step 3 of CreatePreparedDictionary */
|
1929
|
+
return sizeof(PreparedDictionary) + sizeof(uint8_t*) +
|
1930
|
+
(sizeof(uint32_t) << dictionary->slot_bits) +
|
1931
|
+
(sizeof(uint16_t) << dictionary->bucket_bits) +
|
1932
|
+
(sizeof(uint32_t) * dictionary->num_items) + overhead;
|
1933
|
+
} else if (magic == kSharedDictionaryMagic) {
|
1934
|
+
const SharedEncoderDictionary* dictionary =
|
1935
|
+
(const SharedEncoderDictionary*)prepared;
|
1936
|
+
const CompoundDictionary* compound = &dictionary->compound;
|
1937
|
+
const ContextualEncoderDictionary* contextual = &dictionary->contextual;
|
1938
|
+
size_t result = sizeof(*dictionary);
|
1939
|
+
size_t i;
|
1940
|
+
size_t num_instances;
|
1941
|
+
const BrotliEncoderDictionary* instances;
|
1942
|
+
for (i = 0; i < compound->num_prepared_instances_; i++) {
|
1943
|
+
size_t size = BrotliEncoderGetPreparedDictionarySize(
|
1944
|
+
(const BrotliEncoderPreparedDictionary*)
|
1945
|
+
compound->prepared_instances_[i]);
|
1946
|
+
if (!size) return 0; /* error */
|
1947
|
+
result += size;
|
1948
|
+
}
|
1949
|
+
if (contextual->context_based) {
|
1950
|
+
num_instances = contextual->num_instances_;
|
1951
|
+
instances = contextual->instances_;
|
1952
|
+
result += sizeof(*instances) * num_instances;
|
1953
|
+
} else {
|
1954
|
+
num_instances = 1;
|
1955
|
+
instances = &contextual->instance_;
|
1956
|
+
}
|
1957
|
+
for (i = 0; i < num_instances; i++) {
|
1958
|
+
const BrotliEncoderDictionary* dict = &instances[i];
|
1959
|
+
result += dict->trie.pool_capacity * sizeof(BrotliTrieNode);
|
1960
|
+
if (dict->hash_table_data_words_) {
|
1961
|
+
result += sizeof(kStaticDictionaryHashWords);
|
1962
|
+
}
|
1963
|
+
if (dict->hash_table_data_lengths_) {
|
1964
|
+
result += sizeof(kStaticDictionaryHashLengths);
|
1965
|
+
}
|
1966
|
+
if (dict->buckets_data_) {
|
1967
|
+
result += sizeof(*dict->buckets_data_) * dict->buckets_alloc_size_;
|
1968
|
+
}
|
1969
|
+
if (dict->dict_words_data_) {
|
1970
|
+
result += sizeof(*dict->dict_words) * dict->dict_words_alloc_size_;
|
1971
|
+
}
|
1972
|
+
if (dict->words_instance_) {
|
1973
|
+
result += sizeof(*dict->words_instance_);
|
1974
|
+
/* data_size not added here: it is never allocated by the
|
1975
|
+
SharedEncoderDictionary, instead it always points to the file
|
1976
|
+
already loaded in memory. So if the caller wants to include
|
1977
|
+
this memory as well, add the size of the loaded dictionary
|
1978
|
+
file to this. */
|
1979
|
+
}
|
1980
|
+
}
|
1981
|
+
return result + overhead;
|
1982
|
+
}
|
1983
|
+
return 0; /* error */
|
1984
|
+
}
|
1985
|
+
|
1986
|
+
#if defined(BROTLI_TEST)
|
1987
|
+
size_t MakeUncompressedStreamForTest(const uint8_t*, size_t, uint8_t*);
|
1988
|
+
size_t MakeUncompressedStreamForTest(
|
1989
|
+
const uint8_t* input, size_t input_size, uint8_t* output) {
|
1990
|
+
return MakeUncompressedStream(input, input_size, output);
|
1991
|
+
}
|
1992
|
+
#endif
|
1993
|
+
|
1925
1994
|
#if defined(__cplusplus) || defined(c_plusplus)
|
1926
1995
|
} /* extern "C" */
|
1927
1996
|
#endif
|