brotli 0.2.3 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +34 -0
- data/.github/workflows/publish.yml +34 -0
- data/Gemfile +6 -3
- data/Rakefile +16 -9
- data/brotli.gemspec +7 -13
- data/ext/brotli/brotli.c +209 -31
- data/ext/brotli/buffer.c +1 -7
- data/ext/brotli/buffer.h +1 -1
- data/ext/brotli/extconf.rb +20 -18
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +104 -0
- data/test/brotli_writer_test.rb +36 -0
- data/test/test_helper.rb +8 -0
- data/vendor/brotli/c/common/constants.c +15 -0
- data/vendor/brotli/c/common/constants.h +136 -0
- data/vendor/brotli/c/common/context.c +156 -0
- data/vendor/brotli/c/common/context.h +4 -152
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +10 -1
- data/vendor/brotli/c/common/platform.c +22 -0
- data/vendor/brotli/c/common/platform.h +43 -17
- data/vendor/brotli/c/common/transform.c +59 -3
- data/vendor/brotli/c/common/transform.h +5 -0
- data/vendor/brotli/c/common/version.h +2 -2
- data/vendor/brotli/c/dec/bit_reader.c +28 -0
- data/vendor/brotli/c/dec/bit_reader.h +58 -16
- data/vendor/brotli/c/dec/decode.c +353 -251
- data/vendor/brotli/c/dec/huffman.h +6 -12
- data/vendor/brotli/c/dec/prefix.h +0 -18
- data/vendor/brotli/c/dec/state.c +9 -14
- data/vendor/brotli/c/dec/state.h +144 -37
- data/vendor/brotli/c/enc/backward_references.c +8 -7
- data/vendor/brotli/c/enc/backward_references.h +5 -4
- data/vendor/brotli/c/enc/backward_references_hq.c +51 -33
- data/vendor/brotli/c/enc/backward_references_hq.h +11 -8
- data/vendor/brotli/c/enc/backward_references_inc.h +24 -14
- data/vendor/brotli/c/enc/block_splitter.c +3 -3
- data/vendor/brotli/c/enc/block_splitter_inc.h +15 -6
- data/vendor/brotli/c/enc/brotli_bit_stream.c +13 -30
- data/vendor/brotli/c/enc/cluster_inc.h +6 -3
- data/vendor/brotli/c/enc/command.c +28 -0
- data/vendor/brotli/c/enc/command.h +12 -12
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1826 -1100
- data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
- data/vendor/brotli/c/enc/encode.c +104 -39
- data/vendor/brotli/c/enc/encoder_dict.c +3 -2
- data/vendor/brotli/c/enc/encoder_dict.h +3 -1
- data/vendor/brotli/c/enc/entropy_encode.c +2 -0
- data/vendor/brotli/c/enc/entropy_encode.h +2 -2
- data/vendor/brotli/c/enc/fast_log.c +105 -0
- data/vendor/brotli/c/enc/fast_log.h +19 -100
- data/vendor/brotli/c/enc/find_match_length.h +2 -3
- data/vendor/brotli/c/enc/hash.h +80 -90
- data/vendor/brotli/c/enc/hash_composite_inc.h +52 -63
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +88 -49
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +50 -50
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +53 -50
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +91 -60
- data/vendor/brotli/c/enc/hash_rolling_inc.h +23 -27
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +39 -38
- data/vendor/brotli/c/enc/memory.h +24 -12
- data/vendor/brotli/c/enc/metablock.c +23 -27
- data/vendor/brotli/c/enc/metablock_inc.h +1 -1
- data/vendor/brotli/c/enc/params.h +3 -1
- data/vendor/brotli/c/enc/ringbuffer.h +4 -1
- data/vendor/brotli/c/enc/utf8_util.c +1 -1
- data/vendor/brotli/c/enc/write_bits.h +27 -25
- data/vendor/brotli/c/include/brotli/encode.h +22 -1
- data/vendor/brotli/c/include/brotli/port.h +14 -0
- metadata +17 -97
- data/.travis.yml +0 -31
- data/docs/Brotli.html +0 -485
- data/docs/Brotli/Error.html +0 -124
- data/docs/_index.html +0 -122
- data/docs/class_list.html +0 -51
- data/docs/css/common.css +0 -1
- data/docs/css/full_list.css +0 -58
- data/docs/css/style.css +0 -496
- data/docs/file.README.html +0 -127
- data/docs/file_list.html +0 -56
- data/docs/frames.html +0 -17
- data/docs/index.html +0 -127
- data/docs/js/app.js +0 -292
- data/docs/js/full_list.js +0 -216
- data/docs/js/jquery.js +0 -4
- data/docs/method_list.html +0 -67
- data/docs/top-level-namespace.html +0 -110
- data/spec/brotli_spec.rb +0 -88
- data/spec/inflate_spec.rb +0 -75
- data/spec/spec_helper.rb +0 -4
@@ -15,7 +15,8 @@
|
|
15
15
|
extern "C" {
|
16
16
|
#endif
|
17
17
|
|
18
|
-
extern const uint16_t
|
18
|
+
extern const uint16_t kStaticDictionaryHashWords[32768];
|
19
|
+
extern const uint8_t kStaticDictionaryHashLengths[32768];
|
19
20
|
|
20
21
|
#if defined(__cplusplus) || defined(c_plusplus)
|
21
22
|
} /* extern "C" */
|
@@ -54,12 +54,19 @@ typedef enum BrotliEncoderStreamState {
|
|
54
54
|
BROTLI_STREAM_METADATA_BODY = 4
|
55
55
|
} BrotliEncoderStreamState;
|
56
56
|
|
57
|
+
typedef enum BrotliEncoderFlintState {
|
58
|
+
BROTLI_FLINT_NEEDS_2_BYTES = 2,
|
59
|
+
BROTLI_FLINT_NEEDS_1_BYTE = 1,
|
60
|
+
BROTLI_FLINT_WAITING_FOR_PROCESSING = 0,
|
61
|
+
BROTLI_FLINT_WAITING_FOR_FLUSHING = -1,
|
62
|
+
BROTLI_FLINT_DONE = -2
|
63
|
+
} BrotliEncoderFlintState;
|
64
|
+
|
57
65
|
typedef struct BrotliEncoderStateStruct {
|
58
66
|
BrotliEncoderParams params;
|
59
67
|
|
60
68
|
MemoryManager memory_manager_;
|
61
69
|
|
62
|
-
HasherHandle hasher_;
|
63
70
|
uint64_t input_pos_;
|
64
71
|
RingBuffer ringbuffer_;
|
65
72
|
size_t cmd_alloc_size_;
|
@@ -73,10 +80,17 @@ typedef struct BrotliEncoderStateStruct {
|
|
73
80
|
int saved_dist_cache_[4];
|
74
81
|
uint16_t last_bytes_;
|
75
82
|
uint8_t last_bytes_bits_;
|
83
|
+
/* "Flint" is a tiny uncompressed block emitted before the continuation
|
84
|
+
block to unwire literal context from previous data. Despite being int8_t,
|
85
|
+
field is actually BrotliEncoderFlintState enum. */
|
86
|
+
int8_t flint_;
|
76
87
|
uint8_t prev_byte_;
|
77
88
|
uint8_t prev_byte2_;
|
78
89
|
size_t storage_size_;
|
79
90
|
uint8_t* storage_;
|
91
|
+
|
92
|
+
Hasher hasher_;
|
93
|
+
|
80
94
|
/* Hash table for FAST_ONE_PASS_COMPRESSION_QUALITY mode. */
|
81
95
|
int small_table_[1 << 10]; /* 4KiB */
|
82
96
|
int* large_table_; /* Allocated only when needed */
|
@@ -114,8 +128,6 @@ typedef struct BrotliEncoderStateStruct {
|
|
114
128
|
BROTLI_BOOL is_initialized_;
|
115
129
|
} BrotliEncoderStateStruct;
|
116
130
|
|
117
|
-
static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s);
|
118
|
-
|
119
131
|
static size_t InputBlockSize(BrotliEncoderState* s) {
|
120
132
|
return (size_t)1 << s->params.lgblock;
|
121
133
|
}
|
@@ -174,6 +186,11 @@ BROTLI_BOOL BrotliEncoderSetParameter(
|
|
174
186
|
state->params.dist.num_direct_distance_codes = value;
|
175
187
|
return BROTLI_TRUE;
|
176
188
|
|
189
|
+
case BROTLI_PARAM_STREAM_OFFSET:
|
190
|
+
if (value > (1u << 30)) return BROTLI_FALSE;
|
191
|
+
state->params.stream_offset = value;
|
192
|
+
return BROTLI_TRUE;
|
193
|
+
|
177
194
|
default: return BROTLI_FALSE;
|
178
195
|
}
|
179
196
|
}
|
@@ -195,7 +212,7 @@ static uint8_t* GetBrotliStorage(BrotliEncoderState* s, size_t size) {
|
|
195
212
|
if (s->storage_size_ < size) {
|
196
213
|
BROTLI_FREE(m, s->storage_);
|
197
214
|
s->storage_ = BROTLI_ALLOC(m, uint8_t, size);
|
198
|
-
if (BROTLI_IS_OOM(m)) return NULL;
|
215
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(s->storage_)) return NULL;
|
199
216
|
s->storage_size_ = size;
|
200
217
|
}
|
201
218
|
return s->storage_;
|
@@ -234,7 +251,7 @@ static int* GetHashTable(BrotliEncoderState* s, int quality,
|
|
234
251
|
s->large_table_size_ = htsize;
|
235
252
|
BROTLI_FREE(m, s->large_table_);
|
236
253
|
s->large_table_ = BROTLI_ALLOC(m, int, htsize);
|
237
|
-
if (BROTLI_IS_OOM(m)) return 0;
|
254
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(s->large_table_)) return 0;
|
238
255
|
}
|
239
256
|
table = s->large_table_;
|
240
257
|
}
|
@@ -499,7 +516,7 @@ static BROTLI_BOOL ShouldCompress(
|
|
499
516
|
/* TODO: find more precise minimal block overhead. */
|
500
517
|
if (bytes <= 2) return BROTLI_FALSE;
|
501
518
|
if (num_commands < (bytes >> 8) + 2) {
|
502
|
-
if (num_literals > 0.99 * (double)bytes) {
|
519
|
+
if ((double)num_literals > 0.99 * (double)bytes) {
|
503
520
|
uint32_t literal_histo[256] = { 0 };
|
504
521
|
static const uint32_t kSampleRate = 13;
|
505
522
|
static const double kMinEntropy = 7.92;
|
@@ -617,11 +634,7 @@ static void WriteMetaBlockInternal(MemoryManager* m,
|
|
617
634
|
/* The number of distance symbols effectively used for distance
|
618
635
|
histograms. It might be less than distance alphabet size
|
619
636
|
for "Large Window Brotli" (32-bit). */
|
620
|
-
|
621
|
-
if (num_effective_dist_codes > BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS) {
|
622
|
-
num_effective_dist_codes = BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS;
|
623
|
-
}
|
624
|
-
BrotliOptimizeHistograms(num_effective_dist_codes, &mb);
|
637
|
+
BrotliOptimizeHistograms(block_params.dist.alphabet_size_limit, &mb);
|
625
638
|
}
|
626
639
|
BrotliStoreMetaBlock(m, data, wrapped_last_flush_pos, bytes, mask,
|
627
640
|
prev_byte, prev_byte2,
|
@@ -678,12 +691,23 @@ static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s) {
|
|
678
691
|
|
679
692
|
s->last_bytes_bits_ = 0;
|
680
693
|
s->last_bytes_ = 0;
|
694
|
+
s->flint_ = BROTLI_FLINT_DONE;
|
681
695
|
s->remaining_metadata_bytes_ = BROTLI_UINT32_MAX;
|
682
696
|
|
683
697
|
SanitizeParams(&s->params);
|
684
698
|
s->params.lgblock = ComputeLgBlock(&s->params);
|
685
699
|
ChooseDistanceParams(&s->params);
|
686
700
|
|
701
|
+
if (s->params.stream_offset != 0) {
|
702
|
+
s->flint_ = BROTLI_FLINT_NEEDS_2_BYTES;
|
703
|
+
/* Poison the distance cache. -16 +- 3 is still less than zero (invalid). */
|
704
|
+
s->dist_cache_[0] = -16;
|
705
|
+
s->dist_cache_[1] = -16;
|
706
|
+
s->dist_cache_[2] = -16;
|
707
|
+
s->dist_cache_[3] = -16;
|
708
|
+
memcpy(s->saved_dist_cache_, s->dist_cache_, sizeof(s->saved_dist_cache_));
|
709
|
+
}
|
710
|
+
|
687
711
|
RingBufferSetup(&s->params, &s->ringbuffer_);
|
688
712
|
|
689
713
|
/* Initialize last byte with stream header. */
|
@@ -693,8 +717,14 @@ static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s) {
|
|
693
717
|
s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
|
694
718
|
lgwin = BROTLI_MAX(int, lgwin, 18);
|
695
719
|
}
|
696
|
-
|
697
|
-
|
720
|
+
if (s->params.stream_offset == 0) {
|
721
|
+
EncodeWindowBits(lgwin, s->params.large_window,
|
722
|
+
&s->last_bytes_, &s->last_bytes_bits_);
|
723
|
+
} else {
|
724
|
+
/* Bigger values have the same effect, but could cause overflows. */
|
725
|
+
s->params.stream_offset = BROTLI_MIN(size_t,
|
726
|
+
s->params.stream_offset, BROTLI_MAX_BACKWARD_LIMIT(lgwin));
|
727
|
+
}
|
698
728
|
}
|
699
729
|
|
700
730
|
if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
|
@@ -712,13 +742,15 @@ static void BrotliEncoderInitParams(BrotliEncoderParams* params) {
|
|
712
742
|
params->quality = BROTLI_DEFAULT_QUALITY;
|
713
743
|
params->lgwin = BROTLI_DEFAULT_WINDOW;
|
714
744
|
params->lgblock = 0;
|
745
|
+
params->stream_offset = 0;
|
715
746
|
params->size_hint = 0;
|
716
747
|
params->disable_literal_context_modeling = BROTLI_FALSE;
|
717
748
|
BrotliInitEncoderDictionary(¶ms->dictionary);
|
718
749
|
params->dist.distance_postfix_bits = 0;
|
719
750
|
params->dist.num_direct_distance_codes = 0;
|
720
|
-
params->dist.
|
751
|
+
params->dist.alphabet_size_max =
|
721
752
|
BROTLI_DISTANCE_ALPHABET_SIZE(0, 0, BROTLI_MAX_DISTANCE_BITS);
|
753
|
+
params->dist.alphabet_size_limit = params->dist.alphabet_size_max;
|
722
754
|
params->dist.max_distance = BROTLI_MAX_DISTANCE;
|
723
755
|
}
|
724
756
|
|
@@ -734,7 +766,7 @@ static void BrotliEncoderInitState(BrotliEncoderState* s) {
|
|
734
766
|
s->prev_byte2_ = 0;
|
735
767
|
s->storage_size_ = 0;
|
736
768
|
s->storage_ = 0;
|
737
|
-
s->hasher_
|
769
|
+
HasherInit(&s->hasher_);
|
738
770
|
s->large_table_ = NULL;
|
739
771
|
s->large_table_size_ = 0;
|
740
772
|
s->cmd_code_numbits_ = 0;
|
@@ -902,6 +934,7 @@ static void ExtendLastCommand(BrotliEncoderState* s, uint32_t* bytes,
|
|
902
934
|
(*bytes)--;
|
903
935
|
(*wrapped_last_processed_pos)++;
|
904
936
|
}
|
937
|
+
} else {
|
905
938
|
}
|
906
939
|
/* The copy length is at most the metablock size, and thus expressible. */
|
907
940
|
GetLengthCode(last_command->insert_len_,
|
@@ -934,6 +967,7 @@ static BROTLI_BOOL EncodeData(
|
|
934
967
|
uint32_t mask;
|
935
968
|
MemoryManager* m = &s->memory_manager_;
|
936
969
|
ContextType literal_context_mode;
|
970
|
+
ContextLut literal_context_lut;
|
937
971
|
|
938
972
|
data = s->ringbuffer_.buffer_;
|
939
973
|
mask = s->ringbuffer_.mask_;
|
@@ -951,7 +985,10 @@ static BROTLI_BOOL EncodeData(
|
|
951
985
|
BROTLI_ALLOC(m, uint32_t, kCompressFragmentTwoPassBlockSize);
|
952
986
|
s->literal_buf_ =
|
953
987
|
BROTLI_ALLOC(m, uint8_t, kCompressFragmentTwoPassBlockSize);
|
954
|
-
if (BROTLI_IS_OOM(m))
|
988
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(s->command_buf_) ||
|
989
|
+
BROTLI_IS_NULL(s->literal_buf_)) {
|
990
|
+
return BROTLI_FALSE;
|
991
|
+
}
|
955
992
|
}
|
956
993
|
|
957
994
|
if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
|
@@ -1009,7 +1046,7 @@ static BROTLI_BOOL EncodeData(
|
|
1009
1046
|
newsize += (bytes / 4) + 16;
|
1010
1047
|
s->cmd_alloc_size_ = newsize;
|
1011
1048
|
new_commands = BROTLI_ALLOC(m, Command, newsize);
|
1012
|
-
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1049
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_commands)) return BROTLI_FALSE;
|
1013
1050
|
if (s->commands_) {
|
1014
1051
|
memcpy(new_commands, s->commands_, sizeof(Command) * s->num_commands_);
|
1015
1052
|
BROTLI_FREE(m, s->commands_);
|
@@ -1024,6 +1061,7 @@ static BROTLI_BOOL EncodeData(
|
|
1024
1061
|
literal_context_mode = ChooseContextMode(
|
1025
1062
|
&s->params, data, WrapPosition(s->last_flush_pos_),
|
1026
1063
|
mask, (size_t)(s->input_pos_ - s->last_flush_pos_));
|
1064
|
+
literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
|
1027
1065
|
|
1028
1066
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1029
1067
|
|
@@ -1034,20 +1072,23 @@ static BROTLI_BOOL EncodeData(
|
|
1034
1072
|
if (s->params.quality == ZOPFLIFICATION_QUALITY) {
|
1035
1073
|
BROTLI_DCHECK(s->params.hasher.type == 10);
|
1036
1074
|
BrotliCreateZopfliBackwardReferences(m, bytes, wrapped_last_processed_pos,
|
1037
|
-
data, mask, &s->params,
|
1075
|
+
data, mask, literal_context_lut, &s->params,
|
1076
|
+
&s->hasher_, s->dist_cache_,
|
1038
1077
|
&s->last_insert_len_, &s->commands_[s->num_commands_],
|
1039
1078
|
&s->num_commands_, &s->num_literals_);
|
1040
1079
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1041
1080
|
} else if (s->params.quality == HQ_ZOPFLIFICATION_QUALITY) {
|
1042
1081
|
BROTLI_DCHECK(s->params.hasher.type == 10);
|
1043
1082
|
BrotliCreateHqZopfliBackwardReferences(m, bytes, wrapped_last_processed_pos,
|
1044
|
-
data, mask, &s->params,
|
1083
|
+
data, mask, literal_context_lut, &s->params,
|
1084
|
+
&s->hasher_, s->dist_cache_,
|
1045
1085
|
&s->last_insert_len_, &s->commands_[s->num_commands_],
|
1046
1086
|
&s->num_commands_, &s->num_literals_);
|
1047
1087
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1048
1088
|
} else {
|
1049
1089
|
BrotliCreateBackwardReferences(bytes, wrapped_last_processed_pos,
|
1050
|
-
data, mask, &s->params,
|
1090
|
+
data, mask, literal_context_lut, &s->params,
|
1091
|
+
&s->hasher_, s->dist_cache_,
|
1051
1092
|
&s->last_insert_len_, &s->commands_[s->num_commands_],
|
1052
1093
|
&s->num_commands_, &s->num_literals_);
|
1053
1094
|
}
|
@@ -1072,7 +1113,7 @@ static BROTLI_BOOL EncodeData(
|
|
1072
1113
|
s->num_commands_ < max_commands) {
|
1073
1114
|
/* Merge with next input block. Everything will happen later. */
|
1074
1115
|
if (UpdateLastProcessedPos(s)) {
|
1075
|
-
HasherReset(s->hasher_);
|
1116
|
+
HasherReset(&s->hasher_);
|
1076
1117
|
}
|
1077
1118
|
*out_size = 0;
|
1078
1119
|
return BROTLI_TRUE;
|
@@ -1113,7 +1154,7 @@ static BROTLI_BOOL EncodeData(
|
|
1113
1154
|
s->last_bytes_bits_ = storage_ix & 7u;
|
1114
1155
|
s->last_flush_pos_ = s->input_pos_;
|
1115
1156
|
if (UpdateLastProcessedPos(s)) {
|
1116
|
-
HasherReset(s->hasher_);
|
1157
|
+
HasherReset(&s->hasher_);
|
1117
1158
|
}
|
1118
1159
|
if (s->last_flush_pos_ > 0) {
|
1119
1160
|
s->prev_byte_ = data[((uint32_t)s->last_flush_pos_ - 1) & mask];
|
@@ -1174,7 +1215,6 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1174
1215
|
size_t total_out_size = 0;
|
1175
1216
|
uint16_t last_bytes;
|
1176
1217
|
uint8_t last_bytes_bits;
|
1177
|
-
HasherHandle hasher = NULL;
|
1178
1218
|
|
1179
1219
|
const size_t hasher_eff_size = BROTLI_MIN(size_t,
|
1180
1220
|
input_size, BROTLI_MAX_BACKWARD_LIMIT(lgwin) + BROTLI_WINDOW_GAP);
|
@@ -1190,6 +1230,9 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1190
1230
|
uint8_t prev_byte = 0;
|
1191
1231
|
uint8_t prev_byte2 = 0;
|
1192
1232
|
|
1233
|
+
Hasher hasher;
|
1234
|
+
HasherInit(&hasher);
|
1235
|
+
|
1193
1236
|
BrotliEncoderInitParams(¶ms);
|
1194
1237
|
params.quality = 10;
|
1195
1238
|
params.lgwin = lgwin;
|
@@ -1226,6 +1269,7 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1226
1269
|
|
1227
1270
|
ContextType literal_context_mode = ChooseContextMode(¶ms,
|
1228
1271
|
input_buffer, metablock_start, mask, metablock_end - metablock_start);
|
1272
|
+
ContextLut literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
|
1229
1273
|
|
1230
1274
|
size_t block_start;
|
1231
1275
|
for (block_start = metablock_start; block_start < metablock_end; ) {
|
@@ -1234,12 +1278,12 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1234
1278
|
ZopfliNode* nodes = BROTLI_ALLOC(m, ZopfliNode, block_size + 1);
|
1235
1279
|
size_t path_size;
|
1236
1280
|
size_t new_cmd_alloc_size;
|
1237
|
-
if (BROTLI_IS_OOM(m)) goto oom;
|
1281
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(nodes)) goto oom;
|
1238
1282
|
BrotliInitZopfliNodes(nodes, block_size + 1);
|
1239
|
-
StitchToPreviousBlockH10(hasher, block_size, block_start,
|
1283
|
+
StitchToPreviousBlockH10(&hasher.privat._H10, block_size, block_start,
|
1240
1284
|
input_buffer, mask);
|
1241
1285
|
path_size = BrotliZopfliComputeShortestPath(m, block_size, block_start,
|
1242
|
-
input_buffer, mask, ¶ms, dist_cache, hasher,
|
1286
|
+
input_buffer, mask, literal_context_lut, ¶ms, dist_cache, &hasher,
|
1243
1287
|
nodes);
|
1244
1288
|
if (BROTLI_IS_OOM(m)) goto oom;
|
1245
1289
|
/* We allocate a command buffer in the first iteration of this loop that
|
@@ -1254,7 +1298,7 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1254
1298
|
num_commands + path_size + 1);
|
1255
1299
|
if (cmd_alloc_size != new_cmd_alloc_size) {
|
1256
1300
|
Command* new_commands = BROTLI_ALLOC(m, Command, new_cmd_alloc_size);
|
1257
|
-
if (BROTLI_IS_OOM(m)) goto oom;
|
1301
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_commands)) goto oom;
|
1258
1302
|
cmd_alloc_size = new_cmd_alloc_size;
|
1259
1303
|
if (commands) {
|
1260
1304
|
memcpy(new_commands, commands, sizeof(Command) * num_commands);
|
@@ -1286,7 +1330,7 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1286
1330
|
if (metablock_size == 0) {
|
1287
1331
|
/* Write the ISLAST and ISEMPTY bits. */
|
1288
1332
|
storage = BROTLI_ALLOC(m, uint8_t, 16);
|
1289
|
-
if (BROTLI_IS_OOM(m)) goto oom;
|
1333
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(storage)) goto oom;
|
1290
1334
|
storage[0] = (uint8_t)last_bytes;
|
1291
1335
|
storage[1] = (uint8_t)(last_bytes >> 8);
|
1292
1336
|
BrotliWriteBits(2, 3, &storage_ix, storage);
|
@@ -1297,7 +1341,7 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1297
1341
|
CreateBackwardReferences is now unused. */
|
1298
1342
|
memcpy(dist_cache, saved_dist_cache, 4 * sizeof(dist_cache[0]));
|
1299
1343
|
storage = BROTLI_ALLOC(m, uint8_t, metablock_size + 16);
|
1300
|
-
if (BROTLI_IS_OOM(m)) goto oom;
|
1344
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(storage)) goto oom;
|
1301
1345
|
storage[0] = (uint8_t)last_bytes;
|
1302
1346
|
storage[1] = (uint8_t)(last_bytes >> 8);
|
1303
1347
|
BrotliStoreUncompressedMetaBlock(is_last, input_buffer,
|
@@ -1318,14 +1362,10 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
|
1318
1362
|
/* The number of distance symbols effectively used for distance
|
1319
1363
|
histograms. It might be less than distance alphabet size
|
1320
1364
|
for "Large Window Brotli" (32-bit). */
|
1321
|
-
|
1322
|
-
if (num_effective_dist_codes > BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS) {
|
1323
|
-
num_effective_dist_codes = BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS;
|
1324
|
-
}
|
1325
|
-
BrotliOptimizeHistograms(num_effective_dist_codes, &mb);
|
1365
|
+
BrotliOptimizeHistograms(block_params.dist.alphabet_size_limit, &mb);
|
1326
1366
|
}
|
1327
1367
|
storage = BROTLI_ALLOC(m, uint8_t, 2 * metablock_size + 503);
|
1328
|
-
if (BROTLI_IS_OOM(m)) goto oom;
|
1368
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(storage)) goto oom;
|
1329
1369
|
storage[0] = (uint8_t)last_bytes;
|
1330
1370
|
storage[1] = (uint8_t)(last_bytes >> 8);
|
1331
1371
|
BrotliStoreMetaBlock(m, input_buffer, metablock_start, metablock_size,
|
@@ -1576,7 +1616,10 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
|
|
1576
1616
|
BROTLI_ALLOC(m, uint32_t, kCompressFragmentTwoPassBlockSize);
|
1577
1617
|
s->literal_buf_ =
|
1578
1618
|
BROTLI_ALLOC(m, uint8_t, kCompressFragmentTwoPassBlockSize);
|
1579
|
-
if (BROTLI_IS_OOM(m))
|
1619
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(s->command_buf_) ||
|
1620
|
+
BROTLI_IS_NULL(s->literal_buf_)) {
|
1621
|
+
return BROTLI_FALSE;
|
1622
|
+
}
|
1580
1623
|
}
|
1581
1624
|
if (s->command_buf_) {
|
1582
1625
|
command_buf = s->command_buf_;
|
@@ -1584,7 +1627,10 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
|
|
1584
1627
|
} else {
|
1585
1628
|
tmp_command_buf = BROTLI_ALLOC(m, uint32_t, buf_size);
|
1586
1629
|
tmp_literal_buf = BROTLI_ALLOC(m, uint8_t, buf_size);
|
1587
|
-
if (BROTLI_IS_OOM(m))
|
1630
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tmp_command_buf) ||
|
1631
|
+
BROTLI_IS_NULL(tmp_literal_buf)) {
|
1632
|
+
return BROTLI_FALSE;
|
1633
|
+
}
|
1588
1634
|
command_buf = tmp_command_buf;
|
1589
1635
|
literal_buf = tmp_literal_buf;
|
1590
1636
|
}
|
@@ -1640,8 +1686,10 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast(
|
|
1640
1686
|
&storage_ix, storage);
|
1641
1687
|
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
|
1642
1688
|
}
|
1643
|
-
|
1644
|
-
|
1689
|
+
if (block_size != 0) {
|
1690
|
+
*next_in += block_size;
|
1691
|
+
*available_in -= block_size;
|
1692
|
+
}
|
1645
1693
|
if (inplace) {
|
1646
1694
|
size_t out_bytes = storage_ix >> 3;
|
1647
1695
|
BROTLI_DCHECK(out_bytes <= *available_out);
|
@@ -1786,6 +1834,10 @@ BROTLI_BOOL BrotliEncoderCompressStream(
|
|
1786
1834
|
}
|
1787
1835
|
while (BROTLI_TRUE) {
|
1788
1836
|
size_t remaining_block_size = RemainingInputBlockSize(s);
|
1837
|
+
/* Shorten input to flint size. */
|
1838
|
+
if (s->flint_ >= 0 && remaining_block_size > (size_t)s->flint_) {
|
1839
|
+
remaining_block_size = (size_t)s->flint_;
|
1840
|
+
}
|
1789
1841
|
|
1790
1842
|
if (remaining_block_size != 0 && *available_in != 0) {
|
1791
1843
|
size_t copy_input_size =
|
@@ -1793,10 +1845,18 @@ BROTLI_BOOL BrotliEncoderCompressStream(
|
|
1793
1845
|
CopyInputToRingBuffer(s, copy_input_size, *next_in);
|
1794
1846
|
*next_in += copy_input_size;
|
1795
1847
|
*available_in -= copy_input_size;
|
1848
|
+
if (s->flint_ > 0) s->flint_ = (int8_t)(s->flint_ - (int)copy_input_size);
|
1796
1849
|
continue;
|
1797
1850
|
}
|
1798
1851
|
|
1799
1852
|
if (InjectFlushOrPushOutput(s, available_out, next_out, total_out)) {
|
1853
|
+
/* Exit the "emit flint" workflow. */
|
1854
|
+
if (s->flint_ == BROTLI_FLINT_WAITING_FOR_FLUSHING) {
|
1855
|
+
CheckFlushComplete(s);
|
1856
|
+
if (s->stream_state_ == BROTLI_STREAM_PROCESSING) {
|
1857
|
+
s->flint_ = BROTLI_FLINT_DONE;
|
1858
|
+
}
|
1859
|
+
}
|
1800
1860
|
continue;
|
1801
1861
|
}
|
1802
1862
|
|
@@ -1810,6 +1870,11 @@ BROTLI_BOOL BrotliEncoderCompressStream(
|
|
1810
1870
|
BROTLI_BOOL force_flush = TO_BROTLI_BOOL(
|
1811
1871
|
(*available_in == 0) && op == BROTLI_OPERATION_FLUSH);
|
1812
1872
|
BROTLI_BOOL result;
|
1873
|
+
/* Force emitting (uncompressed) piece containing flint. */
|
1874
|
+
if (!is_last && s->flint_ == 0) {
|
1875
|
+
s->flint_ = BROTLI_FLINT_WAITING_FOR_FLUSHING;
|
1876
|
+
force_flush = BROTLI_TRUE;
|
1877
|
+
}
|
1813
1878
|
UpdateSizeHint(s, *available_in);
|
1814
1879
|
result = EncodeData(s, is_last, force_flush,
|
1815
1880
|
&s->available_out_, &s->next_out_);
|
@@ -17,14 +17,15 @@ extern "C" {
|
|
17
17
|
|
18
18
|
void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict) {
|
19
19
|
dict->words = BrotliGetDictionary();
|
20
|
+
dict->num_transforms = (uint32_t)BrotliGetTransforms()->num_transforms;
|
20
21
|
|
21
|
-
dict->
|
22
|
+
dict->hash_table_words = kStaticDictionaryHashWords;
|
23
|
+
dict->hash_table_lengths = kStaticDictionaryHashLengths;
|
22
24
|
dict->buckets = kStaticDictionaryBuckets;
|
23
25
|
dict->dict_words = kStaticDictionaryWords;
|
24
26
|
|
25
27
|
dict->cutoffTransformsCount = kCutoffTransformsCount;
|
26
28
|
dict->cutoffTransforms = kCutoffTransforms;
|
27
|
-
|
28
29
|
}
|
29
30
|
|
30
31
|
#if defined(__cplusplus) || defined(c_plusplus)
|
@@ -19,13 +19,15 @@ extern "C" {
|
|
19
19
|
/* Dictionary data (words and transforms) for 1 possible context */
|
20
20
|
typedef struct BrotliEncoderDictionary {
|
21
21
|
const BrotliDictionary* words;
|
22
|
+
uint32_t num_transforms;
|
22
23
|
|
23
24
|
/* cut off for fast encoder */
|
24
25
|
uint32_t cutoffTransformsCount;
|
25
26
|
uint64_t cutoffTransforms;
|
26
27
|
|
27
28
|
/* from dictionary_hash.h, for fast encoder */
|
28
|
-
const uint16_t*
|
29
|
+
const uint16_t* hash_table_words;
|
30
|
+
const uint8_t* hash_table_lengths;
|
29
31
|
|
30
32
|
/* from static_dict_lut.h, for slow encoder */
|
31
33
|
const uint16_t* buckets;
|