brotli 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/main.yml +34 -0
- data/.github/workflows/publish.yml +34 -0
- data/Gemfile +6 -2
- data/Rakefile +18 -6
- data/bin/before_install.sh +9 -0
- data/brotli.gemspec +7 -13
- data/ext/brotli/brotli.c +209 -11
- data/ext/brotli/buffer.c +1 -7
- data/ext/brotli/buffer.h +1 -1
- data/ext/brotli/extconf.rb +45 -26
- data/lib/brotli/version.rb +1 -1
- data/smoke.sh +1 -1
- data/test/brotli_test.rb +104 -0
- data/test/brotli_writer_test.rb +36 -0
- data/test/test_helper.rb +8 -0
- data/vendor/brotli/c/common/constants.c +15 -0
- data/vendor/brotli/c/common/constants.h +149 -6
- data/vendor/brotli/c/{dec/context.h → common/context.c} +91 -186
- data/vendor/brotli/c/common/context.h +113 -0
- data/vendor/brotli/c/common/dictionary.bin +0 -0
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +11 -2
- data/vendor/brotli/c/common/dictionary.h +4 -4
- data/vendor/brotli/c/common/platform.c +22 -0
- data/vendor/brotli/c/common/platform.h +594 -0
- data/vendor/brotli/c/common/transform.c +291 -0
- data/vendor/brotli/c/common/transform.h +85 -0
- data/vendor/brotli/c/common/version.h +8 -1
- data/vendor/brotli/c/dec/bit_reader.c +29 -1
- data/vendor/brotli/c/dec/bit_reader.h +91 -100
- data/vendor/brotli/c/dec/decode.c +665 -437
- data/vendor/brotli/c/dec/huffman.c +65 -84
- data/vendor/brotli/c/dec/huffman.h +67 -14
- data/vendor/brotli/c/dec/prefix.h +1 -20
- data/vendor/brotli/c/dec/state.c +32 -45
- data/vendor/brotli/c/dec/state.h +173 -55
- data/vendor/brotli/c/enc/backward_references.c +27 -16
- data/vendor/brotli/c/enc/backward_references.h +7 -7
- data/vendor/brotli/c/enc/backward_references_hq.c +155 -116
- data/vendor/brotli/c/enc/backward_references_hq.h +22 -23
- data/vendor/brotli/c/enc/backward_references_inc.h +32 -22
- data/vendor/brotli/c/enc/bit_cost.c +1 -1
- data/vendor/brotli/c/enc/bit_cost.h +5 -5
- data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
- data/vendor/brotli/c/enc/block_splitter.c +5 -6
- data/vendor/brotli/c/enc/block_splitter.h +1 -1
- data/vendor/brotli/c/enc/block_splitter_inc.h +26 -17
- data/vendor/brotli/c/enc/brotli_bit_stream.c +107 -123
- data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
- data/vendor/brotli/c/enc/cluster.c +1 -1
- data/vendor/brotli/c/enc/cluster.h +1 -1
- data/vendor/brotli/c/enc/cluster_inc.h +6 -3
- data/vendor/brotli/c/enc/command.c +28 -0
- data/vendor/brotli/c/enc/command.h +52 -42
- data/vendor/brotli/c/enc/compress_fragment.c +21 -22
- data/vendor/brotli/c/enc/compress_fragment.h +1 -1
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +102 -69
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1827 -1101
- data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
- data/vendor/brotli/c/enc/encode.c +358 -195
- data/vendor/brotli/c/enc/encoder_dict.c +33 -0
- data/vendor/brotli/c/enc/encoder_dict.h +43 -0
- data/vendor/brotli/c/enc/entropy_encode.c +16 -14
- data/vendor/brotli/c/enc/entropy_encode.h +7 -7
- data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
- data/vendor/brotli/c/enc/fast_log.c +105 -0
- data/vendor/brotli/c/enc/fast_log.h +20 -99
- data/vendor/brotli/c/enc/find_match_length.h +5 -6
- data/vendor/brotli/c/enc/hash.h +145 -103
- data/vendor/brotli/c/enc/hash_composite_inc.h +125 -0
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +93 -53
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +54 -53
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +58 -54
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +95 -63
- data/vendor/brotli/c/enc/hash_rolling_inc.h +212 -0
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +46 -43
- data/vendor/brotli/c/enc/histogram.c +9 -6
- data/vendor/brotli/c/enc/histogram.h +6 -3
- data/vendor/brotli/c/enc/histogram_inc.h +1 -1
- data/vendor/brotli/c/enc/literal_cost.c +5 -5
- data/vendor/brotli/c/enc/literal_cost.h +2 -2
- data/vendor/brotli/c/enc/memory.c +5 -16
- data/vendor/brotli/c/enc/memory.h +52 -1
- data/vendor/brotli/c/enc/metablock.c +171 -36
- data/vendor/brotli/c/enc/metablock.h +13 -8
- data/vendor/brotli/c/enc/metablock_inc.h +2 -2
- data/vendor/brotli/c/enc/params.h +46 -0
- data/vendor/brotli/c/enc/prefix.h +3 -4
- data/vendor/brotli/c/enc/quality.h +29 -24
- data/vendor/brotli/c/enc/ringbuffer.h +19 -12
- data/vendor/brotli/c/enc/static_dict.c +49 -45
- data/vendor/brotli/c/enc/static_dict.h +4 -3
- data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
- data/vendor/brotli/c/enc/utf8_util.c +21 -21
- data/vendor/brotli/c/enc/utf8_util.h +1 -1
- data/vendor/brotli/c/enc/write_bits.h +35 -38
- data/vendor/brotli/c/include/brotli/decode.h +13 -8
- data/vendor/brotli/c/include/brotli/encode.h +54 -8
- data/vendor/brotli/c/include/brotli/port.h +225 -83
- data/vendor/brotli/c/include/brotli/types.h +0 -7
- metadata +28 -87
- data/.travis.yml +0 -30
- data/spec/brotli_spec.rb +0 -88
- data/spec/inflate_spec.rb +0 -75
- data/spec/spec_helper.rb +0 -4
- data/vendor/brotli/c/dec/port.h +0 -168
- data/vendor/brotli/c/dec/transform.h +0 -300
- data/vendor/brotli/c/enc/context.h +0 -184
- data/vendor/brotli/c/enc/port.h +0 -184
@@ -16,13 +16,13 @@
|
|
16
16
|
#ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
17
17
|
#define BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
18
18
|
|
19
|
+
#include "../common/context.h"
|
20
|
+
#include "../common/platform.h"
|
19
21
|
#include <brotli/types.h>
|
20
22
|
#include "./command.h"
|
21
|
-
#include "./context.h"
|
22
23
|
#include "./entropy_encode.h"
|
23
24
|
#include "./memory.h"
|
24
25
|
#include "./metablock.h"
|
25
|
-
#include "./port.h"
|
26
26
|
|
27
27
|
#if defined(__cplusplus) || defined(c_plusplus)
|
28
28
|
extern "C" {
|
@@ -32,7 +32,7 @@ extern "C" {
|
|
32
32
|
position for the current storage. */
|
33
33
|
|
34
34
|
BROTLI_INTERNAL void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
|
35
|
-
HuffmanTree* tree, size_t
|
35
|
+
HuffmanTree* tree, size_t* storage_ix, uint8_t* storage);
|
36
36
|
|
37
37
|
BROTLI_INTERNAL void BrotliBuildAndStoreHuffmanTreeFast(
|
38
38
|
MemoryManager* m, const uint32_t* histogram, const size_t histogram_total,
|
@@ -42,59 +42,40 @@ BROTLI_INTERNAL void BrotliBuildAndStoreHuffmanTreeFast(
|
|
42
42
|
/* REQUIRES: length > 0 */
|
43
43
|
/* REQUIRES: length <= (1 << 24) */
|
44
44
|
BROTLI_INTERNAL void BrotliStoreMetaBlock(MemoryManager* m,
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
uint8_t prev_byte2,
|
51
|
-
BROTLI_BOOL is_final_block,
|
52
|
-
uint32_t num_direct_distance_codes,
|
53
|
-
uint32_t distance_postfix_bits,
|
54
|
-
ContextType literal_context_mode,
|
55
|
-
const Command* commands,
|
56
|
-
size_t n_commands,
|
57
|
-
const MetaBlockSplit* mb,
|
58
|
-
size_t* storage_ix,
|
59
|
-
uint8_t* storage);
|
45
|
+
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
|
46
|
+
uint8_t prev_byte, uint8_t prev_byte2, BROTLI_BOOL is_last,
|
47
|
+
const BrotliEncoderParams* params, ContextType literal_context_mode,
|
48
|
+
const Command* commands, size_t n_commands, const MetaBlockSplit* mb,
|
49
|
+
size_t* storage_ix, uint8_t* storage);
|
60
50
|
|
61
51
|
/* Stores the meta-block without doing any block splitting, just collects
|
62
52
|
one histogram per block category and uses that for entropy coding.
|
63
53
|
REQUIRES: length > 0
|
64
54
|
REQUIRES: length <= (1 << 24) */
|
65
55
|
BROTLI_INTERNAL void BrotliStoreMetaBlockTrivial(MemoryManager* m,
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
BROTLI_BOOL is_last,
|
71
|
-
const Command *commands,
|
72
|
-
size_t n_commands,
|
73
|
-
size_t* storage_ix,
|
74
|
-
uint8_t* storage);
|
56
|
+
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
|
57
|
+
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
|
58
|
+
const Command* commands, size_t n_commands,
|
59
|
+
size_t* storage_ix, uint8_t* storage);
|
75
60
|
|
76
61
|
/* Same as above, but uses static prefix codes for histograms with a only a few
|
77
62
|
symbols, and uses static code length prefix codes for all other histograms.
|
78
63
|
REQUIRES: length > 0
|
79
64
|
REQUIRES: length <= (1 << 24) */
|
80
65
|
BROTLI_INTERNAL void BrotliStoreMetaBlockFast(MemoryManager* m,
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
BROTLI_BOOL is_last,
|
86
|
-
const Command *commands,
|
87
|
-
size_t n_commands,
|
88
|
-
size_t* storage_ix,
|
89
|
-
uint8_t* storage);
|
66
|
+
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
|
67
|
+
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
|
68
|
+
const Command* commands, size_t n_commands,
|
69
|
+
size_t* storage_ix, uint8_t* storage);
|
90
70
|
|
91
71
|
/* This is for storing uncompressed blocks (simple raw storage of
|
92
72
|
bytes-as-bytes).
|
93
73
|
REQUIRES: length > 0
|
94
74
|
REQUIRES: length <= (1 << 24) */
|
95
75
|
BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
|
96
|
-
BROTLI_BOOL is_final_block, const uint8_t* input,
|
97
|
-
size_t
|
76
|
+
BROTLI_BOOL is_final_block, const uint8_t* BROTLI_RESTRICT input,
|
77
|
+
size_t position, size_t mask, size_t len,
|
78
|
+
size_t* BROTLI_RESTRICT storage_ix, uint8_t* BROTLI_RESTRICT storage);
|
98
79
|
|
99
80
|
#if defined(__cplusplus) || defined(c_plusplus)
|
100
81
|
} /* extern "C" */
|
@@ -8,12 +8,12 @@
|
|
8
8
|
|
9
9
|
#include "./cluster.h"
|
10
10
|
|
11
|
+
#include "../common/platform.h"
|
11
12
|
#include <brotli/types.h>
|
12
13
|
#include "./bit_cost.h" /* BrotliPopulationCost */
|
13
14
|
#include "./fast_log.h"
|
14
15
|
#include "./histogram.h"
|
15
16
|
#include "./memory.h"
|
16
|
-
#include "./port.h"
|
17
17
|
|
18
18
|
#if defined(__cplusplus) || defined(c_plusplus)
|
19
19
|
extern "C" {
|
@@ -9,10 +9,10 @@
|
|
9
9
|
#ifndef BROTLI_ENC_CLUSTER_H_
|
10
10
|
#define BROTLI_ENC_CLUSTER_H_
|
11
11
|
|
12
|
+
#include "../common/platform.h"
|
12
13
|
#include <brotli/types.h>
|
13
14
|
#include "./histogram.h"
|
14
15
|
#include "./memory.h"
|
15
|
-
#include "./port.h"
|
16
16
|
|
17
17
|
#if defined(__cplusplus) || defined(c_plusplus)
|
18
18
|
extern "C" {
|
@@ -215,7 +215,7 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramReindex)(MemoryManager* m,
|
|
215
215
|
uint32_t next_index;
|
216
216
|
HistogramType* tmp;
|
217
217
|
size_t i;
|
218
|
-
if (BROTLI_IS_OOM(m)) return 0;
|
218
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_index)) return 0;
|
219
219
|
for (i = 0; i < length; ++i) {
|
220
220
|
new_index[i] = kInvalidIndex;
|
221
221
|
}
|
@@ -229,7 +229,7 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramReindex)(MemoryManager* m,
|
|
229
229
|
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
|
230
230
|
tmp and reduce the number of copying by the factor of 2. */
|
231
231
|
tmp = BROTLI_ALLOC(m, HistogramType, next_index);
|
232
|
-
if (BROTLI_IS_OOM(m)) return 0;
|
232
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tmp)) return 0;
|
233
233
|
next_index = 0;
|
234
234
|
for (i = 0; i < length; ++i) {
|
235
235
|
if (new_index[symbols[i]] == next_index) {
|
@@ -259,7 +259,10 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
|
|
259
259
|
HistogramPair* pairs = BROTLI_ALLOC(m, HistogramPair, pairs_capacity + 1);
|
260
260
|
size_t i;
|
261
261
|
|
262
|
-
if (BROTLI_IS_OOM(m))
|
262
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(cluster_size) ||
|
263
|
+
BROTLI_IS_NULL(clusters) || BROTLI_IS_NULL(pairs)) {
|
264
|
+
return;
|
265
|
+
}
|
263
266
|
|
264
267
|
for (i = 0; i < in_size; ++i) {
|
265
268
|
cluster_size[i] = 1;
|
@@ -0,0 +1,28 @@
|
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
#include "./command.h"
|
8
|
+
|
9
|
+
#include <brotli/types.h>
|
10
|
+
|
11
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
12
|
+
extern "C" {
|
13
|
+
#endif
|
14
|
+
|
15
|
+
const uint32_t kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES] = {
|
16
|
+
0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26,
|
17
|
+
34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210, 22594};
|
18
|
+
const uint32_t kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES] = {
|
19
|
+
0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24};
|
20
|
+
const uint32_t kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES] = {
|
21
|
+
2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18,
|
22
|
+
22, 30, 38, 54, 70, 102, 134, 198, 326, 582, 1094, 2118};
|
23
|
+
const uint32_t kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES] = {
|
24
|
+
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24};
|
25
|
+
|
26
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
27
|
+
} /* extern "C" */
|
28
|
+
#endif
|
@@ -10,23 +10,24 @@
|
|
10
10
|
#define BROTLI_ENC_COMMAND_H_
|
11
11
|
|
12
12
|
#include "../common/constants.h"
|
13
|
-
#include
|
13
|
+
#include "../common/platform.h"
|
14
14
|
#include <brotli/types.h>
|
15
15
|
#include "./fast_log.h"
|
16
|
+
#include "./params.h"
|
16
17
|
#include "./prefix.h"
|
17
18
|
|
18
19
|
#if defined(__cplusplus) || defined(c_plusplus)
|
19
20
|
extern "C" {
|
20
21
|
#endif
|
21
22
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
23
|
+
BROTLI_INTERNAL extern const uint32_t
|
24
|
+
kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES];
|
25
|
+
BROTLI_INTERNAL extern const uint32_t
|
26
|
+
kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES];
|
27
|
+
BROTLI_INTERNAL extern const uint32_t
|
28
|
+
kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES];
|
29
|
+
BROTLI_INTERNAL extern const uint32_t
|
30
|
+
kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES];
|
30
31
|
|
31
32
|
static BROTLI_INLINE uint16_t GetInsertLengthCode(size_t insertlen) {
|
32
33
|
if (insertlen < 6) {
|
@@ -61,21 +62,21 @@ static BROTLI_INLINE uint16_t GetCopyLengthCode(size_t copylen) {
|
|
61
62
|
static BROTLI_INLINE uint16_t CombineLengthCodes(
|
62
63
|
uint16_t inscode, uint16_t copycode, BROTLI_BOOL use_last_distance) {
|
63
64
|
uint16_t bits64 =
|
64
|
-
(uint16_t)((copycode & 0x7u) | ((inscode & 0x7u) <<
|
65
|
-
if (use_last_distance && inscode <
|
66
|
-
return (copycode <
|
65
|
+
(uint16_t)((copycode & 0x7u) | ((inscode & 0x7u) << 3u));
|
66
|
+
if (use_last_distance && inscode < 8u && copycode < 16u) {
|
67
|
+
return (copycode < 8u) ? bits64 : (bits64 | 64u);
|
67
68
|
} else {
|
68
69
|
/* Specification: 5 Encoding of ... (last table) */
|
69
70
|
/* offset = 2 * index, where index is in range [0..8] */
|
70
|
-
|
71
|
+
uint32_t offset = 2u * ((copycode >> 3u) + 3u * (inscode >> 3u));
|
71
72
|
/* All values in specification are K * 64,
|
72
73
|
where K = [2, 3, 6, 4, 5, 8, 7, 9, 10],
|
73
74
|
i + 1 = [1, 2, 3, 4, 5, 6, 7, 8, 9],
|
74
75
|
K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1, 2] = D.
|
75
76
|
All values in D require only 2 bits to encode.
|
76
77
|
Magic constant is shifted 6 bits left, to avoid final multiplication. */
|
77
|
-
offset = (offset <<
|
78
|
-
return (uint16_t)offset | bits64;
|
78
|
+
offset = (offset << 5u) + 0x40u + ((0x520D40u >> offset) & 0xC0u);
|
79
|
+
return (uint16_t)(offset | bits64);
|
79
80
|
}
|
80
81
|
}
|
81
82
|
|
@@ -88,70 +89,78 @@ static BROTLI_INLINE void GetLengthCode(size_t insertlen, size_t copylen,
|
|
88
89
|
}
|
89
90
|
|
90
91
|
static BROTLI_INLINE uint32_t GetInsertBase(uint16_t inscode) {
|
91
|
-
return
|
92
|
+
return kBrotliInsBase[inscode];
|
92
93
|
}
|
93
94
|
|
94
95
|
static BROTLI_INLINE uint32_t GetInsertExtra(uint16_t inscode) {
|
95
|
-
return
|
96
|
+
return kBrotliInsExtra[inscode];
|
96
97
|
}
|
97
98
|
|
98
99
|
static BROTLI_INLINE uint32_t GetCopyBase(uint16_t copycode) {
|
99
|
-
return
|
100
|
+
return kBrotliCopyBase[copycode];
|
100
101
|
}
|
101
102
|
|
102
103
|
static BROTLI_INLINE uint32_t GetCopyExtra(uint16_t copycode) {
|
103
|
-
return
|
104
|
+
return kBrotliCopyExtra[copycode];
|
104
105
|
}
|
105
106
|
|
106
107
|
typedef struct Command {
|
107
108
|
uint32_t insert_len_;
|
108
|
-
/* Stores copy_len in low
|
109
|
+
/* Stores copy_len in low 25 bits and copy_code - copy_len in high 7 bit. */
|
109
110
|
uint32_t copy_len_;
|
111
|
+
/* Stores distance extra bits. */
|
110
112
|
uint32_t dist_extra_;
|
111
113
|
uint16_t cmd_prefix_;
|
114
|
+
/* Stores distance code in low 10 bits
|
115
|
+
and number of extra bits in high 6 bits. */
|
112
116
|
uint16_t dist_prefix_;
|
113
117
|
} Command;
|
114
118
|
|
115
119
|
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
|
116
|
-
static BROTLI_INLINE void InitCommand(Command* self,
|
120
|
+
static BROTLI_INLINE void InitCommand(Command* self,
|
121
|
+
const BrotliDistanceParams* dist, size_t insertlen,
|
117
122
|
size_t copylen, int copylen_code_delta, size_t distance_code) {
|
118
123
|
/* Don't rely on signed int representation, use honest casts. */
|
119
124
|
uint32_t delta = (uint8_t)((int8_t)copylen_code_delta);
|
120
125
|
self->insert_len_ = (uint32_t)insertlen;
|
121
|
-
self->copy_len_ = (uint32_t)(copylen | (delta <<
|
126
|
+
self->copy_len_ = (uint32_t)(copylen | (delta << 25));
|
122
127
|
/* The distance prefix and extra bits are stored in this Command as if
|
123
128
|
npostfix and ndirect were 0, they are only recomputed later after the
|
124
129
|
clustering if needed. */
|
125
130
|
PrefixEncodeCopyDistance(
|
126
|
-
distance_code,
|
131
|
+
distance_code, dist->num_direct_distance_codes,
|
132
|
+
dist->distance_postfix_bits, &self->dist_prefix_, &self->dist_extra_);
|
127
133
|
GetLengthCode(
|
128
134
|
insertlen, (size_t)((int)copylen + copylen_code_delta),
|
129
|
-
TO_BROTLI_BOOL(self->dist_prefix_ == 0), &self->cmd_prefix_);
|
135
|
+
TO_BROTLI_BOOL((self->dist_prefix_ & 0x3FF) == 0), &self->cmd_prefix_);
|
130
136
|
}
|
131
137
|
|
132
138
|
static BROTLI_INLINE void InitInsertCommand(Command* self, size_t insertlen) {
|
133
139
|
self->insert_len_ = (uint32_t)insertlen;
|
134
|
-
self->copy_len_ = 4 <<
|
140
|
+
self->copy_len_ = 4 << 25;
|
135
141
|
self->dist_extra_ = 0;
|
136
142
|
self->dist_prefix_ = BROTLI_NUM_DISTANCE_SHORT_CODES;
|
137
143
|
GetLengthCode(insertlen, 4, BROTLI_FALSE, &self->cmd_prefix_);
|
138
144
|
}
|
139
145
|
|
140
|
-
static BROTLI_INLINE uint32_t CommandRestoreDistanceCode(
|
141
|
-
|
142
|
-
|
146
|
+
static BROTLI_INLINE uint32_t CommandRestoreDistanceCode(
|
147
|
+
const Command* self, const BrotliDistanceParams* dist) {
|
148
|
+
if ((self->dist_prefix_ & 0x3FFu) <
|
149
|
+
BROTLI_NUM_DISTANCE_SHORT_CODES + dist->num_direct_distance_codes) {
|
150
|
+
return self->dist_prefix_ & 0x3FFu;
|
143
151
|
} else {
|
144
|
-
uint32_t
|
145
|
-
uint32_t
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
return (
|
152
|
+
uint32_t dcode = self->dist_prefix_ & 0x3FFu;
|
153
|
+
uint32_t nbits = self->dist_prefix_ >> 10;
|
154
|
+
uint32_t extra = self->dist_extra_;
|
155
|
+
uint32_t postfix_mask = (1U << dist->distance_postfix_bits) - 1U;
|
156
|
+
uint32_t hcode = (dcode - dist->num_direct_distance_codes -
|
157
|
+
BROTLI_NUM_DISTANCE_SHORT_CODES) >>
|
158
|
+
dist->distance_postfix_bits;
|
159
|
+
uint32_t lcode = (dcode - dist->num_direct_distance_codes -
|
160
|
+
BROTLI_NUM_DISTANCE_SHORT_CODES) & postfix_mask;
|
161
|
+
uint32_t offset = ((2U + (hcode & 1U)) << nbits) - 4U;
|
162
|
+
return ((offset + extra) << dist->distance_postfix_bits) + lcode +
|
163
|
+
dist->num_direct_distance_codes + BROTLI_NUM_DISTANCE_SHORT_CODES;
|
155
164
|
}
|
156
165
|
}
|
157
166
|
|
@@ -165,12 +174,13 @@ static BROTLI_INLINE uint32_t CommandDistanceContext(const Command* self) {
|
|
165
174
|
}
|
166
175
|
|
167
176
|
static BROTLI_INLINE uint32_t CommandCopyLen(const Command* self) {
|
168
|
-
return self->copy_len_ &
|
177
|
+
return self->copy_len_ & 0x1FFFFFF;
|
169
178
|
}
|
170
179
|
|
171
180
|
static BROTLI_INLINE uint32_t CommandCopyLenCode(const Command* self) {
|
172
|
-
|
173
|
-
|
181
|
+
uint32_t modifier = self->copy_len_ >> 25;
|
182
|
+
int32_t delta = (int8_t)((uint8_t)(modifier | ((modifier & 0x40) << 1)));
|
183
|
+
return (uint32_t)((int32_t)(self->copy_len_ & 0x1FFFFFF) + delta);
|
174
184
|
}
|
175
185
|
|
176
186
|
#if defined(__cplusplus) || defined(c_plusplus)
|
@@ -17,16 +17,15 @@
|
|
17
17
|
#include <string.h> /* memcmp, memcpy, memset */
|
18
18
|
|
19
19
|
#include "../common/constants.h"
|
20
|
+
#include "../common/platform.h"
|
20
21
|
#include <brotli/types.h>
|
21
22
|
#include "./brotli_bit_stream.h"
|
22
23
|
#include "./entropy_encode.h"
|
23
24
|
#include "./fast_log.h"
|
24
25
|
#include "./find_match_length.h"
|
25
26
|
#include "./memory.h"
|
26
|
-
#include "./port.h"
|
27
27
|
#include "./write_bits.h"
|
28
28
|
|
29
|
-
|
30
29
|
#if defined(__cplusplus) || defined(c_plusplus)
|
31
30
|
extern "C" {
|
32
31
|
#endif
|
@@ -39,7 +38,7 @@ extern "C" {
|
|
39
38
|
* There is no effort to ensure that it is a prime, the oddity is enough
|
40
39
|
for this use.
|
41
40
|
* The number has been tuned heuristically against compression benchmarks. */
|
42
|
-
static const uint32_t kHashMul32 =
|
41
|
+
static const uint32_t kHashMul32 = 0x1E35A7BD;
|
43
42
|
|
44
43
|
static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) {
|
45
44
|
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(p) << 24) * kHashMul32;
|
@@ -48,8 +47,8 @@ static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) {
|
|
48
47
|
|
49
48
|
static BROTLI_INLINE uint32_t HashBytesAtOffset(
|
50
49
|
uint64_t v, int offset, size_t shift) {
|
51
|
-
|
52
|
-
|
50
|
+
BROTLI_DCHECK(offset >= 0);
|
51
|
+
BROTLI_DCHECK(offset <= 3);
|
53
52
|
{
|
54
53
|
const uint64_t h = ((v >> (8 * offset)) << 24) * kHashMul32;
|
55
54
|
return (uint32_t)(h >> shift);
|
@@ -58,7 +57,7 @@ static BROTLI_INLINE uint32_t HashBytesAtOffset(
|
|
58
57
|
|
59
58
|
static BROTLI_INLINE BROTLI_BOOL IsMatch(const uint8_t* p1, const uint8_t* p2) {
|
60
59
|
return TO_BROTLI_BOOL(
|
61
|
-
|
60
|
+
BrotliUnalignedRead32(p1) == BrotliUnalignedRead32(p2) &&
|
62
61
|
p1[4] == p2[4]);
|
63
62
|
}
|
64
63
|
|
@@ -203,7 +202,7 @@ static BROTLI_INLINE void EmitInsertLen(size_t insertlen,
|
|
203
202
|
} else {
|
204
203
|
BrotliWriteBits(depth[61], bits[61], storage_ix, storage);
|
205
204
|
BrotliWriteBits(12, insertlen - 2114, storage_ix, storage);
|
206
|
-
++histo[
|
205
|
+
++histo[61];
|
207
206
|
}
|
208
207
|
}
|
209
208
|
|
@@ -216,11 +215,11 @@ static BROTLI_INLINE void EmitLongInsertLen(size_t insertlen,
|
|
216
215
|
if (insertlen < 22594) {
|
217
216
|
BrotliWriteBits(depth[62], bits[62], storage_ix, storage);
|
218
217
|
BrotliWriteBits(14, insertlen - 6210, storage_ix, storage);
|
219
|
-
++histo[
|
218
|
+
++histo[62];
|
220
219
|
} else {
|
221
220
|
BrotliWriteBits(depth[63], bits[63], storage_ix, storage);
|
222
221
|
BrotliWriteBits(24, insertlen - 22594, storage_ix, storage);
|
223
|
-
++histo[
|
222
|
+
++histo[63];
|
224
223
|
}
|
225
224
|
}
|
226
225
|
|
@@ -252,7 +251,7 @@ static BROTLI_INLINE void EmitCopyLen(size_t copylen,
|
|
252
251
|
} else {
|
253
252
|
BrotliWriteBits(depth[39], bits[39], storage_ix, storage);
|
254
253
|
BrotliWriteBits(24, copylen - 2118, storage_ix, storage);
|
255
|
-
++histo[
|
254
|
+
++histo[39];
|
256
255
|
}
|
257
256
|
}
|
258
257
|
|
@@ -294,7 +293,7 @@ static BROTLI_INLINE void EmitCopyLenLastDistance(size_t copylen,
|
|
294
293
|
BrotliWriteBits(depth[39], bits[39], storage_ix, storage);
|
295
294
|
BrotliWriteBits(24, copylen - 2120, storage_ix, storage);
|
296
295
|
BrotliWriteBits(depth[64], bits[64], storage_ix, storage);
|
297
|
-
++histo[
|
296
|
+
++histo[39];
|
298
297
|
++histo[64];
|
299
298
|
}
|
300
299
|
}
|
@@ -344,7 +343,7 @@ static void BrotliStoreMetaBlockHeader(
|
|
344
343
|
}
|
345
344
|
|
346
345
|
static void UpdateBits(size_t n_bits, uint32_t bits, size_t pos,
|
347
|
-
uint8_t
|
346
|
+
uint8_t* array) {
|
348
347
|
while (n_bits > 0) {
|
349
348
|
size_t byte_pos = pos >> 3;
|
350
349
|
size_t n_unchanged_bits = pos & 7;
|
@@ -522,12 +521,12 @@ static BROTLI_INLINE void BrotliCompressFragmentFastImpl(
|
|
522
521
|
|
523
522
|
const uint8_t* next_ip = ip;
|
524
523
|
const uint8_t* candidate;
|
525
|
-
|
524
|
+
BROTLI_DCHECK(next_emit < ip);
|
526
525
|
trawl:
|
527
526
|
do {
|
528
527
|
uint32_t hash = next_hash;
|
529
528
|
uint32_t bytes_between_hash_lookups = skip++ >> 5;
|
530
|
-
|
529
|
+
BROTLI_DCHECK(hash == Hash(next_ip, shift));
|
531
530
|
ip = next_ip;
|
532
531
|
next_ip = ip + bytes_between_hash_lookups;
|
533
532
|
if (BROTLI_PREDICT_FALSE(next_ip > ip_limit)) {
|
@@ -542,8 +541,8 @@ trawl:
|
|
542
541
|
}
|
543
542
|
}
|
544
543
|
candidate = base_ip + table[hash];
|
545
|
-
|
546
|
-
|
544
|
+
BROTLI_DCHECK(candidate >= base_ip);
|
545
|
+
BROTLI_DCHECK(candidate < ip);
|
547
546
|
|
548
547
|
table[hash] = (int)(ip - base_ip);
|
549
548
|
} while (BROTLI_PREDICT_TRUE(!IsMatch(ip, candidate)));
|
@@ -566,7 +565,7 @@ trawl:
|
|
566
565
|
int distance = (int)(base - candidate); /* > 0 */
|
567
566
|
size_t insert = (size_t)(base - next_emit);
|
568
567
|
ip += matched;
|
569
|
-
|
568
|
+
BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
|
570
569
|
if (BROTLI_PREDICT_TRUE(insert < 6210)) {
|
571
570
|
EmitInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
|
572
571
|
storage_ix, storage);
|
@@ -626,7 +625,7 @@ trawl:
|
|
626
625
|
if (ip - candidate > MAX_DISTANCE) break;
|
627
626
|
ip += matched;
|
628
627
|
last_distance = (int)(base - candidate); /* > 0 */
|
629
|
-
|
628
|
+
BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
|
630
629
|
EmitCopyLen(matched, cmd_depth, cmd_bits, cmd_histo,
|
631
630
|
storage_ix, storage);
|
632
631
|
EmitDistance((size_t)last_distance, cmd_depth, cmd_bits,
|
@@ -659,7 +658,7 @@ trawl:
|
|
659
658
|
}
|
660
659
|
|
661
660
|
emit_remainder:
|
662
|
-
|
661
|
+
BROTLI_DCHECK(next_emit <= ip_end);
|
663
662
|
input += block_size;
|
664
663
|
input_size -= block_size;
|
665
664
|
block_size = BROTLI_MIN(size_t, input_size, kMergeBlockSize);
|
@@ -669,7 +668,7 @@ trawl:
|
|
669
668
|
if (input_size > 0 &&
|
670
669
|
total_block_size + block_size <= (1 << 20) &&
|
671
670
|
ShouldMergeBlock(input, block_size, lit_depth)) {
|
672
|
-
|
671
|
+
BROTLI_DCHECK(total_block_size > (1 << 16));
|
673
672
|
/* Update the size of the current meta-block and continue emitting commands.
|
674
673
|
We can do this because the current size and the new size both have 5
|
675
674
|
nibbles. */
|
@@ -752,7 +751,7 @@ void BrotliCompressFragmentFast(
|
|
752
751
|
const size_t table_bits = Log2FloorNonZero(table_size);
|
753
752
|
|
754
753
|
if (input_size == 0) {
|
755
|
-
|
754
|
+
BROTLI_DCHECK(is_last);
|
756
755
|
BrotliWriteBits(1, 1, storage_ix, storage); /* islast */
|
757
756
|
BrotliWriteBits(1, 1, storage_ix, storage); /* isempty */
|
758
757
|
*storage_ix = (*storage_ix + 7u) & ~7u;
|
@@ -768,7 +767,7 @@ void BrotliCompressFragmentFast(
|
|
768
767
|
break;
|
769
768
|
FOR_TABLE_BITS_(CASE_)
|
770
769
|
#undef CASE_
|
771
|
-
default:
|
770
|
+
default: BROTLI_DCHECK(0); break;
|
772
771
|
}
|
773
772
|
|
774
773
|
/* If output is larger than single uncompressed block, rewrite it. */
|