brotli 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile +1 -0
- data/Rakefile +6 -1
- data/brotli.gemspec +1 -1
- data/docs/Brotli.html +485 -0
- data/docs/Brotli/Error.html +124 -0
- data/docs/_index.html +122 -0
- data/docs/class_list.html +51 -0
- data/docs/css/common.css +1 -0
- data/docs/css/full_list.css +58 -0
- data/docs/css/style.css +496 -0
- data/docs/file.README.html +127 -0
- data/docs/file_list.html +56 -0
- data/docs/frames.html +17 -0
- data/docs/index.html +127 -0
- data/docs/js/app.js +292 -0
- data/docs/js/full_list.js +216 -0
- data/docs/js/jquery.js +4 -0
- data/docs/method_list.html +67 -0
- data/docs/top-level-namespace.html +110 -0
- data/ext/brotli/brotli.c +20 -0
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/c/common/constants.h +13 -6
- data/vendor/brotli/c/{dec → common}/context.h +182 -172
- data/vendor/brotli/c/common/dictionary.bin +0 -0
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +1 -1
- data/vendor/brotli/c/common/dictionary.h +4 -4
- data/vendor/brotli/c/common/platform.h +509 -0
- data/vendor/brotli/c/common/transform.c +235 -0
- data/vendor/brotli/c/common/transform.h +80 -0
- data/vendor/brotli/c/common/version.h +8 -1
- data/vendor/brotli/c/dec/bit_reader.c +1 -1
- data/vendor/brotli/c/dec/bit_reader.h +35 -86
- data/vendor/brotli/c/dec/decode.c +322 -205
- data/vendor/brotli/c/dec/huffman.c +35 -37
- data/vendor/brotli/c/dec/huffman.h +13 -9
- data/vendor/brotli/c/dec/prefix.h +3 -4
- data/vendor/brotli/c/dec/state.c +26 -34
- data/vendor/brotli/c/dec/state.h +34 -23
- data/vendor/brotli/c/enc/backward_references.c +25 -15
- data/vendor/brotli/c/enc/backward_references.h +5 -6
- data/vendor/brotli/c/enc/backward_references_hq.c +94 -68
- data/vendor/brotli/c/enc/backward_references_hq.h +22 -25
- data/vendor/brotli/c/enc/backward_references_inc.h +10 -10
- data/vendor/brotli/c/enc/bit_cost.c +1 -1
- data/vendor/brotli/c/enc/bit_cost.h +5 -5
- data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
- data/vendor/brotli/c/enc/block_splitter.c +2 -3
- data/vendor/brotli/c/enc/block_splitter.h +1 -1
- data/vendor/brotli/c/enc/block_splitter_inc.h +11 -11
- data/vendor/brotli/c/enc/brotli_bit_stream.c +102 -101
- data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
- data/vendor/brotli/c/enc/cluster.c +1 -1
- data/vendor/brotli/c/enc/cluster.h +1 -1
- data/vendor/brotli/c/enc/command.h +40 -30
- data/vendor/brotli/c/enc/compress_fragment.c +21 -22
- data/vendor/brotli/c/enc/compress_fragment.h +1 -1
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +101 -68
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1 -1
- data/vendor/brotli/c/enc/encode.c +262 -162
- data/vendor/brotli/c/enc/encoder_dict.c +32 -0
- data/vendor/brotli/c/enc/encoder_dict.h +41 -0
- data/vendor/brotli/c/enc/entropy_encode.c +14 -14
- data/vendor/brotli/c/enc/entropy_encode.h +5 -5
- data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
- data/vendor/brotli/c/enc/fast_log.h +4 -2
- data/vendor/brotli/c/enc/find_match_length.h +3 -3
- data/vendor/brotli/c/enc/hash.h +75 -24
- data/vendor/brotli/c/enc/hash_composite_inc.h +133 -0
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +9 -8
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +10 -9
- data/vendor/brotli/c/enc/hash_rolling_inc.h +215 -0
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +9 -8
- data/vendor/brotli/c/enc/histogram.c +9 -6
- data/vendor/brotli/c/enc/histogram.h +6 -3
- data/vendor/brotli/c/enc/histogram_inc.h +1 -1
- data/vendor/brotli/c/enc/literal_cost.c +5 -5
- data/vendor/brotli/c/enc/literal_cost.h +2 -2
- data/vendor/brotli/c/enc/memory.c +5 -16
- data/vendor/brotli/c/enc/memory.h +40 -1
- data/vendor/brotli/c/enc/metablock.c +163 -25
- data/vendor/brotli/c/enc/metablock.h +13 -8
- data/vendor/brotli/c/enc/metablock_inc.h +1 -1
- data/vendor/brotli/c/enc/params.h +44 -0
- data/vendor/brotli/c/enc/prefix.h +3 -4
- data/vendor/brotli/c/enc/quality.h +29 -24
- data/vendor/brotli/c/enc/ringbuffer.h +15 -11
- data/vendor/brotli/c/enc/static_dict.c +49 -45
- data/vendor/brotli/c/enc/static_dict.h +4 -3
- data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
- data/vendor/brotli/c/enc/utf8_util.c +20 -20
- data/vendor/brotli/c/enc/utf8_util.h +1 -1
- data/vendor/brotli/c/enc/write_bits.h +16 -21
- data/vendor/brotli/c/include/brotli/decode.h +13 -8
- data/vendor/brotli/c/include/brotli/encode.h +33 -8
- data/vendor/brotli/c/include/brotli/port.h +211 -83
- data/vendor/brotli/c/include/brotli/types.h +0 -7
- metadata +33 -12
- data/vendor/brotli/c/dec/port.h +0 -168
- data/vendor/brotli/c/dec/transform.h +0 -300
- data/vendor/brotli/c/enc/context.h +0 -184
- data/vendor/brotli/c/enc/port.h +0 -184
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/* Copyright 2017 Google Inc. All Rights Reserved.
|
|
2
|
+
|
|
3
|
+
Distributed under MIT license.
|
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
#include "./encoder_dict.h"
|
|
8
|
+
|
|
9
|
+
#include "../common/dictionary.h"
|
|
10
|
+
#include "../common/transform.h"
|
|
11
|
+
#include "./dictionary_hash.h"
|
|
12
|
+
#include "./hash.h"
|
|
13
|
+
|
|
14
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
|
15
|
+
extern "C" {
|
|
16
|
+
#endif
|
|
17
|
+
|
|
18
|
+
void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict) {
|
|
19
|
+
dict->words = BrotliGetDictionary();
|
|
20
|
+
|
|
21
|
+
dict->hash_table = kStaticDictionaryHash;
|
|
22
|
+
dict->buckets = kStaticDictionaryBuckets;
|
|
23
|
+
dict->dict_words = kStaticDictionaryWords;
|
|
24
|
+
|
|
25
|
+
dict->cutoffTransformsCount = kCutoffTransformsCount;
|
|
26
|
+
dict->cutoffTransforms = kCutoffTransforms;
|
|
27
|
+
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
|
31
|
+
} /* extern "C" */
|
|
32
|
+
#endif
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/* Copyright 2017 Google Inc. All Rights Reserved.
|
|
2
|
+
|
|
3
|
+
Distributed under MIT license.
|
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
#ifndef BROTLI_ENC_ENCODER_DICT_H_
|
|
8
|
+
#define BROTLI_ENC_ENCODER_DICT_H_
|
|
9
|
+
|
|
10
|
+
#include "../common/dictionary.h"
|
|
11
|
+
#include "../common/platform.h"
|
|
12
|
+
#include <brotli/types.h>
|
|
13
|
+
#include "./static_dict_lut.h"
|
|
14
|
+
|
|
15
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
|
16
|
+
extern "C" {
|
|
17
|
+
#endif
|
|
18
|
+
|
|
19
|
+
/* Dictionary data (words and transforms) for 1 possible context */
|
|
20
|
+
typedef struct BrotliEncoderDictionary {
|
|
21
|
+
const BrotliDictionary* words;
|
|
22
|
+
|
|
23
|
+
/* cut off for fast encoder */
|
|
24
|
+
uint32_t cutoffTransformsCount;
|
|
25
|
+
uint64_t cutoffTransforms;
|
|
26
|
+
|
|
27
|
+
/* from dictionary_hash.h, for fast encoder */
|
|
28
|
+
const uint16_t* hash_table;
|
|
29
|
+
|
|
30
|
+
/* from static_dict_lut.h, for slow encoder */
|
|
31
|
+
const uint16_t* buckets;
|
|
32
|
+
const DictWord* dict_words;
|
|
33
|
+
} BrotliEncoderDictionary;
|
|
34
|
+
|
|
35
|
+
BROTLI_INTERNAL void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict);
|
|
36
|
+
|
|
37
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
|
38
|
+
} /* extern "C" */
|
|
39
|
+
#endif
|
|
40
|
+
|
|
41
|
+
#endif /* BROTLI_ENC_ENCODER_DICT_H_ */
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
#include <string.h> /* memset */
|
|
12
12
|
|
|
13
13
|
#include "../common/constants.h"
|
|
14
|
+
#include "../common/platform.h"
|
|
14
15
|
#include <brotli/types.h>
|
|
15
|
-
#include "./port.h"
|
|
16
16
|
|
|
17
17
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
18
18
|
extern "C" {
|
|
@@ -23,7 +23,7 @@ BROTLI_BOOL BrotliSetDepth(
|
|
|
23
23
|
int stack[16];
|
|
24
24
|
int level = 0;
|
|
25
25
|
int p = p0;
|
|
26
|
-
|
|
26
|
+
BROTLI_DCHECK(max_depth <= 15);
|
|
27
27
|
stack[0] = -1;
|
|
28
28
|
while (BROTLI_TRUE) {
|
|
29
29
|
if (pool[p].index_left_ >= 0) {
|
|
@@ -66,11 +66,11 @@ static BROTLI_INLINE BROTLI_BOOL SortHuffmanTree(
|
|
|
66
66
|
we are not planning to use this with extremely long blocks.
|
|
67
67
|
|
|
68
68
|
See http://en.wikipedia.org/wiki/Huffman_coding */
|
|
69
|
-
void BrotliCreateHuffmanTree(const uint32_t
|
|
69
|
+
void BrotliCreateHuffmanTree(const uint32_t* data,
|
|
70
70
|
const size_t length,
|
|
71
71
|
const int tree_limit,
|
|
72
72
|
HuffmanTree* tree,
|
|
73
|
-
uint8_t
|
|
73
|
+
uint8_t* depth) {
|
|
74
74
|
uint32_t count_limit;
|
|
75
75
|
HuffmanTree sentinel;
|
|
76
76
|
InitHuffmanTree(&sentinel, BROTLI_UINT32_MAX, -1, -1);
|
|
@@ -165,7 +165,7 @@ static void BrotliWriteHuffmanTreeRepetitions(
|
|
|
165
165
|
size_t* tree_size,
|
|
166
166
|
uint8_t* tree,
|
|
167
167
|
uint8_t* extra_bits_data) {
|
|
168
|
-
|
|
168
|
+
BROTLI_DCHECK(repetitions > 0);
|
|
169
169
|
if (previous_value != value) {
|
|
170
170
|
tree[*tree_size] = value;
|
|
171
171
|
extra_bits_data[*tree_size] = 0;
|
|
@@ -371,8 +371,8 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
|
|
|
371
371
|
}
|
|
372
372
|
|
|
373
373
|
static void DecideOverRleUse(const uint8_t* depth, const size_t length,
|
|
374
|
-
BROTLI_BOOL
|
|
375
|
-
BROTLI_BOOL
|
|
374
|
+
BROTLI_BOOL* use_rle_for_non_zero,
|
|
375
|
+
BROTLI_BOOL* use_rle_for_zero) {
|
|
376
376
|
size_t total_reps_zero = 0;
|
|
377
377
|
size_t total_reps_non_zero = 0;
|
|
378
378
|
size_t count_reps_zero = 1;
|
|
@@ -454,26 +454,26 @@ void BrotliWriteHuffmanTree(const uint8_t* depth,
|
|
|
454
454
|
|
|
455
455
|
static uint16_t BrotliReverseBits(size_t num_bits, uint16_t bits) {
|
|
456
456
|
static const size_t kLut[16] = { /* Pre-reversed 4-bit values. */
|
|
457
|
-
|
|
458
|
-
|
|
457
|
+
0x00, 0x08, 0x04, 0x0C, 0x02, 0x0A, 0x06, 0x0E,
|
|
458
|
+
0x01, 0x09, 0x05, 0x0D, 0x03, 0x0B, 0x07, 0x0F
|
|
459
459
|
};
|
|
460
|
-
size_t retval = kLut[bits &
|
|
460
|
+
size_t retval = kLut[bits & 0x0F];
|
|
461
461
|
size_t i;
|
|
462
462
|
for (i = 4; i < num_bits; i += 4) {
|
|
463
463
|
retval <<= 4;
|
|
464
464
|
bits = (uint16_t)(bits >> 4);
|
|
465
|
-
retval |= kLut[bits &
|
|
465
|
+
retval |= kLut[bits & 0x0F];
|
|
466
466
|
}
|
|
467
|
-
retval >>= ((0 - num_bits) &
|
|
467
|
+
retval >>= ((0 - num_bits) & 0x03);
|
|
468
468
|
return (uint16_t)retval;
|
|
469
469
|
}
|
|
470
470
|
|
|
471
471
|
/* 0..15 are values for bits */
|
|
472
472
|
#define MAX_HUFFMAN_BITS 16
|
|
473
473
|
|
|
474
|
-
void BrotliConvertBitDepthsToSymbols(const uint8_t
|
|
474
|
+
void BrotliConvertBitDepthsToSymbols(const uint8_t* depth,
|
|
475
475
|
size_t len,
|
|
476
|
-
uint16_t
|
|
476
|
+
uint16_t* bits) {
|
|
477
477
|
/* In Brotli, all bit depths are [1..15]
|
|
478
478
|
0 bit depth means that the symbol does not exist. */
|
|
479
479
|
uint16_t bl_count[MAX_HUFFMAN_BITS] = { 0 };
|
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
#ifndef BROTLI_ENC_ENTROPY_ENCODE_H_
|
|
10
10
|
#define BROTLI_ENC_ENTROPY_ENCODE_H_
|
|
11
11
|
|
|
12
|
+
#include "../common/platform.h"
|
|
12
13
|
#include <brotli/types.h>
|
|
13
|
-
#include "./port.h"
|
|
14
14
|
|
|
15
15
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
16
16
|
extern "C" {
|
|
@@ -46,11 +46,11 @@ BROTLI_INTERNAL BROTLI_BOOL BrotliSetDepth(
|
|
|
46
46
|
be at least 2 * length + 1 long.
|
|
47
47
|
|
|
48
48
|
See http://en.wikipedia.org/wiki/Huffman_coding */
|
|
49
|
-
BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t
|
|
49
|
+
BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t* data,
|
|
50
50
|
const size_t length,
|
|
51
51
|
const int tree_limit,
|
|
52
52
|
HuffmanTree* tree,
|
|
53
|
-
uint8_t
|
|
53
|
+
uint8_t* depth);
|
|
54
54
|
|
|
55
55
|
/* Change the population counts in a way that the consequent
|
|
56
56
|
Huffman tree compression, especially its RLE-part will be more
|
|
@@ -72,9 +72,9 @@ BROTLI_INTERNAL void BrotliWriteHuffmanTree(const uint8_t* depth,
|
|
|
72
72
|
uint8_t* extra_bits_data);
|
|
73
73
|
|
|
74
74
|
/* Get the actual bit values for a tree of bit depths. */
|
|
75
|
-
BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t
|
|
75
|
+
BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t* depth,
|
|
76
76
|
size_t len,
|
|
77
|
-
uint16_t
|
|
77
|
+
uint16_t* bits);
|
|
78
78
|
|
|
79
79
|
/* Input size optimized Shell sort. */
|
|
80
80
|
typedef BROTLI_BOOL (*HuffmanTreeComparator)(
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
#define BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_
|
|
11
11
|
|
|
12
12
|
#include "../common/constants.h"
|
|
13
|
-
#include
|
|
13
|
+
#include "../common/platform.h"
|
|
14
14
|
#include <brotli/types.h>
|
|
15
15
|
#include "./write_bits.h"
|
|
16
16
|
|
|
@@ -83,7 +83,7 @@ static const uint32_t kCodeLengthBits[18] = {
|
|
|
83
83
|
static BROTLI_INLINE void StoreStaticCodeLengthCode(
|
|
84
84
|
size_t* storage_ix, uint8_t* storage) {
|
|
85
85
|
BrotliWriteBits(
|
|
86
|
-
40, BROTLI_MAKE_UINT64_T(
|
|
86
|
+
40, BROTLI_MAKE_UINT64_T(0x0000FFu, 0x55555554u), storage_ix, storage);
|
|
87
87
|
}
|
|
88
88
|
|
|
89
89
|
static const uint64_t kZeroRepsBits[BROTLI_NUM_COMMAND_SYMBOLS] = {
|
|
@@ -529,7 +529,7 @@ static const uint16_t kStaticDistanceCodeBits[64] = {
|
|
|
529
529
|
|
|
530
530
|
static BROTLI_INLINE void StoreStaticDistanceHuffmanTree(
|
|
531
531
|
size_t* storage_ix, uint8_t* storage) {
|
|
532
|
-
BrotliWriteBits(28,
|
|
532
|
+
BrotliWriteBits(28, 0x0369DC03u, storage_ix, storage);
|
|
533
533
|
}
|
|
534
534
|
|
|
535
535
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
@@ -11,15 +11,17 @@
|
|
|
11
11
|
|
|
12
12
|
#include <math.h>
|
|
13
13
|
|
|
14
|
+
#include "../common/platform.h"
|
|
14
15
|
#include <brotli/types.h>
|
|
15
|
-
#include <brotli/port.h>
|
|
16
16
|
|
|
17
17
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
18
18
|
extern "C" {
|
|
19
19
|
#endif
|
|
20
20
|
|
|
21
21
|
static BROTLI_INLINE uint32_t Log2FloorNonZero(size_t n) {
|
|
22
|
-
|
|
22
|
+
/* TODO: generalize and move to platform.h */
|
|
23
|
+
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_clz, 3, 4, 0) || \
|
|
24
|
+
BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
|
|
23
25
|
return 31u ^ (uint32_t)__builtin_clz((uint32_t)n);
|
|
24
26
|
#else
|
|
25
27
|
uint32_t result = 0;
|
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
#ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_
|
|
10
10
|
#define BROTLI_ENC_FIND_MATCH_LENGTH_H_
|
|
11
11
|
|
|
12
|
+
#include "../common/platform.h"
|
|
12
13
|
#include <brotli/types.h>
|
|
13
|
-
#include "./port.h"
|
|
14
14
|
|
|
15
15
|
#if defined(__cplusplus) || defined(c_plusplus)
|
|
16
16
|
extern "C" {
|
|
@@ -60,8 +60,8 @@ static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
|
|
|
60
60
|
the first non-matching bit and use that to calculate the total
|
|
61
61
|
length of the match. */
|
|
62
62
|
while (s2_ptr <= s2_limit - 4 &&
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
BrotliUnalignedRead32(s2_ptr) ==
|
|
64
|
+
BrotliUnalignedRead32(s1 + matched)) {
|
|
65
65
|
s2_ptr += 4;
|
|
66
66
|
matched += 4;
|
|
67
67
|
}
|
data/vendor/brotli/c/enc/hash.h
CHANGED
|
@@ -14,11 +14,12 @@
|
|
|
14
14
|
|
|
15
15
|
#include "../common/constants.h"
|
|
16
16
|
#include "../common/dictionary.h"
|
|
17
|
+
#include "../common/platform.h"
|
|
17
18
|
#include <brotli/types.h>
|
|
19
|
+
#include "./encoder_dict.h"
|
|
18
20
|
#include "./fast_log.h"
|
|
19
21
|
#include "./find_match_length.h"
|
|
20
22
|
#include "./memory.h"
|
|
21
|
-
#include "./port.h"
|
|
22
23
|
#include "./quality.h"
|
|
23
24
|
#include "./static_dict.h"
|
|
24
25
|
|
|
@@ -35,8 +36,10 @@ extern "C" {
|
|
|
35
36
|
* * HasherCommon structure
|
|
36
37
|
* * private structured hasher data, depending on hasher type
|
|
37
38
|
* * private dynamic hasher data, depending on hasher type and parameters
|
|
38
|
-
|
|
39
|
-
|
|
39
|
+
*
|
|
40
|
+
* Using "define" instead of "typedef", because on MSVC __restrict does not work
|
|
41
|
+
* on typedef pointer types. */
|
|
42
|
+
#define HasherHandle uint8_t*
|
|
40
43
|
|
|
41
44
|
typedef struct {
|
|
42
45
|
BrotliHasherParams params;
|
|
@@ -73,13 +76,13 @@ typedef struct HasherSearchResult {
|
|
|
73
76
|
* There is no effort to ensure that it is a prime, the oddity is enough
|
|
74
77
|
for this use.
|
|
75
78
|
* The number has been tuned heuristically against compression benchmarks. */
|
|
76
|
-
static const uint32_t kHashMul32 =
|
|
77
|
-
static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(
|
|
79
|
+
static const uint32_t kHashMul32 = 0x1E35A7BD;
|
|
80
|
+
static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(0x1E35A7BD, 0x1E35A7BD);
|
|
78
81
|
static const uint64_t kHashMul64Long =
|
|
79
|
-
BROTLI_MAKE_UINT64_T(
|
|
82
|
+
BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
|
|
80
83
|
|
|
81
84
|
static BROTLI_INLINE uint32_t Hash14(const uint8_t* data) {
|
|
82
|
-
uint32_t h =
|
|
85
|
+
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
|
83
86
|
/* The higher bits contain more mixture from the multiplication,
|
|
84
87
|
so we take our results from there. */
|
|
85
88
|
return h >> (32 - 14);
|
|
@@ -146,34 +149,35 @@ static BROTLI_INLINE score_t BackwardReferencePenaltyUsingLastDistance(
|
|
|
146
149
|
}
|
|
147
150
|
|
|
148
151
|
static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
|
|
149
|
-
const
|
|
150
|
-
size_t max_length, size_t max_backward,
|
|
152
|
+
const BrotliEncoderDictionary* dictionary, size_t item, const uint8_t* data,
|
|
153
|
+
size_t max_length, size_t max_backward, size_t max_distance,
|
|
154
|
+
HasherSearchResult* out) {
|
|
151
155
|
size_t len;
|
|
152
|
-
size_t
|
|
156
|
+
size_t word_idx;
|
|
153
157
|
size_t offset;
|
|
154
158
|
size_t matchlen;
|
|
155
159
|
size_t backward;
|
|
156
160
|
score_t score;
|
|
157
161
|
len = item & 0x1F;
|
|
158
|
-
|
|
159
|
-
offset = dictionary->offsets_by_length[len] + len *
|
|
162
|
+
word_idx = item >> 5;
|
|
163
|
+
offset = dictionary->words->offsets_by_length[len] + len * word_idx;
|
|
160
164
|
if (len > max_length) {
|
|
161
165
|
return BROTLI_FALSE;
|
|
162
166
|
}
|
|
163
167
|
|
|
164
168
|
matchlen =
|
|
165
|
-
FindMatchLengthWithLimit(data, &dictionary->data[offset], len);
|
|
166
|
-
if (matchlen +
|
|
169
|
+
FindMatchLengthWithLimit(data, &dictionary->words->data[offset], len);
|
|
170
|
+
if (matchlen + dictionary->cutoffTransformsCount <= len || matchlen == 0) {
|
|
167
171
|
return BROTLI_FALSE;
|
|
168
172
|
}
|
|
169
173
|
{
|
|
170
174
|
size_t cut = len - matchlen;
|
|
171
|
-
size_t transform_id =
|
|
172
|
-
(
|
|
173
|
-
backward = max_backward +
|
|
174
|
-
(transform_id << dictionary->size_bits_by_length[len]);
|
|
175
|
+
size_t transform_id = (cut << 2) +
|
|
176
|
+
(size_t)((dictionary->cutoffTransforms >> (cut * 6)) & 0x3F);
|
|
177
|
+
backward = max_backward + 1 + word_idx +
|
|
178
|
+
(transform_id << dictionary->words->size_bits_by_length[len]);
|
|
175
179
|
}
|
|
176
|
-
if (backward
|
|
180
|
+
if (backward > max_distance) {
|
|
177
181
|
return BROTLI_FALSE;
|
|
178
182
|
}
|
|
179
183
|
score = BackwardReferenceScore(matchlen, backward);
|
|
@@ -188,9 +192,10 @@ static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
|
|
|
188
192
|
}
|
|
189
193
|
|
|
190
194
|
static BROTLI_INLINE void SearchInStaticDictionary(
|
|
191
|
-
const
|
|
195
|
+
const BrotliEncoderDictionary* dictionary,
|
|
192
196
|
HasherHandle handle, const uint8_t* data, size_t max_length,
|
|
193
|
-
size_t max_backward,
|
|
197
|
+
size_t max_backward, size_t max_distance,
|
|
198
|
+
HasherSearchResult* out, BROTLI_BOOL shallow) {
|
|
194
199
|
size_t key;
|
|
195
200
|
size_t i;
|
|
196
201
|
HasherCommon* self = GetHasherCommon(handle);
|
|
@@ -199,11 +204,11 @@ static BROTLI_INLINE void SearchInStaticDictionary(
|
|
|
199
204
|
}
|
|
200
205
|
key = Hash14(data) << 1;
|
|
201
206
|
for (i = 0; i < (shallow ? 1u : 2u); ++i, ++key) {
|
|
202
|
-
size_t item =
|
|
207
|
+
size_t item = dictionary->hash_table[key];
|
|
203
208
|
self->dict_num_lookups++;
|
|
204
209
|
if (item != 0) {
|
|
205
210
|
BROTLI_BOOL item_matches = TestStaticDictionaryItem(
|
|
206
|
-
dictionary, item, data, max_length, max_backward, out);
|
|
211
|
+
dictionary, item, data, max_length, max_backward, max_distance, out);
|
|
207
212
|
if (item_matches) {
|
|
208
213
|
self->dict_num_matches++;
|
|
209
214
|
}
|
|
@@ -338,11 +343,57 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
|
|
|
338
343
|
#undef BUCKET_BITS
|
|
339
344
|
#undef HASHER
|
|
340
345
|
|
|
346
|
+
/* fast large window hashers */
|
|
347
|
+
|
|
348
|
+
#define HASHER() HROLLING_FAST
|
|
349
|
+
#define CHUNKLEN 32
|
|
350
|
+
#define JUMP 4
|
|
351
|
+
#define NUMBUCKETS 16777216
|
|
352
|
+
#define MASK ((NUMBUCKETS * 64) - 1)
|
|
353
|
+
#include "./hash_rolling_inc.h" /* NOLINT(build/include) */
|
|
354
|
+
#undef JUMP
|
|
355
|
+
#undef HASHER
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
#define HASHER() HROLLING
|
|
359
|
+
#define JUMP 1
|
|
360
|
+
#include "./hash_rolling_inc.h" /* NOLINT(build/include) */
|
|
361
|
+
#undef MASK
|
|
362
|
+
#undef NUMBUCKETS
|
|
363
|
+
#undef JUMP
|
|
364
|
+
#undef CHUNKLEN
|
|
365
|
+
#undef HASHER
|
|
366
|
+
|
|
367
|
+
#define HASHER() H35
|
|
368
|
+
#define HASHER_A H3
|
|
369
|
+
#define HASHER_B HROLLING_FAST
|
|
370
|
+
#include "./hash_composite_inc.h" /* NOLINT(build/include) */
|
|
371
|
+
#undef HASHER_A
|
|
372
|
+
#undef HASHER_B
|
|
373
|
+
#undef HASHER
|
|
374
|
+
|
|
375
|
+
#define HASHER() H55
|
|
376
|
+
#define HASHER_A H54
|
|
377
|
+
#define HASHER_B HROLLING_FAST
|
|
378
|
+
#include "./hash_composite_inc.h" /* NOLINT(build/include) */
|
|
379
|
+
#undef HASHER_A
|
|
380
|
+
#undef HASHER_B
|
|
381
|
+
#undef HASHER
|
|
382
|
+
|
|
383
|
+
#define HASHER() H65
|
|
384
|
+
#define HASHER_A H6
|
|
385
|
+
#define HASHER_B HROLLING
|
|
386
|
+
#include "./hash_composite_inc.h" /* NOLINT(build/include) */
|
|
387
|
+
#undef HASHER_A
|
|
388
|
+
#undef HASHER_B
|
|
389
|
+
#undef HASHER
|
|
390
|
+
|
|
341
391
|
#undef FN
|
|
342
392
|
#undef CAT
|
|
343
393
|
#undef EXPAND_CAT
|
|
344
394
|
|
|
345
|
-
#define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)
|
|
395
|
+
#define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)\
|
|
396
|
+
H(35) H(55) H(65)
|
|
346
397
|
#define FOR_ALL_HASHERS(H) FOR_GENERIC_HASHERS(H) H(10)
|
|
347
398
|
|
|
348
399
|
static BROTLI_INLINE void DestroyHasher(
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/* NOLINT(build/header_guard) */
|
|
2
|
+
/* Copyright 2018 Google Inc. All Rights Reserved.
|
|
3
|
+
|
|
4
|
+
Distributed under MIT license.
|
|
5
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/* template parameters: FN, HASHER_A, HASHER_B */
|
|
9
|
+
|
|
10
|
+
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
|
|
11
|
+
and HASHER_B. */
|
|
12
|
+
|
|
13
|
+
#define HashComposite HASHER()
|
|
14
|
+
|
|
15
|
+
#define FN_A(X) EXPAND_CAT(X, HASHER_A)
|
|
16
|
+
#define FN_B(X) EXPAND_CAT(X, HASHER_B)
|
|
17
|
+
|
|
18
|
+
static BROTLI_INLINE size_t FN(HashTypeLength)(void) {
|
|
19
|
+
size_t a = FN_A(HashTypeLength)();
|
|
20
|
+
size_t b = FN_B(HashTypeLength)();
|
|
21
|
+
return a > b ? a : b;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
|
|
25
|
+
size_t a = FN_A(StoreLookahead)();
|
|
26
|
+
size_t b = FN_B(StoreLookahead)();
|
|
27
|
+
return a > b ? a : b;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
typedef struct HashComposite {
|
|
31
|
+
HasherHandle ha;
|
|
32
|
+
HasherHandle hb;
|
|
33
|
+
const BrotliEncoderParams* params;
|
|
34
|
+
} HashComposite;
|
|
35
|
+
|
|
36
|
+
static BROTLI_INLINE HashComposite* FN(Self)(HasherHandle handle) {
|
|
37
|
+
return (HashComposite*)&(GetHasherCommon(handle)[1]);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
static void FN(Initialize)(
|
|
41
|
+
HasherHandle handle, const BrotliEncoderParams* params) {
|
|
42
|
+
HashComposite* self = FN(Self)(handle);
|
|
43
|
+
self->ha = 0;
|
|
44
|
+
self->hb = 0;
|
|
45
|
+
self->params = params;
|
|
46
|
+
/* TODO: Initialize of the hashers is defered to Prepare (and params
|
|
47
|
+
remembered here) because we don't get the one_shot and input_size params
|
|
48
|
+
here that are needed to know the memory size of them. Instead provide
|
|
49
|
+
those params to all hashers FN(Initialize) */
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
|
|
53
|
+
size_t input_size, const uint8_t* data) {
|
|
54
|
+
HashComposite* self = FN(Self)(handle);
|
|
55
|
+
if (!self->ha) {
|
|
56
|
+
HasherCommon* common_a;
|
|
57
|
+
HasherCommon* common_b;
|
|
58
|
+
|
|
59
|
+
self->ha = handle + sizeof(HasherCommon) + sizeof(HashComposite);
|
|
60
|
+
common_a = (HasherCommon*)self->ha;
|
|
61
|
+
common_a->params = self->params->hasher;
|
|
62
|
+
common_a->is_prepared_ = BROTLI_FALSE;
|
|
63
|
+
common_a->dict_num_lookups = 0;
|
|
64
|
+
common_a->dict_num_matches = 0;
|
|
65
|
+
FN_A(Initialize)(self->ha, self->params);
|
|
66
|
+
|
|
67
|
+
self->hb = self->ha + sizeof(HasherCommon) + FN_A(HashMemAllocInBytes)(
|
|
68
|
+
self->params, one_shot, input_size);
|
|
69
|
+
common_b = (HasherCommon*)self->hb;
|
|
70
|
+
common_b->params = self->params->hasher;
|
|
71
|
+
common_b->is_prepared_ = BROTLI_FALSE;
|
|
72
|
+
common_b->dict_num_lookups = 0;
|
|
73
|
+
common_b->dict_num_matches = 0;
|
|
74
|
+
FN_B(Initialize)(self->hb, self->params);
|
|
75
|
+
}
|
|
76
|
+
FN_A(Prepare)(self->ha, one_shot, input_size, data);
|
|
77
|
+
FN_B(Prepare)(self->hb, one_shot, input_size, data);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
|
81
|
+
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
|
82
|
+
size_t input_size) {
|
|
83
|
+
return sizeof(HashComposite) + 2 * sizeof(HasherCommon) +
|
|
84
|
+
FN_A(HashMemAllocInBytes)(params, one_shot, input_size) +
|
|
85
|
+
FN_B(HashMemAllocInBytes)(params, one_shot, input_size);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
|
|
89
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
|
|
90
|
+
HashComposite* self = FN(Self)(handle);
|
|
91
|
+
FN_A(Store)(self->ha, data, mask, ix);
|
|
92
|
+
FN_B(Store)(self->hb, data, mask, ix);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
|
96
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
|
97
|
+
const size_t ix_end) {
|
|
98
|
+
HashComposite* self = FN(Self)(handle);
|
|
99
|
+
FN_A(StoreRange)(self->ha, data, mask, ix_start, ix_end);
|
|
100
|
+
FN_B(StoreRange)(self->hb, data, mask, ix_start, ix_end);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
|
|
104
|
+
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
|
105
|
+
size_t ring_buffer_mask) {
|
|
106
|
+
HashComposite* self = FN(Self)(handle);
|
|
107
|
+
FN_A(StitchToPreviousBlock)(self->ha, num_bytes, position, ringbuffer,
|
|
108
|
+
ring_buffer_mask);
|
|
109
|
+
FN_B(StitchToPreviousBlock)(self->hb, num_bytes, position, ringbuffer,
|
|
110
|
+
ring_buffer_mask);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
|
114
|
+
HasherHandle handle, int* BROTLI_RESTRICT distance_cache) {
|
|
115
|
+
HashComposite* self = FN(Self)(handle);
|
|
116
|
+
FN_A(PrepareDistanceCache)(self->ha, distance_cache);
|
|
117
|
+
FN_B(PrepareDistanceCache)(self->hb, distance_cache);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
|
121
|
+
const BrotliEncoderDictionary* dictionary,
|
|
122
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
|
123
|
+
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
|
124
|
+
const size_t max_length, const size_t max_backward, const size_t gap,
|
|
125
|
+
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
|
|
126
|
+
HashComposite* self = FN(Self)(handle);
|
|
127
|
+
FN_A(FindLongestMatch)(self->ha, dictionary, data, ring_buffer_mask,
|
|
128
|
+
distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
|
|
129
|
+
FN_B(FindLongestMatch)(self->hb, dictionary, data, ring_buffer_mask,
|
|
130
|
+
distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
#undef HashComposite
|