brotli 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Gemfile +1 -0
- data/Rakefile +6 -1
- data/brotli.gemspec +1 -1
- data/docs/Brotli.html +485 -0
- data/docs/Brotli/Error.html +124 -0
- data/docs/_index.html +122 -0
- data/docs/class_list.html +51 -0
- data/docs/css/common.css +1 -0
- data/docs/css/full_list.css +58 -0
- data/docs/css/style.css +496 -0
- data/docs/file.README.html +127 -0
- data/docs/file_list.html +56 -0
- data/docs/frames.html +17 -0
- data/docs/index.html +127 -0
- data/docs/js/app.js +292 -0
- data/docs/js/full_list.js +216 -0
- data/docs/js/jquery.js +4 -0
- data/docs/method_list.html +67 -0
- data/docs/top-level-namespace.html +110 -0
- data/ext/brotli/brotli.c +20 -0
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/c/common/constants.h +13 -6
- data/vendor/brotli/c/{dec → common}/context.h +182 -172
- data/vendor/brotli/c/common/dictionary.bin +0 -0
- data/vendor/brotli/c/common/dictionary.bin.br +0 -0
- data/vendor/brotli/c/common/dictionary.c +1 -1
- data/vendor/brotli/c/common/dictionary.h +4 -4
- data/vendor/brotli/c/common/platform.h +509 -0
- data/vendor/brotli/c/common/transform.c +235 -0
- data/vendor/brotli/c/common/transform.h +80 -0
- data/vendor/brotli/c/common/version.h +8 -1
- data/vendor/brotli/c/dec/bit_reader.c +1 -1
- data/vendor/brotli/c/dec/bit_reader.h +35 -86
- data/vendor/brotli/c/dec/decode.c +322 -205
- data/vendor/brotli/c/dec/huffman.c +35 -37
- data/vendor/brotli/c/dec/huffman.h +13 -9
- data/vendor/brotli/c/dec/prefix.h +3 -4
- data/vendor/brotli/c/dec/state.c +26 -34
- data/vendor/brotli/c/dec/state.h +34 -23
- data/vendor/brotli/c/enc/backward_references.c +25 -15
- data/vendor/brotli/c/enc/backward_references.h +5 -6
- data/vendor/brotli/c/enc/backward_references_hq.c +94 -68
- data/vendor/brotli/c/enc/backward_references_hq.h +22 -25
- data/vendor/brotli/c/enc/backward_references_inc.h +10 -10
- data/vendor/brotli/c/enc/bit_cost.c +1 -1
- data/vendor/brotli/c/enc/bit_cost.h +5 -5
- data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
- data/vendor/brotli/c/enc/block_splitter.c +2 -3
- data/vendor/brotli/c/enc/block_splitter.h +1 -1
- data/vendor/brotli/c/enc/block_splitter_inc.h +11 -11
- data/vendor/brotli/c/enc/brotli_bit_stream.c +102 -101
- data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
- data/vendor/brotli/c/enc/cluster.c +1 -1
- data/vendor/brotli/c/enc/cluster.h +1 -1
- data/vendor/brotli/c/enc/command.h +40 -30
- data/vendor/brotli/c/enc/compress_fragment.c +21 -22
- data/vendor/brotli/c/enc/compress_fragment.h +1 -1
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +101 -68
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
- data/vendor/brotli/c/enc/dictionary_hash.c +1 -1
- data/vendor/brotli/c/enc/encode.c +262 -162
- data/vendor/brotli/c/enc/encoder_dict.c +32 -0
- data/vendor/brotli/c/enc/encoder_dict.h +41 -0
- data/vendor/brotli/c/enc/entropy_encode.c +14 -14
- data/vendor/brotli/c/enc/entropy_encode.h +5 -5
- data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
- data/vendor/brotli/c/enc/fast_log.h +4 -2
- data/vendor/brotli/c/enc/find_match_length.h +3 -3
- data/vendor/brotli/c/enc/hash.h +75 -24
- data/vendor/brotli/c/enc/hash_composite_inc.h +133 -0
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +9 -8
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +8 -8
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +10 -9
- data/vendor/brotli/c/enc/hash_rolling_inc.h +215 -0
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +9 -8
- data/vendor/brotli/c/enc/histogram.c +9 -6
- data/vendor/brotli/c/enc/histogram.h +6 -3
- data/vendor/brotli/c/enc/histogram_inc.h +1 -1
- data/vendor/brotli/c/enc/literal_cost.c +5 -5
- data/vendor/brotli/c/enc/literal_cost.h +2 -2
- data/vendor/brotli/c/enc/memory.c +5 -16
- data/vendor/brotli/c/enc/memory.h +40 -1
- data/vendor/brotli/c/enc/metablock.c +163 -25
- data/vendor/brotli/c/enc/metablock.h +13 -8
- data/vendor/brotli/c/enc/metablock_inc.h +1 -1
- data/vendor/brotli/c/enc/params.h +44 -0
- data/vendor/brotli/c/enc/prefix.h +3 -4
- data/vendor/brotli/c/enc/quality.h +29 -24
- data/vendor/brotli/c/enc/ringbuffer.h +15 -11
- data/vendor/brotli/c/enc/static_dict.c +49 -45
- data/vendor/brotli/c/enc/static_dict.h +4 -3
- data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
- data/vendor/brotli/c/enc/utf8_util.c +20 -20
- data/vendor/brotli/c/enc/utf8_util.h +1 -1
- data/vendor/brotli/c/enc/write_bits.h +16 -21
- data/vendor/brotli/c/include/brotli/decode.h +13 -8
- data/vendor/brotli/c/include/brotli/encode.h +33 -8
- data/vendor/brotli/c/include/brotli/port.h +211 -83
- data/vendor/brotli/c/include/brotli/types.h +0 -7
- metadata +33 -12
- data/vendor/brotli/c/dec/port.h +0 -168
- data/vendor/brotli/c/dec/transform.h +0 -300
- data/vendor/brotli/c/enc/context.h +0 -184
- data/vendor/brotli/c/enc/port.h +0 -184
@@ -0,0 +1,32 @@
|
|
1
|
+
/* Copyright 2017 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
#include "./encoder_dict.h"
|
8
|
+
|
9
|
+
#include "../common/dictionary.h"
|
10
|
+
#include "../common/transform.h"
|
11
|
+
#include "./dictionary_hash.h"
|
12
|
+
#include "./hash.h"
|
13
|
+
|
14
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
15
|
+
extern "C" {
|
16
|
+
#endif
|
17
|
+
|
18
|
+
void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict) {
|
19
|
+
dict->words = BrotliGetDictionary();
|
20
|
+
|
21
|
+
dict->hash_table = kStaticDictionaryHash;
|
22
|
+
dict->buckets = kStaticDictionaryBuckets;
|
23
|
+
dict->dict_words = kStaticDictionaryWords;
|
24
|
+
|
25
|
+
dict->cutoffTransformsCount = kCutoffTransformsCount;
|
26
|
+
dict->cutoffTransforms = kCutoffTransforms;
|
27
|
+
|
28
|
+
}
|
29
|
+
|
30
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
31
|
+
} /* extern "C" */
|
32
|
+
#endif
|
@@ -0,0 +1,41 @@
|
|
1
|
+
/* Copyright 2017 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
#ifndef BROTLI_ENC_ENCODER_DICT_H_
|
8
|
+
#define BROTLI_ENC_ENCODER_DICT_H_
|
9
|
+
|
10
|
+
#include "../common/dictionary.h"
|
11
|
+
#include "../common/platform.h"
|
12
|
+
#include <brotli/types.h>
|
13
|
+
#include "./static_dict_lut.h"
|
14
|
+
|
15
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
16
|
+
extern "C" {
|
17
|
+
#endif
|
18
|
+
|
19
|
+
/* Dictionary data (words and transforms) for 1 possible context */
|
20
|
+
typedef struct BrotliEncoderDictionary {
|
21
|
+
const BrotliDictionary* words;
|
22
|
+
|
23
|
+
/* cut off for fast encoder */
|
24
|
+
uint32_t cutoffTransformsCount;
|
25
|
+
uint64_t cutoffTransforms;
|
26
|
+
|
27
|
+
/* from dictionary_hash.h, for fast encoder */
|
28
|
+
const uint16_t* hash_table;
|
29
|
+
|
30
|
+
/* from static_dict_lut.h, for slow encoder */
|
31
|
+
const uint16_t* buckets;
|
32
|
+
const DictWord* dict_words;
|
33
|
+
} BrotliEncoderDictionary;
|
34
|
+
|
35
|
+
BROTLI_INTERNAL void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict);
|
36
|
+
|
37
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
38
|
+
} /* extern "C" */
|
39
|
+
#endif
|
40
|
+
|
41
|
+
#endif /* BROTLI_ENC_ENCODER_DICT_H_ */
|
@@ -11,8 +11,8 @@
|
|
11
11
|
#include <string.h> /* memset */
|
12
12
|
|
13
13
|
#include "../common/constants.h"
|
14
|
+
#include "../common/platform.h"
|
14
15
|
#include <brotli/types.h>
|
15
|
-
#include "./port.h"
|
16
16
|
|
17
17
|
#if defined(__cplusplus) || defined(c_plusplus)
|
18
18
|
extern "C" {
|
@@ -23,7 +23,7 @@ BROTLI_BOOL BrotliSetDepth(
|
|
23
23
|
int stack[16];
|
24
24
|
int level = 0;
|
25
25
|
int p = p0;
|
26
|
-
|
26
|
+
BROTLI_DCHECK(max_depth <= 15);
|
27
27
|
stack[0] = -1;
|
28
28
|
while (BROTLI_TRUE) {
|
29
29
|
if (pool[p].index_left_ >= 0) {
|
@@ -66,11 +66,11 @@ static BROTLI_INLINE BROTLI_BOOL SortHuffmanTree(
|
|
66
66
|
we are not planning to use this with extremely long blocks.
|
67
67
|
|
68
68
|
See http://en.wikipedia.org/wiki/Huffman_coding */
|
69
|
-
void BrotliCreateHuffmanTree(const uint32_t
|
69
|
+
void BrotliCreateHuffmanTree(const uint32_t* data,
|
70
70
|
const size_t length,
|
71
71
|
const int tree_limit,
|
72
72
|
HuffmanTree* tree,
|
73
|
-
uint8_t
|
73
|
+
uint8_t* depth) {
|
74
74
|
uint32_t count_limit;
|
75
75
|
HuffmanTree sentinel;
|
76
76
|
InitHuffmanTree(&sentinel, BROTLI_UINT32_MAX, -1, -1);
|
@@ -165,7 +165,7 @@ static void BrotliWriteHuffmanTreeRepetitions(
|
|
165
165
|
size_t* tree_size,
|
166
166
|
uint8_t* tree,
|
167
167
|
uint8_t* extra_bits_data) {
|
168
|
-
|
168
|
+
BROTLI_DCHECK(repetitions > 0);
|
169
169
|
if (previous_value != value) {
|
170
170
|
tree[*tree_size] = value;
|
171
171
|
extra_bits_data[*tree_size] = 0;
|
@@ -371,8 +371,8 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
|
|
371
371
|
}
|
372
372
|
|
373
373
|
static void DecideOverRleUse(const uint8_t* depth, const size_t length,
|
374
|
-
BROTLI_BOOL
|
375
|
-
BROTLI_BOOL
|
374
|
+
BROTLI_BOOL* use_rle_for_non_zero,
|
375
|
+
BROTLI_BOOL* use_rle_for_zero) {
|
376
376
|
size_t total_reps_zero = 0;
|
377
377
|
size_t total_reps_non_zero = 0;
|
378
378
|
size_t count_reps_zero = 1;
|
@@ -454,26 +454,26 @@ void BrotliWriteHuffmanTree(const uint8_t* depth,
|
|
454
454
|
|
455
455
|
static uint16_t BrotliReverseBits(size_t num_bits, uint16_t bits) {
|
456
456
|
static const size_t kLut[16] = { /* Pre-reversed 4-bit values. */
|
457
|
-
|
458
|
-
|
457
|
+
0x00, 0x08, 0x04, 0x0C, 0x02, 0x0A, 0x06, 0x0E,
|
458
|
+
0x01, 0x09, 0x05, 0x0D, 0x03, 0x0B, 0x07, 0x0F
|
459
459
|
};
|
460
|
-
size_t retval = kLut[bits &
|
460
|
+
size_t retval = kLut[bits & 0x0F];
|
461
461
|
size_t i;
|
462
462
|
for (i = 4; i < num_bits; i += 4) {
|
463
463
|
retval <<= 4;
|
464
464
|
bits = (uint16_t)(bits >> 4);
|
465
|
-
retval |= kLut[bits &
|
465
|
+
retval |= kLut[bits & 0x0F];
|
466
466
|
}
|
467
|
-
retval >>= ((0 - num_bits) &
|
467
|
+
retval >>= ((0 - num_bits) & 0x03);
|
468
468
|
return (uint16_t)retval;
|
469
469
|
}
|
470
470
|
|
471
471
|
/* 0..15 are values for bits */
|
472
472
|
#define MAX_HUFFMAN_BITS 16
|
473
473
|
|
474
|
-
void BrotliConvertBitDepthsToSymbols(const uint8_t
|
474
|
+
void BrotliConvertBitDepthsToSymbols(const uint8_t* depth,
|
475
475
|
size_t len,
|
476
|
-
uint16_t
|
476
|
+
uint16_t* bits) {
|
477
477
|
/* In Brotli, all bit depths are [1..15]
|
478
478
|
0 bit depth means that the symbol does not exist. */
|
479
479
|
uint16_t bl_count[MAX_HUFFMAN_BITS] = { 0 };
|
@@ -9,8 +9,8 @@
|
|
9
9
|
#ifndef BROTLI_ENC_ENTROPY_ENCODE_H_
|
10
10
|
#define BROTLI_ENC_ENTROPY_ENCODE_H_
|
11
11
|
|
12
|
+
#include "../common/platform.h"
|
12
13
|
#include <brotli/types.h>
|
13
|
-
#include "./port.h"
|
14
14
|
|
15
15
|
#if defined(__cplusplus) || defined(c_plusplus)
|
16
16
|
extern "C" {
|
@@ -46,11 +46,11 @@ BROTLI_INTERNAL BROTLI_BOOL BrotliSetDepth(
|
|
46
46
|
be at least 2 * length + 1 long.
|
47
47
|
|
48
48
|
See http://en.wikipedia.org/wiki/Huffman_coding */
|
49
|
-
BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t
|
49
|
+
BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t* data,
|
50
50
|
const size_t length,
|
51
51
|
const int tree_limit,
|
52
52
|
HuffmanTree* tree,
|
53
|
-
uint8_t
|
53
|
+
uint8_t* depth);
|
54
54
|
|
55
55
|
/* Change the population counts in a way that the consequent
|
56
56
|
Huffman tree compression, especially its RLE-part will be more
|
@@ -72,9 +72,9 @@ BROTLI_INTERNAL void BrotliWriteHuffmanTree(const uint8_t* depth,
|
|
72
72
|
uint8_t* extra_bits_data);
|
73
73
|
|
74
74
|
/* Get the actual bit values for a tree of bit depths. */
|
75
|
-
BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t
|
75
|
+
BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t* depth,
|
76
76
|
size_t len,
|
77
|
-
uint16_t
|
77
|
+
uint16_t* bits);
|
78
78
|
|
79
79
|
/* Input size optimized Shell sort. */
|
80
80
|
typedef BROTLI_BOOL (*HuffmanTreeComparator)(
|
@@ -10,7 +10,7 @@
|
|
10
10
|
#define BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_
|
11
11
|
|
12
12
|
#include "../common/constants.h"
|
13
|
-
#include
|
13
|
+
#include "../common/platform.h"
|
14
14
|
#include <brotli/types.h>
|
15
15
|
#include "./write_bits.h"
|
16
16
|
|
@@ -83,7 +83,7 @@ static const uint32_t kCodeLengthBits[18] = {
|
|
83
83
|
static BROTLI_INLINE void StoreStaticCodeLengthCode(
|
84
84
|
size_t* storage_ix, uint8_t* storage) {
|
85
85
|
BrotliWriteBits(
|
86
|
-
40, BROTLI_MAKE_UINT64_T(
|
86
|
+
40, BROTLI_MAKE_UINT64_T(0x0000FFu, 0x55555554u), storage_ix, storage);
|
87
87
|
}
|
88
88
|
|
89
89
|
static const uint64_t kZeroRepsBits[BROTLI_NUM_COMMAND_SYMBOLS] = {
|
@@ -529,7 +529,7 @@ static const uint16_t kStaticDistanceCodeBits[64] = {
|
|
529
529
|
|
530
530
|
static BROTLI_INLINE void StoreStaticDistanceHuffmanTree(
|
531
531
|
size_t* storage_ix, uint8_t* storage) {
|
532
|
-
BrotliWriteBits(28,
|
532
|
+
BrotliWriteBits(28, 0x0369DC03u, storage_ix, storage);
|
533
533
|
}
|
534
534
|
|
535
535
|
#if defined(__cplusplus) || defined(c_plusplus)
|
@@ -11,15 +11,17 @@
|
|
11
11
|
|
12
12
|
#include <math.h>
|
13
13
|
|
14
|
+
#include "../common/platform.h"
|
14
15
|
#include <brotli/types.h>
|
15
|
-
#include <brotli/port.h>
|
16
16
|
|
17
17
|
#if defined(__cplusplus) || defined(c_plusplus)
|
18
18
|
extern "C" {
|
19
19
|
#endif
|
20
20
|
|
21
21
|
static BROTLI_INLINE uint32_t Log2FloorNonZero(size_t n) {
|
22
|
-
|
22
|
+
/* TODO: generalize and move to platform.h */
|
23
|
+
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_clz, 3, 4, 0) || \
|
24
|
+
BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
|
23
25
|
return 31u ^ (uint32_t)__builtin_clz((uint32_t)n);
|
24
26
|
#else
|
25
27
|
uint32_t result = 0;
|
@@ -9,8 +9,8 @@
|
|
9
9
|
#ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_
|
10
10
|
#define BROTLI_ENC_FIND_MATCH_LENGTH_H_
|
11
11
|
|
12
|
+
#include "../common/platform.h"
|
12
13
|
#include <brotli/types.h>
|
13
|
-
#include "./port.h"
|
14
14
|
|
15
15
|
#if defined(__cplusplus) || defined(c_plusplus)
|
16
16
|
extern "C" {
|
@@ -60,8 +60,8 @@ static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
|
|
60
60
|
the first non-matching bit and use that to calculate the total
|
61
61
|
length of the match. */
|
62
62
|
while (s2_ptr <= s2_limit - 4 &&
|
63
|
-
|
64
|
-
|
63
|
+
BrotliUnalignedRead32(s2_ptr) ==
|
64
|
+
BrotliUnalignedRead32(s1 + matched)) {
|
65
65
|
s2_ptr += 4;
|
66
66
|
matched += 4;
|
67
67
|
}
|
data/vendor/brotli/c/enc/hash.h
CHANGED
@@ -14,11 +14,12 @@
|
|
14
14
|
|
15
15
|
#include "../common/constants.h"
|
16
16
|
#include "../common/dictionary.h"
|
17
|
+
#include "../common/platform.h"
|
17
18
|
#include <brotli/types.h>
|
19
|
+
#include "./encoder_dict.h"
|
18
20
|
#include "./fast_log.h"
|
19
21
|
#include "./find_match_length.h"
|
20
22
|
#include "./memory.h"
|
21
|
-
#include "./port.h"
|
22
23
|
#include "./quality.h"
|
23
24
|
#include "./static_dict.h"
|
24
25
|
|
@@ -35,8 +36,10 @@ extern "C" {
|
|
35
36
|
* * HasherCommon structure
|
36
37
|
* * private structured hasher data, depending on hasher type
|
37
38
|
* * private dynamic hasher data, depending on hasher type and parameters
|
38
|
-
|
39
|
-
|
39
|
+
*
|
40
|
+
* Using "define" instead of "typedef", because on MSVC __restrict does not work
|
41
|
+
* on typedef pointer types. */
|
42
|
+
#define HasherHandle uint8_t*
|
40
43
|
|
41
44
|
typedef struct {
|
42
45
|
BrotliHasherParams params;
|
@@ -73,13 +76,13 @@ typedef struct HasherSearchResult {
|
|
73
76
|
* There is no effort to ensure that it is a prime, the oddity is enough
|
74
77
|
for this use.
|
75
78
|
* The number has been tuned heuristically against compression benchmarks. */
|
76
|
-
static const uint32_t kHashMul32 =
|
77
|
-
static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(
|
79
|
+
static const uint32_t kHashMul32 = 0x1E35A7BD;
|
80
|
+
static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(0x1E35A7BD, 0x1E35A7BD);
|
78
81
|
static const uint64_t kHashMul64Long =
|
79
|
-
BROTLI_MAKE_UINT64_T(
|
82
|
+
BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
|
80
83
|
|
81
84
|
static BROTLI_INLINE uint32_t Hash14(const uint8_t* data) {
|
82
|
-
uint32_t h =
|
85
|
+
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
|
83
86
|
/* The higher bits contain more mixture from the multiplication,
|
84
87
|
so we take our results from there. */
|
85
88
|
return h >> (32 - 14);
|
@@ -146,34 +149,35 @@ static BROTLI_INLINE score_t BackwardReferencePenaltyUsingLastDistance(
|
|
146
149
|
}
|
147
150
|
|
148
151
|
static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
|
149
|
-
const
|
150
|
-
size_t max_length, size_t max_backward,
|
152
|
+
const BrotliEncoderDictionary* dictionary, size_t item, const uint8_t* data,
|
153
|
+
size_t max_length, size_t max_backward, size_t max_distance,
|
154
|
+
HasherSearchResult* out) {
|
151
155
|
size_t len;
|
152
|
-
size_t
|
156
|
+
size_t word_idx;
|
153
157
|
size_t offset;
|
154
158
|
size_t matchlen;
|
155
159
|
size_t backward;
|
156
160
|
score_t score;
|
157
161
|
len = item & 0x1F;
|
158
|
-
|
159
|
-
offset = dictionary->offsets_by_length[len] + len *
|
162
|
+
word_idx = item >> 5;
|
163
|
+
offset = dictionary->words->offsets_by_length[len] + len * word_idx;
|
160
164
|
if (len > max_length) {
|
161
165
|
return BROTLI_FALSE;
|
162
166
|
}
|
163
167
|
|
164
168
|
matchlen =
|
165
|
-
FindMatchLengthWithLimit(data, &dictionary->data[offset], len);
|
166
|
-
if (matchlen +
|
169
|
+
FindMatchLengthWithLimit(data, &dictionary->words->data[offset], len);
|
170
|
+
if (matchlen + dictionary->cutoffTransformsCount <= len || matchlen == 0) {
|
167
171
|
return BROTLI_FALSE;
|
168
172
|
}
|
169
173
|
{
|
170
174
|
size_t cut = len - matchlen;
|
171
|
-
size_t transform_id =
|
172
|
-
(
|
173
|
-
backward = max_backward +
|
174
|
-
(transform_id << dictionary->size_bits_by_length[len]);
|
175
|
+
size_t transform_id = (cut << 2) +
|
176
|
+
(size_t)((dictionary->cutoffTransforms >> (cut * 6)) & 0x3F);
|
177
|
+
backward = max_backward + 1 + word_idx +
|
178
|
+
(transform_id << dictionary->words->size_bits_by_length[len]);
|
175
179
|
}
|
176
|
-
if (backward
|
180
|
+
if (backward > max_distance) {
|
177
181
|
return BROTLI_FALSE;
|
178
182
|
}
|
179
183
|
score = BackwardReferenceScore(matchlen, backward);
|
@@ -188,9 +192,10 @@ static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
|
|
188
192
|
}
|
189
193
|
|
190
194
|
static BROTLI_INLINE void SearchInStaticDictionary(
|
191
|
-
const
|
195
|
+
const BrotliEncoderDictionary* dictionary,
|
192
196
|
HasherHandle handle, const uint8_t* data, size_t max_length,
|
193
|
-
size_t max_backward,
|
197
|
+
size_t max_backward, size_t max_distance,
|
198
|
+
HasherSearchResult* out, BROTLI_BOOL shallow) {
|
194
199
|
size_t key;
|
195
200
|
size_t i;
|
196
201
|
HasherCommon* self = GetHasherCommon(handle);
|
@@ -199,11 +204,11 @@ static BROTLI_INLINE void SearchInStaticDictionary(
|
|
199
204
|
}
|
200
205
|
key = Hash14(data) << 1;
|
201
206
|
for (i = 0; i < (shallow ? 1u : 2u); ++i, ++key) {
|
202
|
-
size_t item =
|
207
|
+
size_t item = dictionary->hash_table[key];
|
203
208
|
self->dict_num_lookups++;
|
204
209
|
if (item != 0) {
|
205
210
|
BROTLI_BOOL item_matches = TestStaticDictionaryItem(
|
206
|
-
dictionary, item, data, max_length, max_backward, out);
|
211
|
+
dictionary, item, data, max_length, max_backward, max_distance, out);
|
207
212
|
if (item_matches) {
|
208
213
|
self->dict_num_matches++;
|
209
214
|
}
|
@@ -338,11 +343,57 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
|
|
338
343
|
#undef BUCKET_BITS
|
339
344
|
#undef HASHER
|
340
345
|
|
346
|
+
/* fast large window hashers */
|
347
|
+
|
348
|
+
#define HASHER() HROLLING_FAST
|
349
|
+
#define CHUNKLEN 32
|
350
|
+
#define JUMP 4
|
351
|
+
#define NUMBUCKETS 16777216
|
352
|
+
#define MASK ((NUMBUCKETS * 64) - 1)
|
353
|
+
#include "./hash_rolling_inc.h" /* NOLINT(build/include) */
|
354
|
+
#undef JUMP
|
355
|
+
#undef HASHER
|
356
|
+
|
357
|
+
|
358
|
+
#define HASHER() HROLLING
|
359
|
+
#define JUMP 1
|
360
|
+
#include "./hash_rolling_inc.h" /* NOLINT(build/include) */
|
361
|
+
#undef MASK
|
362
|
+
#undef NUMBUCKETS
|
363
|
+
#undef JUMP
|
364
|
+
#undef CHUNKLEN
|
365
|
+
#undef HASHER
|
366
|
+
|
367
|
+
#define HASHER() H35
|
368
|
+
#define HASHER_A H3
|
369
|
+
#define HASHER_B HROLLING_FAST
|
370
|
+
#include "./hash_composite_inc.h" /* NOLINT(build/include) */
|
371
|
+
#undef HASHER_A
|
372
|
+
#undef HASHER_B
|
373
|
+
#undef HASHER
|
374
|
+
|
375
|
+
#define HASHER() H55
|
376
|
+
#define HASHER_A H54
|
377
|
+
#define HASHER_B HROLLING_FAST
|
378
|
+
#include "./hash_composite_inc.h" /* NOLINT(build/include) */
|
379
|
+
#undef HASHER_A
|
380
|
+
#undef HASHER_B
|
381
|
+
#undef HASHER
|
382
|
+
|
383
|
+
#define HASHER() H65
|
384
|
+
#define HASHER_A H6
|
385
|
+
#define HASHER_B HROLLING
|
386
|
+
#include "./hash_composite_inc.h" /* NOLINT(build/include) */
|
387
|
+
#undef HASHER_A
|
388
|
+
#undef HASHER_B
|
389
|
+
#undef HASHER
|
390
|
+
|
341
391
|
#undef FN
|
342
392
|
#undef CAT
|
343
393
|
#undef EXPAND_CAT
|
344
394
|
|
345
|
-
#define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)
|
395
|
+
#define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)\
|
396
|
+
H(35) H(55) H(65)
|
346
397
|
#define FOR_ALL_HASHERS(H) FOR_GENERIC_HASHERS(H) H(10)
|
347
398
|
|
348
399
|
static BROTLI_INLINE void DestroyHasher(
|
@@ -0,0 +1,133 @@
|
|
1
|
+
/* NOLINT(build/header_guard) */
|
2
|
+
/* Copyright 2018 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Distributed under MIT license.
|
5
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
6
|
+
*/
|
7
|
+
|
8
|
+
/* template parameters: FN, HASHER_A, HASHER_B */
|
9
|
+
|
10
|
+
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
|
11
|
+
and HASHER_B. */
|
12
|
+
|
13
|
+
#define HashComposite HASHER()
|
14
|
+
|
15
|
+
#define FN_A(X) EXPAND_CAT(X, HASHER_A)
|
16
|
+
#define FN_B(X) EXPAND_CAT(X, HASHER_B)
|
17
|
+
|
18
|
+
static BROTLI_INLINE size_t FN(HashTypeLength)(void) {
|
19
|
+
size_t a = FN_A(HashTypeLength)();
|
20
|
+
size_t b = FN_B(HashTypeLength)();
|
21
|
+
return a > b ? a : b;
|
22
|
+
}
|
23
|
+
|
24
|
+
static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
|
25
|
+
size_t a = FN_A(StoreLookahead)();
|
26
|
+
size_t b = FN_B(StoreLookahead)();
|
27
|
+
return a > b ? a : b;
|
28
|
+
}
|
29
|
+
|
30
|
+
typedef struct HashComposite {
|
31
|
+
HasherHandle ha;
|
32
|
+
HasherHandle hb;
|
33
|
+
const BrotliEncoderParams* params;
|
34
|
+
} HashComposite;
|
35
|
+
|
36
|
+
static BROTLI_INLINE HashComposite* FN(Self)(HasherHandle handle) {
|
37
|
+
return (HashComposite*)&(GetHasherCommon(handle)[1]);
|
38
|
+
}
|
39
|
+
|
40
|
+
static void FN(Initialize)(
|
41
|
+
HasherHandle handle, const BrotliEncoderParams* params) {
|
42
|
+
HashComposite* self = FN(Self)(handle);
|
43
|
+
self->ha = 0;
|
44
|
+
self->hb = 0;
|
45
|
+
self->params = params;
|
46
|
+
/* TODO: Initialize of the hashers is defered to Prepare (and params
|
47
|
+
remembered here) because we don't get the one_shot and input_size params
|
48
|
+
here that are needed to know the memory size of them. Instead provide
|
49
|
+
those params to all hashers FN(Initialize) */
|
50
|
+
}
|
51
|
+
|
52
|
+
static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
|
53
|
+
size_t input_size, const uint8_t* data) {
|
54
|
+
HashComposite* self = FN(Self)(handle);
|
55
|
+
if (!self->ha) {
|
56
|
+
HasherCommon* common_a;
|
57
|
+
HasherCommon* common_b;
|
58
|
+
|
59
|
+
self->ha = handle + sizeof(HasherCommon) + sizeof(HashComposite);
|
60
|
+
common_a = (HasherCommon*)self->ha;
|
61
|
+
common_a->params = self->params->hasher;
|
62
|
+
common_a->is_prepared_ = BROTLI_FALSE;
|
63
|
+
common_a->dict_num_lookups = 0;
|
64
|
+
common_a->dict_num_matches = 0;
|
65
|
+
FN_A(Initialize)(self->ha, self->params);
|
66
|
+
|
67
|
+
self->hb = self->ha + sizeof(HasherCommon) + FN_A(HashMemAllocInBytes)(
|
68
|
+
self->params, one_shot, input_size);
|
69
|
+
common_b = (HasherCommon*)self->hb;
|
70
|
+
common_b->params = self->params->hasher;
|
71
|
+
common_b->is_prepared_ = BROTLI_FALSE;
|
72
|
+
common_b->dict_num_lookups = 0;
|
73
|
+
common_b->dict_num_matches = 0;
|
74
|
+
FN_B(Initialize)(self->hb, self->params);
|
75
|
+
}
|
76
|
+
FN_A(Prepare)(self->ha, one_shot, input_size, data);
|
77
|
+
FN_B(Prepare)(self->hb, one_shot, input_size, data);
|
78
|
+
}
|
79
|
+
|
80
|
+
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
|
81
|
+
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
|
82
|
+
size_t input_size) {
|
83
|
+
return sizeof(HashComposite) + 2 * sizeof(HasherCommon) +
|
84
|
+
FN_A(HashMemAllocInBytes)(params, one_shot, input_size) +
|
85
|
+
FN_B(HashMemAllocInBytes)(params, one_shot, input_size);
|
86
|
+
}
|
87
|
+
|
88
|
+
static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
|
89
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
|
90
|
+
HashComposite* self = FN(Self)(handle);
|
91
|
+
FN_A(Store)(self->ha, data, mask, ix);
|
92
|
+
FN_B(Store)(self->hb, data, mask, ix);
|
93
|
+
}
|
94
|
+
|
95
|
+
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
|
96
|
+
const uint8_t* data, const size_t mask, const size_t ix_start,
|
97
|
+
const size_t ix_end) {
|
98
|
+
HashComposite* self = FN(Self)(handle);
|
99
|
+
FN_A(StoreRange)(self->ha, data, mask, ix_start, ix_end);
|
100
|
+
FN_B(StoreRange)(self->hb, data, mask, ix_start, ix_end);
|
101
|
+
}
|
102
|
+
|
103
|
+
static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
|
104
|
+
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
|
105
|
+
size_t ring_buffer_mask) {
|
106
|
+
HashComposite* self = FN(Self)(handle);
|
107
|
+
FN_A(StitchToPreviousBlock)(self->ha, num_bytes, position, ringbuffer,
|
108
|
+
ring_buffer_mask);
|
109
|
+
FN_B(StitchToPreviousBlock)(self->hb, num_bytes, position, ringbuffer,
|
110
|
+
ring_buffer_mask);
|
111
|
+
}
|
112
|
+
|
113
|
+
static BROTLI_INLINE void FN(PrepareDistanceCache)(
|
114
|
+
HasherHandle handle, int* BROTLI_RESTRICT distance_cache) {
|
115
|
+
HashComposite* self = FN(Self)(handle);
|
116
|
+
FN_A(PrepareDistanceCache)(self->ha, distance_cache);
|
117
|
+
FN_B(PrepareDistanceCache)(self->hb, distance_cache);
|
118
|
+
}
|
119
|
+
|
120
|
+
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
|
121
|
+
const BrotliEncoderDictionary* dictionary,
|
122
|
+
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
|
123
|
+
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
|
124
|
+
const size_t max_length, const size_t max_backward, const size_t gap,
|
125
|
+
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
|
126
|
+
HashComposite* self = FN(Self)(handle);
|
127
|
+
FN_A(FindLongestMatch)(self->ha, dictionary, data, ring_buffer_mask,
|
128
|
+
distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
|
129
|
+
FN_B(FindLongestMatch)(self->hb, dictionary, data, ring_buffer_mask,
|
130
|
+
distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
|
131
|
+
}
|
132
|
+
|
133
|
+
#undef HashComposite
|