brotli 0.1.3 → 0.1.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +3 -0
  3. data/.travis.yml +11 -3
  4. data/Gemfile +2 -0
  5. data/ext/brotli/brotli.c +279 -0
  6. data/ext/brotli/brotli.h +2 -0
  7. data/ext/brotli/buffer.c +95 -0
  8. data/ext/brotli/buffer.h +19 -0
  9. data/ext/brotli/extconf.rb +21 -81
  10. data/lib/brotli/version.rb +1 -1
  11. data/vendor/brotli/dec/bit_reader.c +5 -5
  12. data/vendor/brotli/dec/bit_reader.h +15 -15
  13. data/vendor/brotli/dec/context.h +1 -1
  14. data/vendor/brotli/dec/decode.c +433 -348
  15. data/vendor/brotli/dec/decode.h +74 -48
  16. data/vendor/brotli/dec/huffman.c +5 -4
  17. data/vendor/brotli/dec/huffman.h +4 -4
  18. data/vendor/brotli/dec/port.h +2 -95
  19. data/vendor/brotli/dec/prefix.h +5 -3
  20. data/vendor/brotli/dec/state.c +15 -27
  21. data/vendor/brotli/dec/state.h +21 -17
  22. data/vendor/brotli/dec/transform.h +1 -1
  23. data/vendor/brotli/enc/backward_references.c +892 -0
  24. data/vendor/brotli/enc/backward_references.h +85 -102
  25. data/vendor/brotli/enc/backward_references_inc.h +147 -0
  26. data/vendor/brotli/enc/bit_cost.c +35 -0
  27. data/vendor/brotli/enc/bit_cost.h +23 -121
  28. data/vendor/brotli/enc/bit_cost_inc.h +127 -0
  29. data/vendor/brotli/enc/block_encoder_inc.h +33 -0
  30. data/vendor/brotli/enc/block_splitter.c +197 -0
  31. data/vendor/brotli/enc/block_splitter.h +40 -50
  32. data/vendor/brotli/enc/block_splitter_inc.h +432 -0
  33. data/vendor/brotli/enc/brotli_bit_stream.c +1334 -0
  34. data/vendor/brotli/enc/brotli_bit_stream.h +95 -167
  35. data/vendor/brotli/enc/cluster.c +56 -0
  36. data/vendor/brotli/enc/cluster.h +23 -305
  37. data/vendor/brotli/enc/cluster_inc.h +315 -0
  38. data/vendor/brotli/enc/command.h +83 -76
  39. data/vendor/brotli/enc/compress_fragment.c +747 -0
  40. data/vendor/brotli/enc/compress_fragment.h +48 -37
  41. data/vendor/brotli/enc/compress_fragment_two_pass.c +557 -0
  42. data/vendor/brotli/enc/compress_fragment_two_pass.h +37 -26
  43. data/vendor/brotli/enc/compressor.cc +139 -0
  44. data/vendor/brotli/enc/compressor.h +146 -0
  45. data/vendor/brotli/enc/context.h +102 -96
  46. data/vendor/brotli/enc/dictionary_hash.h +9 -5
  47. data/vendor/brotli/enc/encode.c +1562 -0
  48. data/vendor/brotli/enc/encode.h +211 -199
  49. data/vendor/brotli/enc/encode_parallel.cc +161 -151
  50. data/vendor/brotli/enc/encode_parallel.h +7 -8
  51. data/vendor/brotli/enc/entropy_encode.c +501 -0
  52. data/vendor/brotli/enc/entropy_encode.h +107 -89
  53. data/vendor/brotli/enc/entropy_encode_static.h +29 -62
  54. data/vendor/brotli/enc/fast_log.h +26 -20
  55. data/vendor/brotli/enc/find_match_length.h +23 -20
  56. data/vendor/brotli/enc/hash.h +614 -871
  57. data/vendor/brotli/enc/hash_forgetful_chain_inc.h +249 -0
  58. data/vendor/brotli/enc/hash_longest_match_inc.h +241 -0
  59. data/vendor/brotli/enc/hash_longest_match_quickly_inc.h +230 -0
  60. data/vendor/brotli/enc/histogram.c +95 -0
  61. data/vendor/brotli/enc/histogram.h +49 -83
  62. data/vendor/brotli/enc/histogram_inc.h +51 -0
  63. data/vendor/brotli/enc/literal_cost.c +178 -0
  64. data/vendor/brotli/enc/literal_cost.h +16 -10
  65. data/vendor/brotli/enc/memory.c +181 -0
  66. data/vendor/brotli/enc/memory.h +62 -0
  67. data/vendor/brotli/enc/metablock.c +515 -0
  68. data/vendor/brotli/enc/metablock.h +87 -57
  69. data/vendor/brotli/enc/metablock_inc.h +183 -0
  70. data/vendor/brotli/enc/port.h +73 -47
  71. data/vendor/brotli/enc/prefix.h +34 -61
  72. data/vendor/brotli/enc/quality.h +130 -0
  73. data/vendor/brotli/enc/ringbuffer.h +137 -122
  74. data/vendor/brotli/enc/{static_dict.cc → static_dict.c} +162 -139
  75. data/vendor/brotli/enc/static_dict.h +23 -18
  76. data/vendor/brotli/enc/static_dict_lut.h +11223 -12037
  77. data/vendor/brotli/enc/streams.cc +7 -7
  78. data/vendor/brotli/enc/streams.h +32 -32
  79. data/vendor/brotli/enc/{utf8_util.cc → utf8_util.c} +22 -20
  80. data/vendor/brotli/enc/utf8_util.h +16 -9
  81. data/vendor/brotli/enc/write_bits.h +49 -43
  82. metadata +34 -25
  83. data/ext/brotli/brotli.cc +0 -181
  84. data/vendor/brotli/dec/Makefile +0 -12
  85. data/vendor/brotli/dec/dictionary.c +0 -9466
  86. data/vendor/brotli/dec/dictionary.h +0 -38
  87. data/vendor/brotli/dec/types.h +0 -38
  88. data/vendor/brotli/enc/Makefile +0 -14
  89. data/vendor/brotli/enc/backward_references.cc +0 -858
  90. data/vendor/brotli/enc/block_splitter.cc +0 -505
  91. data/vendor/brotli/enc/brotli_bit_stream.cc +0 -1181
  92. data/vendor/brotli/enc/compress_fragment.cc +0 -701
  93. data/vendor/brotli/enc/compress_fragment_two_pass.cc +0 -524
  94. data/vendor/brotli/enc/dictionary.cc +0 -9466
  95. data/vendor/brotli/enc/dictionary.h +0 -41
  96. data/vendor/brotli/enc/encode.cc +0 -1180
  97. data/vendor/brotli/enc/entropy_encode.cc +0 -480
  98. data/vendor/brotli/enc/histogram.cc +0 -67
  99. data/vendor/brotli/enc/literal_cost.cc +0 -165
  100. data/vendor/brotli/enc/metablock.cc +0 -539
  101. data/vendor/brotli/enc/transform.h +0 -248
  102. data/vendor/brotli/enc/types.h +0 -29
@@ -0,0 +1,183 @@
1
+ /* NOLINT(build/header_guard) */
2
+ /* Copyright 2015 Google Inc. All Rights Reserved.
3
+
4
+ Distributed under MIT license.
5
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
6
+ */
7
+
8
+ /* template parameters: FN */
9
+
10
+ #define HistogramType FN(Histogram)
11
+
12
+ /* Greedy block splitter for one block category (literal, command or distance).
13
+ */
14
+ typedef struct FN(BlockSplitter) {
15
+ /* Alphabet size of particular block category. */
16
+ size_t alphabet_size_;
17
+ /* We collect at least this many symbols for each block. */
18
+ size_t min_block_size_;
19
+ /* We merge histograms A and B if
20
+ entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,
21
+ where A is the current histogram and B is the histogram of the last or the
22
+ second last block type. */
23
+ double split_threshold_;
24
+
25
+ size_t num_blocks_;
26
+ BlockSplit* split_; /* not owned */
27
+ HistogramType* histograms_; /* not owned */
28
+ size_t* histograms_size_; /* not owned */
29
+
30
+ /* The number of symbols that we want to collect before deciding on whether
31
+ or not to merge the block with a previous one or emit a new block. */
32
+ size_t target_block_size_;
33
+ /* The number of symbols in the current histogram. */
34
+ size_t block_size_;
35
+ /* Offset of the current histogram. */
36
+ size_t curr_histogram_ix_;
37
+ /* Offset of the histograms of the previous two block types. */
38
+ size_t last_histogram_ix_[2];
39
+ /* Entropy of the previous two block types. */
40
+ double last_entropy_[2];
41
+ /* The number of times we merged the current block with the last one. */
42
+ size_t merge_last_count_;
43
+ } FN(BlockSplitter);
44
+
45
+ static void FN(InitBlockSplitter)(
46
+ MemoryManager* m, FN(BlockSplitter)* self, size_t alphabet_size,
47
+ size_t min_block_size, double split_threshold, size_t num_symbols,
48
+ BlockSplit* split, HistogramType** histograms, size_t* histograms_size) {
49
+ size_t max_num_blocks = num_symbols / min_block_size + 1;
50
+ /* We have to allocate one more histogram than the maximum number of block
51
+ types for the current histogram when the meta-block is too big. */
52
+ size_t max_num_types =
53
+ BROTLI_MIN(size_t, max_num_blocks, BROTLI_MAX_NUMBER_OF_BLOCK_TYPES + 1);
54
+ self->alphabet_size_ = alphabet_size;
55
+ self->min_block_size_ = min_block_size;
56
+ self->split_threshold_ = split_threshold;
57
+ self->num_blocks_ = 0;
58
+ self->split_ = split;
59
+ self->histograms_size_ = histograms_size;
60
+ self->target_block_size_ = min_block_size;
61
+ self->block_size_ = 0;
62
+ self->curr_histogram_ix_ = 0;
63
+ self->merge_last_count_ = 0;
64
+ BROTLI_ENSURE_CAPACITY(m, uint8_t,
65
+ split->types, split->types_alloc_size, max_num_blocks);
66
+ BROTLI_ENSURE_CAPACITY(m, uint32_t,
67
+ split->lengths, split->lengths_alloc_size, max_num_blocks);
68
+ if (BROTLI_IS_OOM(m)) return;
69
+ self->split_->num_blocks = max_num_blocks;
70
+ assert(*histograms == 0);
71
+ *histograms_size = max_num_types;
72
+ *histograms = BROTLI_ALLOC(m, HistogramType, *histograms_size);
73
+ self->histograms_ = *histograms;
74
+ if (BROTLI_IS_OOM(m)) return;
75
+ /* Clear only current histogram. */
76
+ FN(HistogramClear)(&self->histograms_[0]);
77
+ self->last_histogram_ix_[0] = self->last_histogram_ix_[1] = 0;
78
+ }
79
+
80
+ /* Does either of three things:
81
+ (1) emits the current block with a new block type;
82
+ (2) emits the current block with the type of the second last block;
83
+ (3) merges the current block with the last block. */
84
+ static void FN(BlockSplitterFinishBlock)(
85
+ FN(BlockSplitter)* self, BROTLI_BOOL is_final) {
86
+ BlockSplit* split = self->split_;
87
+ double* last_entropy = self->last_entropy_;
88
+ HistogramType* histograms = self->histograms_;
89
+ self->block_size_ =
90
+ BROTLI_MAX(size_t, self->block_size_, self->min_block_size_);
91
+ if (self->num_blocks_ == 0) {
92
+ /* Create first block. */
93
+ split->lengths[0] = (uint32_t)self->block_size_;
94
+ split->types[0] = 0;
95
+ last_entropy[0] =
96
+ BitsEntropy(histograms[0].data_, self->alphabet_size_);
97
+ last_entropy[1] = last_entropy[0];
98
+ ++self->num_blocks_;
99
+ ++split->num_types;
100
+ ++self->curr_histogram_ix_;
101
+ if (self->curr_histogram_ix_ < *self->histograms_size_)
102
+ FN(HistogramClear)(&histograms[self->curr_histogram_ix_]);
103
+ self->block_size_ = 0;
104
+ } else if (self->block_size_ > 0) {
105
+ double entropy = BitsEntropy(histograms[self->curr_histogram_ix_].data_,
106
+ self->alphabet_size_);
107
+ HistogramType combined_histo[2];
108
+ double combined_entropy[2];
109
+ double diff[2];
110
+ size_t j;
111
+ for (j = 0; j < 2; ++j) {
112
+ size_t last_histogram_ix = self->last_histogram_ix_[j];
113
+ combined_histo[j] = histograms[self->curr_histogram_ix_];
114
+ FN(HistogramAddHistogram)(&combined_histo[j],
115
+ &histograms[last_histogram_ix]);
116
+ combined_entropy[j] = BitsEntropy(
117
+ &combined_histo[j].data_[0], self->alphabet_size_);
118
+ diff[j] = combined_entropy[j] - entropy - last_entropy[j];
119
+ }
120
+
121
+ if (split->num_types < BROTLI_MAX_NUMBER_OF_BLOCK_TYPES &&
122
+ diff[0] > self->split_threshold_ &&
123
+ diff[1] > self->split_threshold_) {
124
+ /* Create new block. */
125
+ split->lengths[self->num_blocks_] = (uint32_t)self->block_size_;
126
+ split->types[self->num_blocks_] = (uint8_t)split->num_types;
127
+ self->last_histogram_ix_[1] = self->last_histogram_ix_[0];
128
+ self->last_histogram_ix_[0] = (uint8_t)split->num_types;
129
+ last_entropy[1] = last_entropy[0];
130
+ last_entropy[0] = entropy;
131
+ ++self->num_blocks_;
132
+ ++split->num_types;
133
+ ++self->curr_histogram_ix_;
134
+ if (self->curr_histogram_ix_ < *self->histograms_size_)
135
+ FN(HistogramClear)(&histograms[self->curr_histogram_ix_]);
136
+ self->block_size_ = 0;
137
+ self->merge_last_count_ = 0;
138
+ self->target_block_size_ = self->min_block_size_;
139
+ } else if (diff[1] < diff[0] - 20.0) {
140
+ /* Combine this block with second last block. */
141
+ split->lengths[self->num_blocks_] = (uint32_t)self->block_size_;
142
+ split->types[self->num_blocks_] = split->types[self->num_blocks_ - 2];
143
+ BROTLI_SWAP(size_t, self->last_histogram_ix_, 0, 1);
144
+ histograms[self->last_histogram_ix_[0]] = combined_histo[1];
145
+ last_entropy[1] = last_entropy[0];
146
+ last_entropy[0] = combined_entropy[1];
147
+ ++self->num_blocks_;
148
+ self->block_size_ = 0;
149
+ FN(HistogramClear)(&histograms[self->curr_histogram_ix_]);
150
+ self->merge_last_count_ = 0;
151
+ self->target_block_size_ = self->min_block_size_;
152
+ } else {
153
+ /* Combine this block with last block. */
154
+ split->lengths[self->num_blocks_ - 1] += (uint32_t)self->block_size_;
155
+ histograms[self->last_histogram_ix_[0]] = combined_histo[0];
156
+ last_entropy[0] = combined_entropy[0];
157
+ if (split->num_types == 1) {
158
+ last_entropy[1] = last_entropy[0];
159
+ }
160
+ self->block_size_ = 0;
161
+ FN(HistogramClear)(&histograms[self->curr_histogram_ix_]);
162
+ if (++self->merge_last_count_ > 1) {
163
+ self->target_block_size_ += self->min_block_size_;
164
+ }
165
+ }
166
+ }
167
+ if (is_final) {
168
+ *self->histograms_size_ = split->num_types;
169
+ split->num_blocks = self->num_blocks_;
170
+ }
171
+ }
172
+
173
+ /* Adds the next symbol to the current histogram. When the current histogram
174
+ reaches the target size, decides on merging the block. */
175
+ static void FN(BlockSplitterAddSymbol)(FN(BlockSplitter)* self, size_t symbol) {
176
+ FN(HistogramAdd)(&self->histograms_[self->curr_histogram_ix_], symbol);
177
+ ++self->block_size_;
178
+ if (self->block_size_ == self->target_block_size_) {
179
+ FN(BlockSplitterFinishBlock)(self, /* is_final = */ BROTLI_FALSE);
180
+ }
181
+ }
182
+
183
+ #undef HistogramType
@@ -4,14 +4,16 @@
4
4
  See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
5
  */
6
6
 
7
- // Macros for endianness, branch prediction and unaligned loads and stores.
7
+ /* Macros for endianness, branch prediction and unaligned loads and stores. */
8
8
 
9
9
  #ifndef BROTLI_ENC_PORT_H_
10
10
  #define BROTLI_ENC_PORT_H_
11
11
 
12
12
  #include <assert.h>
13
- #include <string.h>
14
- #include "./types.h"
13
+ #include <string.h> /* memcpy */
14
+
15
+ #include "../common/port.h"
16
+ #include "../common/types.h"
15
17
 
16
18
  #if defined OS_LINUX || defined OS_CYGWIN
17
19
  #include <endian.h>
@@ -24,9 +26,9 @@
24
26
  #define __LITTLE_ENDIAN LITTLE_ENDIAN
25
27
  #endif
26
28
 
27
- // define the macro IS_LITTLE_ENDIAN
28
- // using the above endian definitions from endian.h if
29
- // endian.h was included
29
+ /* define the macro IS_LITTLE_ENDIAN
30
+ using the above endian definitions from endian.h if
31
+ endian.h was included */
30
32
  #ifdef __BYTE_ORDER
31
33
  #if __BYTE_ORDER == __LITTLE_ENDIAN
32
34
  #define IS_LITTLE_ENDIAN
@@ -37,49 +39,36 @@
37
39
  #if defined(__LITTLE_ENDIAN__)
38
40
  #define IS_LITTLE_ENDIAN
39
41
  #endif
40
- #endif // __BYTE_ORDER
42
+ #endif /* __BYTE_ORDER */
41
43
 
42
44
  #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
43
45
  #define IS_LITTLE_ENDIAN
44
46
  #endif
45
47
 
46
- // Enable little-endian optimization for x64 architecture on Windows.
48
+ /* Enable little-endian optimization for x64 architecture on Windows. */
47
49
  #if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
48
50
  #define IS_LITTLE_ENDIAN
49
51
  #endif
50
52
 
51
- /* Compatibility with non-clang compilers. */
52
- #ifndef __has_builtin
53
- #define __has_builtin(x) 0
54
- #endif
55
-
56
- #if (__GNUC__ > 2) || (__GNUC__ == 2 && __GNUC_MINOR__ > 95) || \
57
- (defined(__llvm__) && __has_builtin(__builtin_expect))
58
- #define PREDICT_FALSE(x) (__builtin_expect(x, 0))
59
- #define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
60
- #else
61
- #define PREDICT_FALSE(x) (x)
62
- #define PREDICT_TRUE(x) (x)
63
- #endif
64
-
65
- // Portable handling of unaligned loads, stores, and copies.
66
- // On some platforms, like ARM, the copy functions can be more efficient
67
- // then a load and a store.
53
+ /* Portable handling of unaligned loads, stores, and copies.
54
+ On some platforms, like ARM, the copy functions can be more efficient
55
+ then a load and a store. */
68
56
 
69
57
  #if defined(ARCH_PIII) || \
70
58
  defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC)
71
59
 
72
- // x86 and x86-64 can perform unaligned loads/stores directly;
73
- // modern PowerPC hardware can also do unaligned integer loads and stores;
74
- // but note: the FPU still sends unaligned loads and stores to a trap handler!
60
+ /* x86 and x86-64 can perform unaligned loads/stores directly;
61
+ modern PowerPC hardware can also do unaligned integer loads and stores;
62
+ but note: the FPU still sends unaligned loads and stores to a trap handler!
63
+ */
75
64
 
76
- #define BROTLI_UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32_t *>(_p))
77
- #define BROTLI_UNALIGNED_LOAD64(_p) (*reinterpret_cast<const uint64_t *>(_p))
65
+ #define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
66
+ #define BROTLI_UNALIGNED_LOAD64(_p) (*(const uint64_t *)(_p))
78
67
 
79
68
  #define BROTLI_UNALIGNED_STORE32(_p, _val) \
80
- (*reinterpret_cast<uint32_t *>(_p) = (_val))
69
+ (*(uint32_t *)(_p) = (_val))
81
70
  #define BROTLI_UNALIGNED_STORE64(_p, _val) \
82
- (*reinterpret_cast<uint64_t *>(_p) = (_val))
71
+ (*(uint64_t *)(_p) = (_val))
83
72
 
84
73
  #elif defined(__arm__) && \
85
74
  !defined(__ARM_ARCH_5__) && \
@@ -93,50 +82,87 @@
93
82
  !defined(__ARM_ARCH_6ZK__) && \
94
83
  !defined(__ARM_ARCH_6T2__)
95
84
 
96
- // ARMv7 and newer support native unaligned accesses, but only of 16-bit
97
- // and 32-bit values (not 64-bit); older versions either raise a fatal signal,
98
- // do an unaligned read and rotate the words around a bit, or do the reads very
99
- // slowly (trip through kernel mode).
85
+ /* ARMv7 and newer support native unaligned accesses, but only of 16-bit
86
+ and 32-bit values (not 64-bit); older versions either raise a fatal signal,
87
+ do an unaligned read and rotate the words around a bit, or do the reads very
88
+ slowly (trip through kernel mode). */
100
89
 
101
- #define BROTLI_UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32_t *>(_p))
90
+ #define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
102
91
  #define BROTLI_UNALIGNED_STORE32(_p, _val) \
103
- (*reinterpret_cast<uint32_t *>(_p) = (_val))
92
+ (*(uint32_t *)(_p) = (_val))
104
93
 
105
- inline uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
94
+ static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
106
95
  uint64_t t;
107
96
  memcpy(&t, p, sizeof t);
108
97
  return t;
109
98
  }
110
99
 
111
- inline void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
100
+ static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
112
101
  memcpy(p, &v, sizeof v);
113
102
  }
114
103
 
115
104
  #else
116
105
 
117
- // These functions are provided for architectures that don't support
118
- // unaligned loads and stores.
106
+ /* These functions are provided for architectures that don't support */
107
+ /* unaligned loads and stores. */
119
108
 
120
- inline uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) {
109
+ static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) {
121
110
  uint32_t t;
122
111
  memcpy(&t, p, sizeof t);
123
112
  return t;
124
113
  }
125
114
 
126
- inline uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
115
+ static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
127
116
  uint64_t t;
128
117
  memcpy(&t, p, sizeof t);
129
118
  return t;
130
119
  }
131
120
 
132
- inline void BROTLI_UNALIGNED_STORE32(void *p, uint32_t v) {
121
+ static BROTLI_INLINE void BROTLI_UNALIGNED_STORE32(void *p, uint32_t v) {
133
122
  memcpy(p, &v, sizeof v);
134
123
  }
135
124
 
136
- inline void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
125
+ static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
137
126
  memcpy(p, &v, sizeof v);
138
127
  }
139
128
 
140
129
  #endif
141
130
 
142
- #endif // BROTLI_ENC_PORT_H_
131
+ #if !defined(__cplusplus) && !defined(c_plusplus) && __STDC_VERSION__ >= 199901L
132
+ #define BROTLI_RESTRICT restrict
133
+ #elif BROTLI_GCC_VERSION > 295 || defined(__llvm__)
134
+ #define BROTLI_RESTRICT __restrict
135
+ #else
136
+ #define BROTLI_RESTRICT
137
+ #endif
138
+
139
+ #define _TEMPLATE(T) \
140
+ static BROTLI_INLINE T brotli_min_ ## T (T a, T b) { return a < b ? a : b; } \
141
+ static BROTLI_INLINE T brotli_max_ ## T (T a, T b) { return a > b ? a : b; }
142
+ _TEMPLATE(double) _TEMPLATE(float) _TEMPLATE(int)
143
+ _TEMPLATE(size_t) _TEMPLATE(uint32_t) _TEMPLATE(uint8_t)
144
+ #undef _TEMPLATE
145
+ #define BROTLI_MIN(T, A, B) (brotli_min_ ## T((A), (B)))
146
+ #define BROTLI_MAX(T, A, B) (brotli_max_ ## T((A), (B)))
147
+
148
+ #define BROTLI_SWAP(T, A, I, J) { \
149
+ T __brotli_swap_tmp = (A)[(I)]; \
150
+ (A)[(I)] = (A)[(J)]; \
151
+ (A)[(J)] = __brotli_swap_tmp; \
152
+ }
153
+
154
+ #define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \
155
+ if (C < (R)) { \
156
+ size_t _new_size = (C == 0) ? (R) : C; \
157
+ T* new_array; \
158
+ while (_new_size < (R)) _new_size *= 2; \
159
+ new_array = BROTLI_ALLOC((M), T, _new_size); \
160
+ if (!BROTLI_IS_OOM(m) && C != 0) \
161
+ memcpy(new_array, A, C * sizeof(T)); \
162
+ BROTLI_FREE((M), A); \
163
+ A = new_array; \
164
+ C = _new_size; \
165
+ } \
166
+ }
167
+
168
+ #endif /* BROTLI_ENC_PORT_H_ */
@@ -4,76 +4,49 @@
4
4
  See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
5
  */
6
6
 
7
- // Functions for encoding of integers into prefix codes the amount of extra
8
- // bits, and the actual values of the extra bits.
7
+ /* Functions for encoding of integers into prefix codes the amount of extra
8
+ bits, and the actual values of the extra bits. */
9
9
 
10
10
  #ifndef BROTLI_ENC_PREFIX_H_
11
11
  #define BROTLI_ENC_PREFIX_H_
12
12
 
13
+ #include "../common/constants.h"
14
+ #include "../common/port.h"
15
+ #include "../common/types.h"
13
16
  #include "./fast_log.h"
14
- #include "./types.h"
15
17
 
16
- namespace brotli {
17
-
18
- static const uint32_t kNumInsertLenPrefixes = 24;
19
- static const uint32_t kNumCopyLenPrefixes = 24;
20
- static const uint32_t kNumCommandPrefixes = 704;
21
- static const uint32_t kNumBlockLenPrefixes = 26;
22
- static const uint32_t kNumDistanceShortCodes = 16;
23
- static const uint32_t kNumDistancePrefixes = 520;
24
-
25
- // Represents the range of values belonging to a prefix code:
26
- // [offset, offset + 2^nbits)
27
- struct PrefixCodeRange {
28
- uint32_t offset;
29
- uint32_t nbits;
30
- };
31
-
32
- static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
33
- { 1, 2}, { 5, 2}, { 9, 2}, { 13, 2},
34
- { 17, 3}, { 25, 3}, { 33, 3}, { 41, 3},
35
- { 49, 4}, { 65, 4}, { 81, 4}, { 97, 4},
36
- { 113, 5}, { 145, 5}, { 177, 5}, { 209, 5},
37
- { 241, 6}, { 305, 6}, { 369, 7}, { 497, 8},
38
- { 753, 9}, { 1265, 10}, {2289, 11}, {4337, 12},
39
- {8433, 13}, {16625, 24}
40
- };
41
-
42
- inline void GetBlockLengthPrefixCode(uint32_t len, uint32_t* code,
43
- uint32_t* n_extra, uint32_t* extra) {
44
- *code = 0;
45
- while (*code < 25 && len >= kBlockLengthPrefixCode[*code + 1].offset) {
46
- ++(*code);
47
- }
48
- *n_extra = kBlockLengthPrefixCode[*code].nbits;
49
- *extra = len - kBlockLengthPrefixCode[*code].offset;
50
- }
51
-
52
- inline void PrefixEncodeCopyDistance(size_t distance_code,
53
- size_t num_direct_codes,
54
- size_t postfix_bits,
55
- uint16_t* code,
56
- uint32_t* extra_bits) {
57
- if (distance_code < kNumDistanceShortCodes + num_direct_codes) {
58
- *code = static_cast<uint16_t>(distance_code);
18
+ #if defined(__cplusplus) || defined(c_plusplus)
19
+ extern "C" {
20
+ #endif
21
+
22
+ static BROTLI_INLINE void PrefixEncodeCopyDistance(size_t distance_code,
23
+ size_t num_direct_codes,
24
+ size_t postfix_bits,
25
+ uint16_t* code,
26
+ uint32_t* extra_bits) {
27
+ if (distance_code < BROTLI_NUM_DISTANCE_SHORT_CODES + num_direct_codes) {
28
+ *code = (uint16_t)distance_code;
59
29
  *extra_bits = 0;
60
30
  return;
31
+ } else {
32
+ size_t dist = ((size_t)1 << (postfix_bits + 2u)) +
33
+ (distance_code - BROTLI_NUM_DISTANCE_SHORT_CODES - num_direct_codes);
34
+ size_t bucket = Log2FloorNonZero(dist) - 1;
35
+ size_t postfix_mask = (1u << postfix_bits) - 1;
36
+ size_t postfix = dist & postfix_mask;
37
+ size_t prefix = (dist >> bucket) & 1;
38
+ size_t offset = (2 + prefix) << bucket;
39
+ size_t nbits = bucket - postfix_bits;
40
+ *code = (uint16_t)(
41
+ (BROTLI_NUM_DISTANCE_SHORT_CODES + num_direct_codes +
42
+ ((2 * (nbits - 1) + prefix) << postfix_bits) + postfix));
43
+ *extra_bits = (uint32_t)(
44
+ (nbits << 24) | ((dist - offset) >> postfix_bits));
61
45
  }
62
- distance_code -= kNumDistanceShortCodes + num_direct_codes; /* >= 0 */
63
- distance_code += (1u << (postfix_bits + 2u)); /* > 0 */
64
- size_t bucket = Log2FloorNonZero(distance_code) - 1;
65
- size_t postfix_mask = (1 << postfix_bits) - 1;
66
- size_t postfix = distance_code & postfix_mask;
67
- size_t prefix = (distance_code >> bucket) & 1;
68
- size_t offset = (2 + prefix) << bucket;
69
- size_t nbits = bucket - postfix_bits;
70
- *code = static_cast<uint16_t>(
71
- (kNumDistanceShortCodes + num_direct_codes +
72
- ((2 * (nbits - 1) + prefix) << postfix_bits) + postfix));
73
- *extra_bits = static_cast<uint32_t>(
74
- (nbits << 24) | ((distance_code - offset) >> postfix_bits));
75
46
  }
76
47
 
77
- } // namespace brotli
48
+ #if defined(__cplusplus) || defined(c_plusplus)
49
+ } /* extern "C" */
50
+ #endif
78
51
 
79
- #endif // BROTLI_ENC_PREFIX_H_
52
+ #endif /* BROTLI_ENC_PREFIX_H_ */
@@ -0,0 +1,130 @@
1
+ /* Copyright 2016 Google Inc. All Rights Reserved.
2
+
3
+ Distributed under MIT license.
4
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
+ */
6
+
7
+ /* Constants and formulas that affect speed-ratio trade-offs and thus define
8
+ quality levels. */
9
+
10
+ #ifndef BROTLI_ENC_QUALITY_H_
11
+ #define BROTLI_ENC_QUALITY_H_
12
+
13
+ #include "./encode.h"
14
+
15
+ #define FAST_ONE_PASS_COMPRESSION_QUALITY 0
16
+ #define FAST_TWO_PASS_COMPRESSION_QUALITY 1
17
+ #define ZOPFLIFICATION_QUALITY 10
18
+ #define HQ_ZOPFLIFICATION_QUALITY 11
19
+
20
+ #define MAX_QUALITY_FOR_STATIC_ENRTOPY_CODES 2
21
+ #define MIN_QUALITY_FOR_BLOCK_SPLIT 4
22
+ #define MIN_QUALITY_FOR_OPTIMIZE_HISTOGRAMS 4
23
+ #define MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH 5
24
+ #define MIN_QUALITY_FOR_CONTEXT_MODELING 5
25
+ #define MIN_QUALITY_FOR_HQ_CONTEXT_MODELING 7
26
+ #define MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING 10
27
+ /* Only for "font" mode. */
28
+ #define MIN_QUALITY_FOR_RECOMPUTE_DISTANCE_PREFIXES 10
29
+
30
+ /* For quality below MIN_QUALITY_FOR_BLOCK_SPLIT there is no block splitting,
31
+ so we buffer at most this much literals and commands. */
32
+ #define MAX_NUM_DELAYED_SYMBOLS 0x2fff
33
+
34
+ /* Encoding parameters */
35
+ typedef struct BrotliEncoderParams {
36
+ BrotliEncoderMode mode;
37
+ int quality;
38
+ int lgwin;
39
+ int lgblock;
40
+ } BrotliEncoderParams;
41
+
42
+ /* Returns hashtable size for quality levels 0 and 1. */
43
+ static BROTLI_INLINE size_t MaxHashTableSize(int quality) {
44
+ return quality == FAST_ONE_PASS_COMPRESSION_QUALITY ? 1 << 15 : 1 << 17;
45
+ }
46
+
47
+ /* The maximum length for which the zopflification uses distinct distances. */
48
+ #define MAX_ZOPFLI_LEN_QUALITY_10 150
49
+ #define MAX_ZOPFLI_LEN_QUALITY_11 325
50
+
51
+ static BROTLI_INLINE size_t MaxZopfliLen(const BrotliEncoderParams* params) {
52
+ return params->quality <= 10 ?
53
+ MAX_ZOPFLI_LEN_QUALITY_10 :
54
+ MAX_ZOPFLI_LEN_QUALITY_11;
55
+ }
56
+
57
+ /* Number of best candidates to evaluate to expand zopfli chain. */
58
+ static BROTLI_INLINE size_t MaxZopfliCandidates(
59
+ const BrotliEncoderParams* params) {
60
+ return params->quality <= 10 ? 1 : 5;
61
+ }
62
+
63
+ static BROTLI_INLINE void SanitizeParams(BrotliEncoderParams* params) {
64
+ params->quality = BROTLI_MIN(int, BROTLI_MAX_QUALITY,
65
+ BROTLI_MAX(int, BROTLI_MIN_QUALITY, params->quality));
66
+ if (params->lgwin < kBrotliMinWindowBits) {
67
+ params->lgwin = kBrotliMinWindowBits;
68
+ } else if (params->lgwin > kBrotliMaxWindowBits) {
69
+ params->lgwin = kBrotliMaxWindowBits;
70
+ }
71
+ }
72
+
73
+ /* Returns optimized lg_block value. */
74
+ static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
75
+ int lgblock = params->lgblock;
76
+ if (params->quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
77
+ params->quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
78
+ lgblock = params->lgwin;
79
+ } else if (params->quality < MIN_QUALITY_FOR_BLOCK_SPLIT) {
80
+ lgblock = 14;
81
+ } else if (lgblock == 0) {
82
+ lgblock = 16;
83
+ if (params->quality >= 9 && params->lgwin > lgblock) {
84
+ lgblock = BROTLI_MIN(int, 18, params->lgwin);
85
+ }
86
+ } else {
87
+ lgblock = BROTLI_MIN(int, kBrotliMaxInputBlockBits,
88
+ BROTLI_MAX(int, kBrotliMinInputBlockBits, lgblock));
89
+ }
90
+ return lgblock;
91
+ }
92
+
93
+ /* Returns log2 of the size of main ring buffer area.
94
+ Allocate at least lgwin + 1 bits for the ring buffer so that the newly
95
+ added block fits there completely and we still get lgwin bits and at least
96
+ read_block_size_bits + 1 bits because the copy tail length needs to be
97
+ smaller than ringbuffer size. */
98
+ static BROTLI_INLINE int ComputeRbBits(const BrotliEncoderParams* params) {
99
+ return 1 + BROTLI_MAX(int, params->lgwin, params->lgblock);
100
+ }
101
+
102
+ static BROTLI_INLINE size_t MaxMetablockSize(
103
+ const BrotliEncoderParams* params) {
104
+ int bits = BROTLI_MIN(int, ComputeRbBits(params), kBrotliMaxInputBlockBits);
105
+ return (size_t)1 << bits;
106
+ }
107
+
108
+ /* When searching for backward references and have not seen matches for a long
109
+ time, we can skip some match lookups. Unsuccessful match lookups are very
110
+ expensive and this kind of a heuristic speeds up compression quite a lot.
111
+ At first 8 byte strides are taken and every second byte is put to hasher.
112
+ After 4x more literals stride by 16 bytes, every put 4-th byte to hasher.
113
+ Applied only to qualities 2 to 9. */
114
+ static BROTLI_INLINE size_t LiteralSpreeLengthForSparseSearch(
115
+ const BrotliEncoderParams* params) {
116
+ return params->quality < 9 ? 64 : 512;
117
+ }
118
+
119
+ static BROTLI_INLINE int ChooseHasher(const BrotliEncoderParams* params) {
120
+ if (params->quality > 9) {
121
+ return 10;
122
+ } else if (params->quality < 5) {
123
+ return params->quality;
124
+ } else if (params->lgwin <= 16) {
125
+ return params->quality < 7 ? 40 : params->quality < 9 ? 41 : 42;
126
+ }
127
+ return params->quality;
128
+ }
129
+
130
+ #endif /* BROTLI_ENC_QUALITY_H_ */