brotli 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +3 -0
- data/.travis.yml +11 -3
- data/Gemfile +2 -0
- data/ext/brotli/brotli.c +279 -0
- data/ext/brotli/brotli.h +2 -0
- data/ext/brotli/buffer.c +95 -0
- data/ext/brotli/buffer.h +19 -0
- data/ext/brotli/extconf.rb +21 -81
- data/lib/brotli/version.rb +1 -1
- data/vendor/brotli/dec/bit_reader.c +5 -5
- data/vendor/brotli/dec/bit_reader.h +15 -15
- data/vendor/brotli/dec/context.h +1 -1
- data/vendor/brotli/dec/decode.c +433 -348
- data/vendor/brotli/dec/decode.h +74 -48
- data/vendor/brotli/dec/huffman.c +5 -4
- data/vendor/brotli/dec/huffman.h +4 -4
- data/vendor/brotli/dec/port.h +2 -95
- data/vendor/brotli/dec/prefix.h +5 -3
- data/vendor/brotli/dec/state.c +15 -27
- data/vendor/brotli/dec/state.h +21 -17
- data/vendor/brotli/dec/transform.h +1 -1
- data/vendor/brotli/enc/backward_references.c +892 -0
- data/vendor/brotli/enc/backward_references.h +85 -102
- data/vendor/brotli/enc/backward_references_inc.h +147 -0
- data/vendor/brotli/enc/bit_cost.c +35 -0
- data/vendor/brotli/enc/bit_cost.h +23 -121
- data/vendor/brotli/enc/bit_cost_inc.h +127 -0
- data/vendor/brotli/enc/block_encoder_inc.h +33 -0
- data/vendor/brotli/enc/block_splitter.c +197 -0
- data/vendor/brotli/enc/block_splitter.h +40 -50
- data/vendor/brotli/enc/block_splitter_inc.h +432 -0
- data/vendor/brotli/enc/brotli_bit_stream.c +1334 -0
- data/vendor/brotli/enc/brotli_bit_stream.h +95 -167
- data/vendor/brotli/enc/cluster.c +56 -0
- data/vendor/brotli/enc/cluster.h +23 -305
- data/vendor/brotli/enc/cluster_inc.h +315 -0
- data/vendor/brotli/enc/command.h +83 -76
- data/vendor/brotli/enc/compress_fragment.c +747 -0
- data/vendor/brotli/enc/compress_fragment.h +48 -37
- data/vendor/brotli/enc/compress_fragment_two_pass.c +557 -0
- data/vendor/brotli/enc/compress_fragment_two_pass.h +37 -26
- data/vendor/brotli/enc/compressor.cc +139 -0
- data/vendor/brotli/enc/compressor.h +146 -0
- data/vendor/brotli/enc/context.h +102 -96
- data/vendor/brotli/enc/dictionary_hash.h +9 -5
- data/vendor/brotli/enc/encode.c +1562 -0
- data/vendor/brotli/enc/encode.h +211 -199
- data/vendor/brotli/enc/encode_parallel.cc +161 -151
- data/vendor/brotli/enc/encode_parallel.h +7 -8
- data/vendor/brotli/enc/entropy_encode.c +501 -0
- data/vendor/brotli/enc/entropy_encode.h +107 -89
- data/vendor/brotli/enc/entropy_encode_static.h +29 -62
- data/vendor/brotli/enc/fast_log.h +26 -20
- data/vendor/brotli/enc/find_match_length.h +23 -20
- data/vendor/brotli/enc/hash.h +614 -871
- data/vendor/brotli/enc/hash_forgetful_chain_inc.h +249 -0
- data/vendor/brotli/enc/hash_longest_match_inc.h +241 -0
- data/vendor/brotli/enc/hash_longest_match_quickly_inc.h +230 -0
- data/vendor/brotli/enc/histogram.c +95 -0
- data/vendor/brotli/enc/histogram.h +49 -83
- data/vendor/brotli/enc/histogram_inc.h +51 -0
- data/vendor/brotli/enc/literal_cost.c +178 -0
- data/vendor/brotli/enc/literal_cost.h +16 -10
- data/vendor/brotli/enc/memory.c +181 -0
- data/vendor/brotli/enc/memory.h +62 -0
- data/vendor/brotli/enc/metablock.c +515 -0
- data/vendor/brotli/enc/metablock.h +87 -57
- data/vendor/brotli/enc/metablock_inc.h +183 -0
- data/vendor/brotli/enc/port.h +73 -47
- data/vendor/brotli/enc/prefix.h +34 -61
- data/vendor/brotli/enc/quality.h +130 -0
- data/vendor/brotli/enc/ringbuffer.h +137 -122
- data/vendor/brotli/enc/{static_dict.cc → static_dict.c} +162 -139
- data/vendor/brotli/enc/static_dict.h +23 -18
- data/vendor/brotli/enc/static_dict_lut.h +11223 -12037
- data/vendor/brotli/enc/streams.cc +7 -7
- data/vendor/brotli/enc/streams.h +32 -32
- data/vendor/brotli/enc/{utf8_util.cc → utf8_util.c} +22 -20
- data/vendor/brotli/enc/utf8_util.h +16 -9
- data/vendor/brotli/enc/write_bits.h +49 -43
- metadata +34 -25
- data/ext/brotli/brotli.cc +0 -181
- data/vendor/brotli/dec/Makefile +0 -12
- data/vendor/brotli/dec/dictionary.c +0 -9466
- data/vendor/brotli/dec/dictionary.h +0 -38
- data/vendor/brotli/dec/types.h +0 -38
- data/vendor/brotli/enc/Makefile +0 -14
- data/vendor/brotli/enc/backward_references.cc +0 -858
- data/vendor/brotli/enc/block_splitter.cc +0 -505
- data/vendor/brotli/enc/brotli_bit_stream.cc +0 -1181
- data/vendor/brotli/enc/compress_fragment.cc +0 -701
- data/vendor/brotli/enc/compress_fragment_two_pass.cc +0 -524
- data/vendor/brotli/enc/dictionary.cc +0 -9466
- data/vendor/brotli/enc/dictionary.h +0 -41
- data/vendor/brotli/enc/encode.cc +0 -1180
- data/vendor/brotli/enc/entropy_encode.cc +0 -480
- data/vendor/brotli/enc/histogram.cc +0 -67
- data/vendor/brotli/enc/literal_cost.cc +0 -165
- data/vendor/brotli/enc/metablock.cc +0 -539
- data/vendor/brotli/enc/transform.h +0 -248
- data/vendor/brotli/enc/types.h +0 -29
@@ -4,142 +4,157 @@
|
|
4
4
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
5
|
*/
|
6
6
|
|
7
|
-
|
7
|
+
/* Sliding window over the input data. */
|
8
8
|
|
9
9
|
#ifndef BROTLI_ENC_RINGBUFFER_H_
|
10
10
|
#define BROTLI_ENC_RINGBUFFER_H_
|
11
11
|
|
12
|
-
#include <
|
12
|
+
#include <string.h> /* memcpy */
|
13
13
|
|
14
|
+
#include "../common/types.h"
|
15
|
+
#include "./memory.h"
|
14
16
|
#include "./port.h"
|
15
|
-
#include "./
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
mask_((1u << window_bits) - 1),
|
33
|
-
tail_size_(1u << tail_bits),
|
34
|
-
total_size_(size_ + tail_size_),
|
35
|
-
cur_size_(0),
|
36
|
-
pos_(0),
|
37
|
-
data_(0),
|
38
|
-
buffer_(0) {}
|
39
|
-
|
40
|
-
~RingBuffer(void) {
|
41
|
-
free(data_);
|
42
|
-
}
|
43
|
-
|
44
|
-
// Allocates or re-allocates data_ to the given length + plus some slack
|
45
|
-
// region before and after. Fills the slack regions with zeros.
|
46
|
-
inline void InitBuffer(const uint32_t buflen) {
|
47
|
-
static const size_t kSlackForEightByteHashingEverywhere = 7;
|
48
|
-
cur_size_ = buflen;
|
49
|
-
data_ = static_cast<uint8_t*>(realloc(
|
50
|
-
data_, 2 + buflen + kSlackForEightByteHashingEverywhere));
|
51
|
-
buffer_ = data_ + 2;
|
52
|
-
buffer_[-2] = buffer_[-1] = 0;
|
53
|
-
for (size_t i = 0; i < kSlackForEightByteHashingEverywhere; ++i) {
|
54
|
-
buffer_[cur_size_ + i] = 0;
|
55
|
-
}
|
56
|
-
}
|
57
|
-
|
58
|
-
// Push bytes into the ring buffer.
|
59
|
-
void Write(const uint8_t *bytes, size_t n) {
|
60
|
-
if (pos_ == 0 && n < tail_size_) {
|
61
|
-
// Special case for the first write: to process the first block, we don't
|
62
|
-
// need to allocate the whole ringbuffer and we don't need the tail
|
63
|
-
// either. However, we do this memory usage optimization only if the
|
64
|
-
// first write is less than the tail size, which is also the input block
|
65
|
-
// size, otherwise it is likely that other blocks will follow and we
|
66
|
-
// will need to reallocate to the full size anyway.
|
67
|
-
pos_ = static_cast<uint32_t>(n);
|
68
|
-
InitBuffer(pos_);
|
69
|
-
memcpy(buffer_, bytes, n);
|
70
|
-
return;
|
71
|
-
}
|
72
|
-
if (cur_size_ < total_size_) {
|
73
|
-
// Lazily allocate the full buffer.
|
74
|
-
InitBuffer(total_size_);
|
75
|
-
// Initialize the last two bytes to zero, so that we don't have to worry
|
76
|
-
// later when we copy the last two bytes to the first two positions.
|
77
|
-
buffer_[size_ - 2] = 0;
|
78
|
-
buffer_[size_ - 1] = 0;
|
79
|
-
}
|
80
|
-
const size_t masked_pos = pos_ & mask_;
|
81
|
-
// The length of the writes is limited so that we do not need to worry
|
82
|
-
// about a write
|
83
|
-
WriteTail(bytes, n);
|
84
|
-
if (PREDICT_TRUE(masked_pos + n <= size_)) {
|
85
|
-
// A single write fits.
|
86
|
-
memcpy(&buffer_[masked_pos], bytes, n);
|
87
|
-
} else {
|
88
|
-
// Split into two writes.
|
89
|
-
// Copy into the end of the buffer, including the tail buffer.
|
90
|
-
memcpy(&buffer_[masked_pos], bytes,
|
91
|
-
std::min(n, total_size_ - masked_pos));
|
92
|
-
// Copy into the beginning of the buffer
|
93
|
-
memcpy(&buffer_[0], bytes + (size_ - masked_pos),
|
94
|
-
n - (size_ - masked_pos));
|
95
|
-
}
|
96
|
-
buffer_[-2] = buffer_[size_ - 2];
|
97
|
-
buffer_[-1] = buffer_[size_ - 1];
|
98
|
-
pos_ += static_cast<uint32_t>(n);
|
99
|
-
if (pos_ > (1u << 30)) { /* Wrap, but preserve not-a-first-lap feature. */
|
100
|
-
pos_ = (pos_ & ((1u << 30) - 1)) | (1u << 30);
|
101
|
-
}
|
102
|
-
}
|
103
|
-
|
104
|
-
void Reset(void) {
|
105
|
-
pos_ = 0;
|
106
|
-
}
|
107
|
-
|
108
|
-
// Logical cursor position in the ring buffer.
|
109
|
-
uint32_t position(void) const { return pos_; }
|
110
|
-
|
111
|
-
// Bit mask for getting the physical position for a logical position.
|
112
|
-
uint32_t mask(void) const { return mask_; }
|
113
|
-
|
114
|
-
uint8_t *start(void) { return &buffer_[0]; }
|
115
|
-
const uint8_t *start(void) const { return &buffer_[0]; }
|
116
|
-
|
117
|
-
private:
|
118
|
-
void WriteTail(const uint8_t *bytes, size_t n) {
|
119
|
-
const size_t masked_pos = pos_ & mask_;
|
120
|
-
if (PREDICT_FALSE(masked_pos < tail_size_)) {
|
121
|
-
// Just fill the tail buffer with the beginning data.
|
122
|
-
const size_t p = size_ + masked_pos;
|
123
|
-
memcpy(&buffer_[p], bytes, std::min(n, tail_size_ - masked_pos));
|
124
|
-
}
|
125
|
-
}
|
126
|
-
|
127
|
-
// Size of the ringbuffer is (1 << window_bits) + tail_size_.
|
17
|
+
#include "./quality.h"
|
18
|
+
|
19
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
20
|
+
extern "C" {
|
21
|
+
#endif
|
22
|
+
|
23
|
+
/* A RingBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of
|
24
|
+
data in a circular manner: writing a byte writes it to:
|
25
|
+
`position() % (1 << window_bits)'.
|
26
|
+
For convenience, the RingBuffer array contains another copy of the
|
27
|
+
first `1 << tail_bits' bytes:
|
28
|
+
buffer_[i] == buffer_[i + (1 << window_bits)], if i < (1 << tail_bits),
|
29
|
+
and another copy of the last two bytes:
|
30
|
+
buffer_[-1] == buffer_[(1 << window_bits) - 1] and
|
31
|
+
buffer_[-2] == buffer_[(1 << window_bits) - 2]. */
|
32
|
+
typedef struct RingBuffer {
|
33
|
+
/* Size of the ringbuffer is (1 << window_bits) + tail_size_. */
|
128
34
|
const uint32_t size_;
|
129
35
|
const uint32_t mask_;
|
130
36
|
const uint32_t tail_size_;
|
131
37
|
const uint32_t total_size_;
|
132
38
|
|
133
39
|
uint32_t cur_size_;
|
134
|
-
|
40
|
+
/* Position to write in the ring buffer. */
|
135
41
|
uint32_t pos_;
|
136
|
-
|
137
|
-
|
42
|
+
/* The actual ring buffer containing the copy of the last two bytes, the data,
|
43
|
+
and the copy of the beginning as a tail. */
|
138
44
|
uint8_t *data_;
|
139
|
-
|
45
|
+
/* The start of the ringbuffer. */
|
140
46
|
uint8_t *buffer_;
|
141
|
-
};
|
47
|
+
} RingBuffer;
|
48
|
+
|
49
|
+
static BROTLI_INLINE void RingBufferInit(RingBuffer* rb) {
|
50
|
+
rb->cur_size_ = 0;
|
51
|
+
rb->pos_ = 0;
|
52
|
+
rb->data_ = 0;
|
53
|
+
rb->buffer_ = 0;
|
54
|
+
}
|
55
|
+
|
56
|
+
static BROTLI_INLINE void RingBufferSetup(
|
57
|
+
const BrotliEncoderParams* params, RingBuffer* rb) {
|
58
|
+
int window_bits = ComputeRbBits(params);
|
59
|
+
int tail_bits = params->lgblock;
|
60
|
+
*(uint32_t*)&rb->size_ = 1u << window_bits;
|
61
|
+
*(uint32_t*)&rb->mask_ = (1u << window_bits) - 1;
|
62
|
+
*(uint32_t*)&rb->tail_size_ = 1u << tail_bits;
|
63
|
+
*(uint32_t*)&rb->total_size_ = rb->size_ + rb->tail_size_;
|
64
|
+
}
|
65
|
+
|
66
|
+
static BROTLI_INLINE void RingBufferFree(MemoryManager* m, RingBuffer* rb) {
|
67
|
+
BROTLI_FREE(m, rb->data_);
|
68
|
+
}
|
69
|
+
|
70
|
+
/* Allocates or re-allocates data_ to the given length + plus some slack
|
71
|
+
region before and after. Fills the slack regions with zeros. */
|
72
|
+
static BROTLI_INLINE void RingBufferInitBuffer(
|
73
|
+
MemoryManager* m, const uint32_t buflen, RingBuffer* rb) {
|
74
|
+
static const size_t kSlackForEightByteHashingEverywhere = 7;
|
75
|
+
uint8_t* new_data = BROTLI_ALLOC(
|
76
|
+
m, uint8_t, 2 + buflen + kSlackForEightByteHashingEverywhere);
|
77
|
+
size_t i;
|
78
|
+
if (BROTLI_IS_OOM(m)) return;
|
79
|
+
if (rb->data_) {
|
80
|
+
memcpy(new_data, rb->data_,
|
81
|
+
2 + rb->cur_size_ + kSlackForEightByteHashingEverywhere);
|
82
|
+
BROTLI_FREE(m, rb->data_);
|
83
|
+
}
|
84
|
+
rb->data_ = new_data;
|
85
|
+
rb->cur_size_ = buflen;
|
86
|
+
rb->buffer_ = rb->data_ + 2;
|
87
|
+
rb->buffer_[-2] = rb->buffer_[-1] = 0;
|
88
|
+
for (i = 0; i < kSlackForEightByteHashingEverywhere; ++i) {
|
89
|
+
rb->buffer_[rb->cur_size_ + i] = 0;
|
90
|
+
}
|
91
|
+
}
|
92
|
+
|
93
|
+
static BROTLI_INLINE void RingBufferWriteTail(
|
94
|
+
const uint8_t *bytes, size_t n, RingBuffer* rb) {
|
95
|
+
const size_t masked_pos = rb->pos_ & rb->mask_;
|
96
|
+
if (PREDICT_FALSE(masked_pos < rb->tail_size_)) {
|
97
|
+
/* Just fill the tail buffer with the beginning data. */
|
98
|
+
const size_t p = rb->size_ + masked_pos;
|
99
|
+
memcpy(&rb->buffer_[p], bytes,
|
100
|
+
BROTLI_MIN(size_t, n, rb->tail_size_ - masked_pos));
|
101
|
+
}
|
102
|
+
}
|
103
|
+
|
104
|
+
/* Push bytes into the ring buffer. */
|
105
|
+
static BROTLI_INLINE void RingBufferWrite(
|
106
|
+
MemoryManager* m, const uint8_t *bytes, size_t n, RingBuffer* rb) {
|
107
|
+
if (rb->pos_ == 0 && n < rb->tail_size_) {
|
108
|
+
/* Special case for the first write: to process the first block, we don't
|
109
|
+
need to allocate the whole ringbuffer and we don't need the tail
|
110
|
+
either. However, we do this memory usage optimization only if the
|
111
|
+
first write is less than the tail size, which is also the input block
|
112
|
+
size, otherwise it is likely that other blocks will follow and we
|
113
|
+
will need to reallocate to the full size anyway. */
|
114
|
+
rb->pos_ = (uint32_t)n;
|
115
|
+
RingBufferInitBuffer(m, rb->pos_, rb);
|
116
|
+
if (BROTLI_IS_OOM(m)) return;
|
117
|
+
memcpy(rb->buffer_, bytes, n);
|
118
|
+
return;
|
119
|
+
}
|
120
|
+
if (rb->cur_size_ < rb->total_size_) {
|
121
|
+
/* Lazily allocate the full buffer. */
|
122
|
+
RingBufferInitBuffer(m, rb->total_size_, rb);
|
123
|
+
if (BROTLI_IS_OOM(m)) return;
|
124
|
+
/* Initialize the last two bytes to zero, so that we don't have to worry
|
125
|
+
later when we copy the last two bytes to the first two positions. */
|
126
|
+
rb->buffer_[rb->size_ - 2] = 0;
|
127
|
+
rb->buffer_[rb->size_ - 1] = 0;
|
128
|
+
}
|
129
|
+
{
|
130
|
+
const size_t masked_pos = rb->pos_ & rb->mask_;
|
131
|
+
/* The length of the writes is limited so that we do not need to worry
|
132
|
+
about a write */
|
133
|
+
RingBufferWriteTail(bytes, n, rb);
|
134
|
+
if (PREDICT_TRUE(masked_pos + n <= rb->size_)) {
|
135
|
+
/* A single write fits. */
|
136
|
+
memcpy(&rb->buffer_[masked_pos], bytes, n);
|
137
|
+
} else {
|
138
|
+
/* Split into two writes.
|
139
|
+
Copy into the end of the buffer, including the tail buffer. */
|
140
|
+
memcpy(&rb->buffer_[masked_pos], bytes,
|
141
|
+
BROTLI_MIN(size_t, n, rb->total_size_ - masked_pos));
|
142
|
+
/* Copy into the beginning of the buffer */
|
143
|
+
memcpy(&rb->buffer_[0], bytes + (rb->size_ - masked_pos),
|
144
|
+
n - (rb->size_ - masked_pos));
|
145
|
+
}
|
146
|
+
}
|
147
|
+
rb->buffer_[-2] = rb->buffer_[rb->size_ - 2];
|
148
|
+
rb->buffer_[-1] = rb->buffer_[rb->size_ - 1];
|
149
|
+
rb->pos_ += (uint32_t)n;
|
150
|
+
if (rb->pos_ > (1u << 30)) {
|
151
|
+
/* Wrap, but preserve not-a-first-lap feature. */
|
152
|
+
rb->pos_ = (rb->pos_ & ((1u << 30) - 1)) | (1u << 30);
|
153
|
+
}
|
154
|
+
}
|
142
155
|
|
143
|
-
|
156
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
157
|
+
} /* extern "C" */
|
158
|
+
#endif
|
144
159
|
|
145
|
-
#endif
|
160
|
+
#endif /* BROTLI_ENC_RINGBUFFER_H_ */
|