brotli 0.1.3 → 0.1.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +3 -0
  3. data/.travis.yml +11 -3
  4. data/Gemfile +2 -0
  5. data/ext/brotli/brotli.c +279 -0
  6. data/ext/brotli/brotli.h +2 -0
  7. data/ext/brotli/buffer.c +95 -0
  8. data/ext/brotli/buffer.h +19 -0
  9. data/ext/brotli/extconf.rb +21 -81
  10. data/lib/brotli/version.rb +1 -1
  11. data/vendor/brotli/dec/bit_reader.c +5 -5
  12. data/vendor/brotli/dec/bit_reader.h +15 -15
  13. data/vendor/brotli/dec/context.h +1 -1
  14. data/vendor/brotli/dec/decode.c +433 -348
  15. data/vendor/brotli/dec/decode.h +74 -48
  16. data/vendor/brotli/dec/huffman.c +5 -4
  17. data/vendor/brotli/dec/huffman.h +4 -4
  18. data/vendor/brotli/dec/port.h +2 -95
  19. data/vendor/brotli/dec/prefix.h +5 -3
  20. data/vendor/brotli/dec/state.c +15 -27
  21. data/vendor/brotli/dec/state.h +21 -17
  22. data/vendor/brotli/dec/transform.h +1 -1
  23. data/vendor/brotli/enc/backward_references.c +892 -0
  24. data/vendor/brotli/enc/backward_references.h +85 -102
  25. data/vendor/brotli/enc/backward_references_inc.h +147 -0
  26. data/vendor/brotli/enc/bit_cost.c +35 -0
  27. data/vendor/brotli/enc/bit_cost.h +23 -121
  28. data/vendor/brotli/enc/bit_cost_inc.h +127 -0
  29. data/vendor/brotli/enc/block_encoder_inc.h +33 -0
  30. data/vendor/brotli/enc/block_splitter.c +197 -0
  31. data/vendor/brotli/enc/block_splitter.h +40 -50
  32. data/vendor/brotli/enc/block_splitter_inc.h +432 -0
  33. data/vendor/brotli/enc/brotli_bit_stream.c +1334 -0
  34. data/vendor/brotli/enc/brotli_bit_stream.h +95 -167
  35. data/vendor/brotli/enc/cluster.c +56 -0
  36. data/vendor/brotli/enc/cluster.h +23 -305
  37. data/vendor/brotli/enc/cluster_inc.h +315 -0
  38. data/vendor/brotli/enc/command.h +83 -76
  39. data/vendor/brotli/enc/compress_fragment.c +747 -0
  40. data/vendor/brotli/enc/compress_fragment.h +48 -37
  41. data/vendor/brotli/enc/compress_fragment_two_pass.c +557 -0
  42. data/vendor/brotli/enc/compress_fragment_two_pass.h +37 -26
  43. data/vendor/brotli/enc/compressor.cc +139 -0
  44. data/vendor/brotli/enc/compressor.h +146 -0
  45. data/vendor/brotli/enc/context.h +102 -96
  46. data/vendor/brotli/enc/dictionary_hash.h +9 -5
  47. data/vendor/brotli/enc/encode.c +1562 -0
  48. data/vendor/brotli/enc/encode.h +211 -199
  49. data/vendor/brotli/enc/encode_parallel.cc +161 -151
  50. data/vendor/brotli/enc/encode_parallel.h +7 -8
  51. data/vendor/brotli/enc/entropy_encode.c +501 -0
  52. data/vendor/brotli/enc/entropy_encode.h +107 -89
  53. data/vendor/brotli/enc/entropy_encode_static.h +29 -62
  54. data/vendor/brotli/enc/fast_log.h +26 -20
  55. data/vendor/brotli/enc/find_match_length.h +23 -20
  56. data/vendor/brotli/enc/hash.h +614 -871
  57. data/vendor/brotli/enc/hash_forgetful_chain_inc.h +249 -0
  58. data/vendor/brotli/enc/hash_longest_match_inc.h +241 -0
  59. data/vendor/brotli/enc/hash_longest_match_quickly_inc.h +230 -0
  60. data/vendor/brotli/enc/histogram.c +95 -0
  61. data/vendor/brotli/enc/histogram.h +49 -83
  62. data/vendor/brotli/enc/histogram_inc.h +51 -0
  63. data/vendor/brotli/enc/literal_cost.c +178 -0
  64. data/vendor/brotli/enc/literal_cost.h +16 -10
  65. data/vendor/brotli/enc/memory.c +181 -0
  66. data/vendor/brotli/enc/memory.h +62 -0
  67. data/vendor/brotli/enc/metablock.c +515 -0
  68. data/vendor/brotli/enc/metablock.h +87 -57
  69. data/vendor/brotli/enc/metablock_inc.h +183 -0
  70. data/vendor/brotli/enc/port.h +73 -47
  71. data/vendor/brotli/enc/prefix.h +34 -61
  72. data/vendor/brotli/enc/quality.h +130 -0
  73. data/vendor/brotli/enc/ringbuffer.h +137 -122
  74. data/vendor/brotli/enc/{static_dict.cc → static_dict.c} +162 -139
  75. data/vendor/brotli/enc/static_dict.h +23 -18
  76. data/vendor/brotli/enc/static_dict_lut.h +11223 -12037
  77. data/vendor/brotli/enc/streams.cc +7 -7
  78. data/vendor/brotli/enc/streams.h +32 -32
  79. data/vendor/brotli/enc/{utf8_util.cc → utf8_util.c} +22 -20
  80. data/vendor/brotli/enc/utf8_util.h +16 -9
  81. data/vendor/brotli/enc/write_bits.h +49 -43
  82. metadata +34 -25
  83. data/ext/brotli/brotli.cc +0 -181
  84. data/vendor/brotli/dec/Makefile +0 -12
  85. data/vendor/brotli/dec/dictionary.c +0 -9466
  86. data/vendor/brotli/dec/dictionary.h +0 -38
  87. data/vendor/brotli/dec/types.h +0 -38
  88. data/vendor/brotli/enc/Makefile +0 -14
  89. data/vendor/brotli/enc/backward_references.cc +0 -858
  90. data/vendor/brotli/enc/block_splitter.cc +0 -505
  91. data/vendor/brotli/enc/brotli_bit_stream.cc +0 -1181
  92. data/vendor/brotli/enc/compress_fragment.cc +0 -701
  93. data/vendor/brotli/enc/compress_fragment_two_pass.cc +0 -524
  94. data/vendor/brotli/enc/dictionary.cc +0 -9466
  95. data/vendor/brotli/enc/dictionary.h +0 -41
  96. data/vendor/brotli/enc/encode.cc +0 -1180
  97. data/vendor/brotli/enc/entropy_encode.cc +0 -480
  98. data/vendor/brotli/enc/histogram.cc +0 -67
  99. data/vendor/brotli/enc/literal_cost.cc +0 -165
  100. data/vendor/brotli/enc/metablock.cc +0 -539
  101. data/vendor/brotli/enc/transform.h +0 -248
  102. data/vendor/brotli/enc/types.h +0 -29
@@ -4,142 +4,157 @@
4
4
  See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
5
  */
6
6
 
7
- // Sliding window over the input data.
7
+ /* Sliding window over the input data. */
8
8
 
9
9
  #ifndef BROTLI_ENC_RINGBUFFER_H_
10
10
  #define BROTLI_ENC_RINGBUFFER_H_
11
11
 
12
- #include <cstdlib> /* free, realloc */
12
+ #include <string.h> /* memcpy */
13
13
 
14
+ #include "../common/types.h"
15
+ #include "./memory.h"
14
16
  #include "./port.h"
15
- #include "./types.h"
16
-
17
- namespace brotli {
18
-
19
- // A RingBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of
20
- // data in a circular manner: writing a byte writes it to:
21
- // `position() % (1 << window_bits)'.
22
- // For convenience, the RingBuffer array contains another copy of the
23
- // first `1 << tail_bits' bytes:
24
- // buffer_[i] == buffer_[i + (1 << window_bits)], if i < (1 << tail_bits),
25
- // and another copy of the last two bytes:
26
- // buffer_[-1] == buffer_[(1 << window_bits) - 1] and
27
- // buffer_[-2] == buffer_[(1 << window_bits) - 2].
28
- class RingBuffer {
29
- public:
30
- RingBuffer(int window_bits, int tail_bits)
31
- : size_(1u << window_bits),
32
- mask_((1u << window_bits) - 1),
33
- tail_size_(1u << tail_bits),
34
- total_size_(size_ + tail_size_),
35
- cur_size_(0),
36
- pos_(0),
37
- data_(0),
38
- buffer_(0) {}
39
-
40
- ~RingBuffer(void) {
41
- free(data_);
42
- }
43
-
44
- // Allocates or re-allocates data_ to the given length + plus some slack
45
- // region before and after. Fills the slack regions with zeros.
46
- inline void InitBuffer(const uint32_t buflen) {
47
- static const size_t kSlackForEightByteHashingEverywhere = 7;
48
- cur_size_ = buflen;
49
- data_ = static_cast<uint8_t*>(realloc(
50
- data_, 2 + buflen + kSlackForEightByteHashingEverywhere));
51
- buffer_ = data_ + 2;
52
- buffer_[-2] = buffer_[-1] = 0;
53
- for (size_t i = 0; i < kSlackForEightByteHashingEverywhere; ++i) {
54
- buffer_[cur_size_ + i] = 0;
55
- }
56
- }
57
-
58
- // Push bytes into the ring buffer.
59
- void Write(const uint8_t *bytes, size_t n) {
60
- if (pos_ == 0 && n < tail_size_) {
61
- // Special case for the first write: to process the first block, we don't
62
- // need to allocate the whole ringbuffer and we don't need the tail
63
- // either. However, we do this memory usage optimization only if the
64
- // first write is less than the tail size, which is also the input block
65
- // size, otherwise it is likely that other blocks will follow and we
66
- // will need to reallocate to the full size anyway.
67
- pos_ = static_cast<uint32_t>(n);
68
- InitBuffer(pos_);
69
- memcpy(buffer_, bytes, n);
70
- return;
71
- }
72
- if (cur_size_ < total_size_) {
73
- // Lazily allocate the full buffer.
74
- InitBuffer(total_size_);
75
- // Initialize the last two bytes to zero, so that we don't have to worry
76
- // later when we copy the last two bytes to the first two positions.
77
- buffer_[size_ - 2] = 0;
78
- buffer_[size_ - 1] = 0;
79
- }
80
- const size_t masked_pos = pos_ & mask_;
81
- // The length of the writes is limited so that we do not need to worry
82
- // about a write
83
- WriteTail(bytes, n);
84
- if (PREDICT_TRUE(masked_pos + n <= size_)) {
85
- // A single write fits.
86
- memcpy(&buffer_[masked_pos], bytes, n);
87
- } else {
88
- // Split into two writes.
89
- // Copy into the end of the buffer, including the tail buffer.
90
- memcpy(&buffer_[masked_pos], bytes,
91
- std::min(n, total_size_ - masked_pos));
92
- // Copy into the beginning of the buffer
93
- memcpy(&buffer_[0], bytes + (size_ - masked_pos),
94
- n - (size_ - masked_pos));
95
- }
96
- buffer_[-2] = buffer_[size_ - 2];
97
- buffer_[-1] = buffer_[size_ - 1];
98
- pos_ += static_cast<uint32_t>(n);
99
- if (pos_ > (1u << 30)) { /* Wrap, but preserve not-a-first-lap feature. */
100
- pos_ = (pos_ & ((1u << 30) - 1)) | (1u << 30);
101
- }
102
- }
103
-
104
- void Reset(void) {
105
- pos_ = 0;
106
- }
107
-
108
- // Logical cursor position in the ring buffer.
109
- uint32_t position(void) const { return pos_; }
110
-
111
- // Bit mask for getting the physical position for a logical position.
112
- uint32_t mask(void) const { return mask_; }
113
-
114
- uint8_t *start(void) { return &buffer_[0]; }
115
- const uint8_t *start(void) const { return &buffer_[0]; }
116
-
117
- private:
118
- void WriteTail(const uint8_t *bytes, size_t n) {
119
- const size_t masked_pos = pos_ & mask_;
120
- if (PREDICT_FALSE(masked_pos < tail_size_)) {
121
- // Just fill the tail buffer with the beginning data.
122
- const size_t p = size_ + masked_pos;
123
- memcpy(&buffer_[p], bytes, std::min(n, tail_size_ - masked_pos));
124
- }
125
- }
126
-
127
- // Size of the ringbuffer is (1 << window_bits) + tail_size_.
17
+ #include "./quality.h"
18
+
19
+ #if defined(__cplusplus) || defined(c_plusplus)
20
+ extern "C" {
21
+ #endif
22
+
23
+ /* A RingBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of
24
+ data in a circular manner: writing a byte writes it to:
25
+ `position() % (1 << window_bits)'.
26
+ For convenience, the RingBuffer array contains another copy of the
27
+ first `1 << tail_bits' bytes:
28
+ buffer_[i] == buffer_[i + (1 << window_bits)], if i < (1 << tail_bits),
29
+ and another copy of the last two bytes:
30
+ buffer_[-1] == buffer_[(1 << window_bits) - 1] and
31
+ buffer_[-2] == buffer_[(1 << window_bits) - 2]. */
32
+ typedef struct RingBuffer {
33
+ /* Size of the ringbuffer is (1 << window_bits) + tail_size_. */
128
34
  const uint32_t size_;
129
35
  const uint32_t mask_;
130
36
  const uint32_t tail_size_;
131
37
  const uint32_t total_size_;
132
38
 
133
39
  uint32_t cur_size_;
134
- // Position to write in the ring buffer.
40
+ /* Position to write in the ring buffer. */
135
41
  uint32_t pos_;
136
- // The actual ring buffer containing the copy of the last two bytes, the data,
137
- // and the copy of the beginning as a tail.
42
+ /* The actual ring buffer containing the copy of the last two bytes, the data,
43
+ and the copy of the beginning as a tail. */
138
44
  uint8_t *data_;
139
- // The start of the ringbuffer.
45
+ /* The start of the ringbuffer. */
140
46
  uint8_t *buffer_;
141
- };
47
+ } RingBuffer;
48
+
49
+ static BROTLI_INLINE void RingBufferInit(RingBuffer* rb) {
50
+ rb->cur_size_ = 0;
51
+ rb->pos_ = 0;
52
+ rb->data_ = 0;
53
+ rb->buffer_ = 0;
54
+ }
55
+
56
+ static BROTLI_INLINE void RingBufferSetup(
57
+ const BrotliEncoderParams* params, RingBuffer* rb) {
58
+ int window_bits = ComputeRbBits(params);
59
+ int tail_bits = params->lgblock;
60
+ *(uint32_t*)&rb->size_ = 1u << window_bits;
61
+ *(uint32_t*)&rb->mask_ = (1u << window_bits) - 1;
62
+ *(uint32_t*)&rb->tail_size_ = 1u << tail_bits;
63
+ *(uint32_t*)&rb->total_size_ = rb->size_ + rb->tail_size_;
64
+ }
65
+
66
+ static BROTLI_INLINE void RingBufferFree(MemoryManager* m, RingBuffer* rb) {
67
+ BROTLI_FREE(m, rb->data_);
68
+ }
69
+
70
+ /* Allocates or re-allocates data_ to the given length + plus some slack
71
+ region before and after. Fills the slack regions with zeros. */
72
+ static BROTLI_INLINE void RingBufferInitBuffer(
73
+ MemoryManager* m, const uint32_t buflen, RingBuffer* rb) {
74
+ static const size_t kSlackForEightByteHashingEverywhere = 7;
75
+ uint8_t* new_data = BROTLI_ALLOC(
76
+ m, uint8_t, 2 + buflen + kSlackForEightByteHashingEverywhere);
77
+ size_t i;
78
+ if (BROTLI_IS_OOM(m)) return;
79
+ if (rb->data_) {
80
+ memcpy(new_data, rb->data_,
81
+ 2 + rb->cur_size_ + kSlackForEightByteHashingEverywhere);
82
+ BROTLI_FREE(m, rb->data_);
83
+ }
84
+ rb->data_ = new_data;
85
+ rb->cur_size_ = buflen;
86
+ rb->buffer_ = rb->data_ + 2;
87
+ rb->buffer_[-2] = rb->buffer_[-1] = 0;
88
+ for (i = 0; i < kSlackForEightByteHashingEverywhere; ++i) {
89
+ rb->buffer_[rb->cur_size_ + i] = 0;
90
+ }
91
+ }
92
+
93
+ static BROTLI_INLINE void RingBufferWriteTail(
94
+ const uint8_t *bytes, size_t n, RingBuffer* rb) {
95
+ const size_t masked_pos = rb->pos_ & rb->mask_;
96
+ if (PREDICT_FALSE(masked_pos < rb->tail_size_)) {
97
+ /* Just fill the tail buffer with the beginning data. */
98
+ const size_t p = rb->size_ + masked_pos;
99
+ memcpy(&rb->buffer_[p], bytes,
100
+ BROTLI_MIN(size_t, n, rb->tail_size_ - masked_pos));
101
+ }
102
+ }
103
+
104
+ /* Push bytes into the ring buffer. */
105
+ static BROTLI_INLINE void RingBufferWrite(
106
+ MemoryManager* m, const uint8_t *bytes, size_t n, RingBuffer* rb) {
107
+ if (rb->pos_ == 0 && n < rb->tail_size_) {
108
+ /* Special case for the first write: to process the first block, we don't
109
+ need to allocate the whole ringbuffer and we don't need the tail
110
+ either. However, we do this memory usage optimization only if the
111
+ first write is less than the tail size, which is also the input block
112
+ size, otherwise it is likely that other blocks will follow and we
113
+ will need to reallocate to the full size anyway. */
114
+ rb->pos_ = (uint32_t)n;
115
+ RingBufferInitBuffer(m, rb->pos_, rb);
116
+ if (BROTLI_IS_OOM(m)) return;
117
+ memcpy(rb->buffer_, bytes, n);
118
+ return;
119
+ }
120
+ if (rb->cur_size_ < rb->total_size_) {
121
+ /* Lazily allocate the full buffer. */
122
+ RingBufferInitBuffer(m, rb->total_size_, rb);
123
+ if (BROTLI_IS_OOM(m)) return;
124
+ /* Initialize the last two bytes to zero, so that we don't have to worry
125
+ later when we copy the last two bytes to the first two positions. */
126
+ rb->buffer_[rb->size_ - 2] = 0;
127
+ rb->buffer_[rb->size_ - 1] = 0;
128
+ }
129
+ {
130
+ const size_t masked_pos = rb->pos_ & rb->mask_;
131
+ /* The length of the writes is limited so that we do not need to worry
132
+ about a write */
133
+ RingBufferWriteTail(bytes, n, rb);
134
+ if (PREDICT_TRUE(masked_pos + n <= rb->size_)) {
135
+ /* A single write fits. */
136
+ memcpy(&rb->buffer_[masked_pos], bytes, n);
137
+ } else {
138
+ /* Split into two writes.
139
+ Copy into the end of the buffer, including the tail buffer. */
140
+ memcpy(&rb->buffer_[masked_pos], bytes,
141
+ BROTLI_MIN(size_t, n, rb->total_size_ - masked_pos));
142
+ /* Copy into the beginning of the buffer */
143
+ memcpy(&rb->buffer_[0], bytes + (rb->size_ - masked_pos),
144
+ n - (rb->size_ - masked_pos));
145
+ }
146
+ }
147
+ rb->buffer_[-2] = rb->buffer_[rb->size_ - 2];
148
+ rb->buffer_[-1] = rb->buffer_[rb->size_ - 1];
149
+ rb->pos_ += (uint32_t)n;
150
+ if (rb->pos_ > (1u << 30)) {
151
+ /* Wrap, but preserve not-a-first-lap feature. */
152
+ rb->pos_ = (rb->pos_ & ((1u << 30) - 1)) | (1u << 30);
153
+ }
154
+ }
142
155
 
143
- } // namespace brotli
156
+ #if defined(__cplusplus) || defined(c_plusplus)
157
+ } /* extern "C" */
158
+ #endif
144
159
 
145
- #endif // BROTLI_ENC_RINGBUFFER_H_
160
+ #endif /* BROTLI_ENC_RINGBUFFER_H_ */