extbrotli 0.0.1.PROTOTYPE

Sign up to get free protection for your applications and to get access to all the features.
Files changed (68) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +28 -0
  3. data/README.md +67 -0
  4. data/Rakefile +158 -0
  5. data/contrib/brotli/LICENSE +202 -0
  6. data/contrib/brotli/README.md +18 -0
  7. data/contrib/brotli/dec/bit_reader.c +55 -0
  8. data/contrib/brotli/dec/bit_reader.h +256 -0
  9. data/contrib/brotli/dec/context.h +260 -0
  10. data/contrib/brotli/dec/decode.c +1573 -0
  11. data/contrib/brotli/dec/decode.h +160 -0
  12. data/contrib/brotli/dec/dictionary.h +9494 -0
  13. data/contrib/brotli/dec/huffman.c +325 -0
  14. data/contrib/brotli/dec/huffman.h +77 -0
  15. data/contrib/brotli/dec/port.h +148 -0
  16. data/contrib/brotli/dec/prefix.h +756 -0
  17. data/contrib/brotli/dec/state.c +149 -0
  18. data/contrib/brotli/dec/state.h +185 -0
  19. data/contrib/brotli/dec/streams.c +99 -0
  20. data/contrib/brotli/dec/streams.h +100 -0
  21. data/contrib/brotli/dec/transform.h +315 -0
  22. data/contrib/brotli/dec/types.h +36 -0
  23. data/contrib/brotli/enc/backward_references.cc +769 -0
  24. data/contrib/brotli/enc/backward_references.h +50 -0
  25. data/contrib/brotli/enc/bit_cost.h +147 -0
  26. data/contrib/brotli/enc/block_splitter.cc +418 -0
  27. data/contrib/brotli/enc/block_splitter.h +78 -0
  28. data/contrib/brotli/enc/brotli_bit_stream.cc +884 -0
  29. data/contrib/brotli/enc/brotli_bit_stream.h +149 -0
  30. data/contrib/brotli/enc/cluster.h +290 -0
  31. data/contrib/brotli/enc/command.h +140 -0
  32. data/contrib/brotli/enc/context.h +185 -0
  33. data/contrib/brotli/enc/dictionary.h +9485 -0
  34. data/contrib/brotli/enc/dictionary_hash.h +4125 -0
  35. data/contrib/brotli/enc/encode.cc +715 -0
  36. data/contrib/brotli/enc/encode.h +196 -0
  37. data/contrib/brotli/enc/encode_parallel.cc +354 -0
  38. data/contrib/brotli/enc/encode_parallel.h +37 -0
  39. data/contrib/brotli/enc/entropy_encode.cc +492 -0
  40. data/contrib/brotli/enc/entropy_encode.h +88 -0
  41. data/contrib/brotli/enc/fast_log.h +179 -0
  42. data/contrib/brotli/enc/find_match_length.h +87 -0
  43. data/contrib/brotli/enc/hash.h +686 -0
  44. data/contrib/brotli/enc/histogram.cc +76 -0
  45. data/contrib/brotli/enc/histogram.h +100 -0
  46. data/contrib/brotli/enc/literal_cost.cc +172 -0
  47. data/contrib/brotli/enc/literal_cost.h +38 -0
  48. data/contrib/brotli/enc/metablock.cc +544 -0
  49. data/contrib/brotli/enc/metablock.h +88 -0
  50. data/contrib/brotli/enc/port.h +151 -0
  51. data/contrib/brotli/enc/prefix.h +85 -0
  52. data/contrib/brotli/enc/ringbuffer.h +108 -0
  53. data/contrib/brotli/enc/static_dict.cc +441 -0
  54. data/contrib/brotli/enc/static_dict.h +40 -0
  55. data/contrib/brotli/enc/static_dict_lut.h +12063 -0
  56. data/contrib/brotli/enc/streams.cc +127 -0
  57. data/contrib/brotli/enc/streams.h +129 -0
  58. data/contrib/brotli/enc/transform.h +250 -0
  59. data/contrib/brotli/enc/write_bits.h +91 -0
  60. data/ext/extbrotli.cc +24 -0
  61. data/ext/extbrotli.h +73 -0
  62. data/ext/extconf.rb +35 -0
  63. data/ext/lldecoder.c +220 -0
  64. data/ext/llencoder.cc +433 -0
  65. data/gemstub.rb +21 -0
  66. data/lib/extbrotli.rb +243 -0
  67. data/lib/extbrotli/version.rb +3 -0
  68. metadata +140 -0
@@ -0,0 +1,127 @@
1
+ // Copyright 2009 Google Inc. All Rights Reserved.
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+ //
15
+ // Convience routines to make Brotli I/O classes from some memory containers and
16
+ // files.
17
+
18
+ #include "./streams.h"
19
+
20
+ #include <assert.h>
21
+ #include <stdlib.h>
22
+ #include <string.h>
23
+
24
+ namespace brotli {
25
+
26
+ BrotliMemOut::BrotliMemOut(void* buf, int len)
27
+ : buf_(buf),
28
+ len_(len),
29
+ pos_(0) {}
30
+
31
+ void BrotliMemOut::Reset(void* buf, int len) {
32
+ buf_ = buf;
33
+ len_ = len;
34
+ pos_ = 0;
35
+ }
36
+
37
+ // Brotli output routine: copy n bytes to the output buffer.
38
+ bool BrotliMemOut::Write(const void *buf, size_t n) {
39
+ if (n + pos_ > len_)
40
+ return false;
41
+ char* p = reinterpret_cast<char*>(buf_) + pos_;
42
+ memcpy(p, buf, n);
43
+ pos_ += n;
44
+ return true;
45
+ }
46
+
47
+ BrotliStringOut::BrotliStringOut(std::string* buf, int max_size)
48
+ : buf_(buf),
49
+ max_size_(max_size) {
50
+ assert(buf->empty());
51
+ }
52
+
53
+ void BrotliStringOut::Reset(std::string* buf, int max_size) {
54
+ buf_ = buf;
55
+ max_size_ = max_size;
56
+ }
57
+
58
+ // Brotli output routine: add n bytes to a string.
59
+ bool BrotliStringOut::Write(const void *buf, size_t n) {
60
+ if (buf_->size() + n > max_size_)
61
+ return false;
62
+ buf_->append(static_cast<const char*>(buf), n);
63
+ return true;
64
+ }
65
+
66
+ BrotliMemIn::BrotliMemIn(const void* buf, int len)
67
+ : buf_(buf),
68
+ len_(len),
69
+ pos_(0) {}
70
+
71
+ void BrotliMemIn::Reset(const void* buf, int len) {
72
+ buf_ = buf;
73
+ len_ = len;
74
+ pos_ = 0;
75
+ }
76
+
77
+ // Brotli input routine: read the next chunk of memory.
78
+ const void* BrotliMemIn::Read(size_t n, size_t* output) {
79
+ if (pos_ == len_) {
80
+ return NULL;
81
+ }
82
+ if (n > len_ - pos_)
83
+ n = len_ - pos_;
84
+ const char* p = reinterpret_cast<const char*>(buf_) + pos_;
85
+ pos_ += n;
86
+ *output = n;
87
+ return p;
88
+ }
89
+
90
+ BrotliFileIn::BrotliFileIn(FILE* f, size_t max_read_size)
91
+ : f_(f),
92
+ buf_(malloc(max_read_size)),
93
+ buf_size_(max_read_size) {}
94
+
95
+ BrotliFileIn::~BrotliFileIn() {
96
+ if (buf_) free(buf_);
97
+ }
98
+
99
+ const void* BrotliFileIn::Read(size_t n, size_t* bytes_read) {
100
+ if (buf_ == NULL) {
101
+ *bytes_read = 0;
102
+ return NULL;
103
+ }
104
+ if (n > buf_size_) {
105
+ n = buf_size_;
106
+ } else if (n == 0) {
107
+ return feof(f_) ? NULL : buf_;
108
+ }
109
+ *bytes_read = fread(buf_, 1, n, f_);
110
+ if (*bytes_read == 0) {
111
+ return NULL;
112
+ } else {
113
+ return buf_;
114
+ }
115
+ }
116
+
117
+ BrotliFileOut::BrotliFileOut(FILE* f) : f_(f) {}
118
+
119
+ bool BrotliFileOut::Write(const void* buf, size_t n) {
120
+ if (fwrite(buf, n, 1, f_) != 1) {
121
+ return false;
122
+ }
123
+ return true;
124
+ }
125
+
126
+
127
+ } // namespace brotli
@@ -0,0 +1,129 @@
1
+ // Copyright 2009 Google Inc. All Rights Reserved.
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+ //
15
+ // Input and output classes for streaming brotli compression.
16
+
17
+ #ifndef BROTLI_ENC_STREAMS_H_
18
+ #define BROTLI_ENC_STREAMS_H_
19
+
20
+ #include <stddef.h>
21
+ #include <stdio.h>
22
+ #include <string>
23
+
24
+ namespace brotli {
25
+
26
+ // Input interface for the compression routines.
27
+ class BrotliIn {
28
+ public:
29
+ virtual ~BrotliIn() {}
30
+
31
+ // Return a pointer to the next block of input of at most n bytes.
32
+ // Return the actual length in *nread.
33
+ // At end of data, return NULL. Don't return NULL if there is more data
34
+ // to read, even if called with n == 0.
35
+ // Read will only be called if some of its bytes are needed.
36
+ virtual const void* Read(size_t n, size_t* nread) = 0;
37
+ };
38
+
39
+ // Output interface for the compression routines.
40
+ class BrotliOut {
41
+ public:
42
+ virtual ~BrotliOut() {}
43
+
44
+ // Write n bytes of data from buf.
45
+ // Return true if all written, false otherwise.
46
+ virtual bool Write(const void *buf, size_t n) = 0;
47
+ };
48
+
49
+ // Adapter class to make BrotliIn objects from raw memory.
50
+ class BrotliMemIn : public BrotliIn {
51
+ public:
52
+ BrotliMemIn(const void* buf, int len);
53
+
54
+ void Reset(const void* buf, int len);
55
+
56
+ // returns the amount of data consumed
57
+ int position() const { return pos_; }
58
+
59
+ const void* Read(size_t n, size_t* OUTPUT) override;
60
+
61
+ private:
62
+ const void* buf_; // start of input buffer
63
+ int len_; // length of input
64
+ int pos_; // current read position within input
65
+ };
66
+
67
+ // Adapter class to make BrotliOut objects from raw memory.
68
+ class BrotliMemOut : public BrotliOut {
69
+ public:
70
+ BrotliMemOut(void* buf, int len);
71
+
72
+ void Reset(void* buf, int len);
73
+
74
+ // returns the amount of data written
75
+ int position() const { return pos_; }
76
+
77
+ bool Write(const void* buf, size_t n) override;
78
+
79
+ private:
80
+ void* buf_; // start of output buffer
81
+ int len_; // length of output
82
+ int pos_; // current write position within output
83
+ };
84
+
85
+ // Adapter class to make BrotliOut objects from a string.
86
+ class BrotliStringOut : public BrotliOut {
87
+ public:
88
+ // Create a writer that appends its data to buf.
89
+ // buf->size() will grow to at most max_size
90
+ // buf is expected to be empty when constructing BrotliStringOut.
91
+ BrotliStringOut(std::string* buf, int max_size);
92
+
93
+ void Reset(std::string* buf, int max_len);
94
+
95
+ bool Write(const void* buf, size_t n) override;
96
+
97
+ private:
98
+ std::string* buf_; // start of output buffer
99
+ int max_size_; // max length of output
100
+ };
101
+
102
+ // Adapter class to make BrotliIn object from a file.
103
+ class BrotliFileIn : public BrotliIn {
104
+ public:
105
+ BrotliFileIn(FILE* f, size_t max_read_size);
106
+ ~BrotliFileIn();
107
+
108
+ const void* Read(size_t n, size_t* bytes_read) override;
109
+
110
+ private:
111
+ FILE* f_;
112
+ void* buf_;
113
+ size_t buf_size_;
114
+ };
115
+
116
+ // Adapter class to make BrotliOut object from a file.
117
+ class BrotliFileOut : public BrotliOut {
118
+ public:
119
+ explicit BrotliFileOut(FILE* f);
120
+
121
+ bool Write(const void* buf, size_t n) override;
122
+ private:
123
+ FILE* f_;
124
+ };
125
+
126
+
127
+ } // namespace brotli
128
+
129
+ #endif // BROTLI_ENC_STREAMS_H_
@@ -0,0 +1,250 @@
1
+ // Copyright 2010 Google Inc. All Rights Reserved.
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+ //
15
+ // Transformations on dictionary words.
16
+
17
+ #ifndef BROTLI_ENC_TRANSFORM_H_
18
+ #define BROTLI_ENC_TRANSFORM_H_
19
+
20
+ #include <string>
21
+
22
+ #include "./dictionary.h"
23
+
24
+ namespace brotli {
25
+
26
+ enum WordTransformType {
27
+ kIdentity = 0,
28
+ kOmitLast1 = 1,
29
+ kOmitLast2 = 2,
30
+ kOmitLast3 = 3,
31
+ kOmitLast4 = 4,
32
+ kOmitLast5 = 5,
33
+ kOmitLast6 = 6,
34
+ kOmitLast7 = 7,
35
+ kOmitLast8 = 8,
36
+ kOmitLast9 = 9,
37
+ kUppercaseFirst = 10,
38
+ kUppercaseAll = 11,
39
+ kOmitFirst1 = 12,
40
+ kOmitFirst2 = 13,
41
+ kOmitFirst3 = 14,
42
+ kOmitFirst4 = 15,
43
+ kOmitFirst5 = 16,
44
+ kOmitFirst6 = 17,
45
+ kOmitFirst7 = 18,
46
+ kOmitFirst8 = 19,
47
+ kOmitFirst9 = 20,
48
+ };
49
+
50
+ struct Transform {
51
+ const char* prefix;
52
+ WordTransformType word_transform;
53
+ const char* suffix;
54
+ };
55
+
56
+ static const Transform kTransforms[] = {
57
+ { "", kIdentity, "" },
58
+ { "", kIdentity, " " },
59
+ { " ", kIdentity, " " },
60
+ { "", kOmitFirst1, "" },
61
+ { "", kUppercaseFirst, " " },
62
+ { "", kIdentity, " the " },
63
+ { " ", kIdentity, "" },
64
+ { "s ", kIdentity, " " },
65
+ { "", kIdentity, " of " },
66
+ { "", kUppercaseFirst, "" },
67
+ { "", kIdentity, " and " },
68
+ { "", kOmitFirst2, "" },
69
+ { "", kOmitLast1, "" },
70
+ { ", ", kIdentity, " " },
71
+ { "", kIdentity, ", " },
72
+ { " ", kUppercaseFirst, " " },
73
+ { "", kIdentity, " in " },
74
+ { "", kIdentity, " to " },
75
+ { "e ", kIdentity, " " },
76
+ { "", kIdentity, "\"" },
77
+ { "", kIdentity, "." },
78
+ { "", kIdentity, "\">" },
79
+ { "", kIdentity, "\n" },
80
+ { "", kOmitLast3, "" },
81
+ { "", kIdentity, "]" },
82
+ { "", kIdentity, " for " },
83
+ { "", kOmitFirst3, "" },
84
+ { "", kOmitLast2, "" },
85
+ { "", kIdentity, " a " },
86
+ { "", kIdentity, " that " },
87
+ { " ", kUppercaseFirst, "" },
88
+ { "", kIdentity, ". " },
89
+ { ".", kIdentity, "" },
90
+ { " ", kIdentity, ", " },
91
+ { "", kOmitFirst4, "" },
92
+ { "", kIdentity, " with " },
93
+ { "", kIdentity, "'" },
94
+ { "", kIdentity, " from " },
95
+ { "", kIdentity, " by " },
96
+ { "", kOmitFirst5, "" },
97
+ { "", kOmitFirst6, "" },
98
+ { " the ", kIdentity, "" },
99
+ { "", kOmitLast4, "" },
100
+ { "", kIdentity, ". The " },
101
+ { "", kUppercaseAll, "" },
102
+ { "", kIdentity, " on " },
103
+ { "", kIdentity, " as " },
104
+ { "", kIdentity, " is " },
105
+ { "", kOmitLast7, "" },
106
+ { "", kOmitLast1, "ing " },
107
+ { "", kIdentity, "\n\t" },
108
+ { "", kIdentity, ":" },
109
+ { " ", kIdentity, ". " },
110
+ { "", kIdentity, "ed " },
111
+ { "", kOmitFirst9, "" },
112
+ { "", kOmitFirst7, "" },
113
+ { "", kOmitLast6, "" },
114
+ { "", kIdentity, "(" },
115
+ { "", kUppercaseFirst, ", " },
116
+ { "", kOmitLast8, "" },
117
+ { "", kIdentity, " at " },
118
+ { "", kIdentity, "ly " },
119
+ { " the ", kIdentity, " of " },
120
+ { "", kOmitLast5, "" },
121
+ { "", kOmitLast9, "" },
122
+ { " ", kUppercaseFirst, ", " },
123
+ { "", kUppercaseFirst, "\"" },
124
+ { ".", kIdentity, "(" },
125
+ { "", kUppercaseAll, " " },
126
+ { "", kUppercaseFirst, "\">" },
127
+ { "", kIdentity, "=\"" },
128
+ { " ", kIdentity, "." },
129
+ { ".com/", kIdentity, "" },
130
+ { " the ", kIdentity, " of the " },
131
+ { "", kUppercaseFirst, "'" },
132
+ { "", kIdentity, ". This " },
133
+ { "", kIdentity, "," },
134
+ { ".", kIdentity, " " },
135
+ { "", kUppercaseFirst, "(" },
136
+ { "", kUppercaseFirst, "." },
137
+ { "", kIdentity, " not " },
138
+ { " ", kIdentity, "=\"" },
139
+ { "", kIdentity, "er " },
140
+ { " ", kUppercaseAll, " " },
141
+ { "", kIdentity, "al " },
142
+ { " ", kUppercaseAll, "" },
143
+ { "", kIdentity, "='" },
144
+ { "", kUppercaseAll, "\"" },
145
+ { "", kUppercaseFirst, ". " },
146
+ { " ", kIdentity, "(" },
147
+ { "", kIdentity, "ful " },
148
+ { " ", kUppercaseFirst, ". " },
149
+ { "", kIdentity, "ive " },
150
+ { "", kIdentity, "less " },
151
+ { "", kUppercaseAll, "'" },
152
+ { "", kIdentity, "est " },
153
+ { " ", kUppercaseFirst, "." },
154
+ { "", kUppercaseAll, "\">" },
155
+ { " ", kIdentity, "='" },
156
+ { "", kUppercaseFirst, "," },
157
+ { "", kIdentity, "ize " },
158
+ { "", kUppercaseAll, "." },
159
+ { "\xc2\xa0", kIdentity, "" },
160
+ { " ", kIdentity, "," },
161
+ { "", kUppercaseFirst, "=\"" },
162
+ { "", kUppercaseAll, "=\"" },
163
+ { "", kIdentity, "ous " },
164
+ { "", kUppercaseAll, ", " },
165
+ { "", kUppercaseFirst, "='" },
166
+ { " ", kUppercaseFirst, "," },
167
+ { " ", kUppercaseAll, "=\"" },
168
+ { " ", kUppercaseAll, ", " },
169
+ { "", kUppercaseAll, "," },
170
+ { "", kUppercaseAll, "(" },
171
+ { "", kUppercaseAll, ". " },
172
+ { " ", kUppercaseAll, "." },
173
+ { "", kUppercaseAll, "='" },
174
+ { " ", kUppercaseAll, ". " },
175
+ { " ", kUppercaseFirst, "=\"" },
176
+ { " ", kUppercaseAll, "='" },
177
+ { " ", kUppercaseFirst, "='" },
178
+ };
179
+
180
+ static const int kNumTransforms = sizeof(kTransforms) / sizeof(kTransforms[0]);
181
+
182
+ static const int kOmitFirstNTransforms[10] = {
183
+ 0, 3, 11, 26, 34, 39, 40, 55, 0, 54
184
+ };
185
+
186
+ static const int kOmitLastNTransforms[10] = {
187
+ 0, 12, 27, 23, 42, 63, 56, 48, 59, 64,
188
+ };
189
+
190
+ static int ToUpperCase(uint8_t *p, int len) {
191
+ if (len == 1 || p[0] < 0xc0) {
192
+ if (p[0] >= 'a' && p[0] <= 'z') {
193
+ p[0] ^= 32;
194
+ }
195
+ return 1;
196
+ }
197
+ if (p[0] < 0xe0) {
198
+ p[1] ^= 32;
199
+ return 2;
200
+ }
201
+ if (len == 2) {
202
+ return 2;
203
+ }
204
+ p[2] ^= 5;
205
+ return 3;
206
+ }
207
+
208
+ inline std::string ApplyTransform(
209
+ const Transform& t, const uint8_t* word, int len) {
210
+ std::string ret(t.prefix);
211
+ if (t.word_transform <= kOmitLast9) {
212
+ len -= t.word_transform;
213
+ }
214
+ if (len > 0) {
215
+ if (t.word_transform >= kOmitFirst1) {
216
+ const int skip = t.word_transform - (kOmitFirst1 - 1);
217
+ if (len > skip) {
218
+ ret += std::string(word + skip, word + len);
219
+ }
220
+ } else {
221
+ ret += std::string(word, word + len);
222
+ uint8_t *uppercase = reinterpret_cast<uint8_t*>(&ret[ret.size() - len]);
223
+ if (t.word_transform == kUppercaseFirst) {
224
+ ToUpperCase(uppercase, len);
225
+ } else if (t.word_transform == kUppercaseAll) {
226
+ while (len > 0) {
227
+ int step = ToUpperCase(uppercase, len);
228
+ uppercase += step;
229
+ len -= step;
230
+ }
231
+ }
232
+ }
233
+ }
234
+ ret += std::string(t.suffix);
235
+ return ret;
236
+ }
237
+
238
+ inline std::string GetTransformedDictionaryWord(int len_code, int word_id) {
239
+ int num_words = 1 << kBrotliDictionarySizeBitsByLength[len_code];
240
+ int offset = kBrotliDictionaryOffsetsByLength[len_code];
241
+ int t = word_id / num_words;
242
+ int word_idx = word_id % num_words;
243
+ offset += len_code * word_idx;
244
+ const uint8_t* word = &kBrotliDictionary[offset];
245
+ return ApplyTransform(kTransforms[t], word, len_code);
246
+ }
247
+
248
+ } // namespace brotli
249
+
250
+ #endif // BROTLI_ENC_TRANSFORM_H_