extbrotli 0.0.1.PROTOTYPE

Sign up to get free protection for your applications and to get access to all the features.
Files changed (68) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +28 -0
  3. data/README.md +67 -0
  4. data/Rakefile +158 -0
  5. data/contrib/brotli/LICENSE +202 -0
  6. data/contrib/brotli/README.md +18 -0
  7. data/contrib/brotli/dec/bit_reader.c +55 -0
  8. data/contrib/brotli/dec/bit_reader.h +256 -0
  9. data/contrib/brotli/dec/context.h +260 -0
  10. data/contrib/brotli/dec/decode.c +1573 -0
  11. data/contrib/brotli/dec/decode.h +160 -0
  12. data/contrib/brotli/dec/dictionary.h +9494 -0
  13. data/contrib/brotli/dec/huffman.c +325 -0
  14. data/contrib/brotli/dec/huffman.h +77 -0
  15. data/contrib/brotli/dec/port.h +148 -0
  16. data/contrib/brotli/dec/prefix.h +756 -0
  17. data/contrib/brotli/dec/state.c +149 -0
  18. data/contrib/brotli/dec/state.h +185 -0
  19. data/contrib/brotli/dec/streams.c +99 -0
  20. data/contrib/brotli/dec/streams.h +100 -0
  21. data/contrib/brotli/dec/transform.h +315 -0
  22. data/contrib/brotli/dec/types.h +36 -0
  23. data/contrib/brotli/enc/backward_references.cc +769 -0
  24. data/contrib/brotli/enc/backward_references.h +50 -0
  25. data/contrib/brotli/enc/bit_cost.h +147 -0
  26. data/contrib/brotli/enc/block_splitter.cc +418 -0
  27. data/contrib/brotli/enc/block_splitter.h +78 -0
  28. data/contrib/brotli/enc/brotli_bit_stream.cc +884 -0
  29. data/contrib/brotli/enc/brotli_bit_stream.h +149 -0
  30. data/contrib/brotli/enc/cluster.h +290 -0
  31. data/contrib/brotli/enc/command.h +140 -0
  32. data/contrib/brotli/enc/context.h +185 -0
  33. data/contrib/brotli/enc/dictionary.h +9485 -0
  34. data/contrib/brotli/enc/dictionary_hash.h +4125 -0
  35. data/contrib/brotli/enc/encode.cc +715 -0
  36. data/contrib/brotli/enc/encode.h +196 -0
  37. data/contrib/brotli/enc/encode_parallel.cc +354 -0
  38. data/contrib/brotli/enc/encode_parallel.h +37 -0
  39. data/contrib/brotli/enc/entropy_encode.cc +492 -0
  40. data/contrib/brotli/enc/entropy_encode.h +88 -0
  41. data/contrib/brotli/enc/fast_log.h +179 -0
  42. data/contrib/brotli/enc/find_match_length.h +87 -0
  43. data/contrib/brotli/enc/hash.h +686 -0
  44. data/contrib/brotli/enc/histogram.cc +76 -0
  45. data/contrib/brotli/enc/histogram.h +100 -0
  46. data/contrib/brotli/enc/literal_cost.cc +172 -0
  47. data/contrib/brotli/enc/literal_cost.h +38 -0
  48. data/contrib/brotli/enc/metablock.cc +544 -0
  49. data/contrib/brotli/enc/metablock.h +88 -0
  50. data/contrib/brotli/enc/port.h +151 -0
  51. data/contrib/brotli/enc/prefix.h +85 -0
  52. data/contrib/brotli/enc/ringbuffer.h +108 -0
  53. data/contrib/brotli/enc/static_dict.cc +441 -0
  54. data/contrib/brotli/enc/static_dict.h +40 -0
  55. data/contrib/brotli/enc/static_dict_lut.h +12063 -0
  56. data/contrib/brotli/enc/streams.cc +127 -0
  57. data/contrib/brotli/enc/streams.h +129 -0
  58. data/contrib/brotli/enc/transform.h +250 -0
  59. data/contrib/brotli/enc/write_bits.h +91 -0
  60. data/ext/extbrotli.cc +24 -0
  61. data/ext/extbrotli.h +73 -0
  62. data/ext/extconf.rb +35 -0
  63. data/ext/lldecoder.c +220 -0
  64. data/ext/llencoder.cc +433 -0
  65. data/gemstub.rb +21 -0
  66. data/lib/extbrotli.rb +243 -0
  67. data/lib/extbrotli/version.rb +3 -0
  68. metadata +140 -0
@@ -0,0 +1,91 @@
1
+ // Copyright 2010 Google Inc. All Rights Reserved.
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+ //
15
+ // Write bits into a byte array.
16
+
17
+ #ifndef BROTLI_ENC_WRITE_BITS_H_
18
+ #define BROTLI_ENC_WRITE_BITS_H_
19
+
20
+ #include <assert.h>
21
+ #include <stdint.h>
22
+ #include <stdio.h>
23
+
24
+ #include "./port.h"
25
+
26
+ namespace brotli {
27
+
28
+ //#define BIT_WRITER_DEBUG
29
+
30
+ // This function writes bits into bytes in increasing addresses, and within
31
+ // a byte least-significant-bit first.
32
+ //
33
+ // The function can write up to 56 bits in one go with WriteBits
34
+ // Example: let's assume that 3 bits (Rs below) have been written already:
35
+ //
36
+ // BYTE-0 BYTE+1 BYTE+2
37
+ //
38
+ // 0000 0RRR 0000 0000 0000 0000
39
+ //
40
+ // Now, we could write 5 or less bits in MSB by just sifting by 3
41
+ // and OR'ing to BYTE-0.
42
+ //
43
+ // For n bits, we take the last 5 bits, OR that with high bits in BYTE-0,
44
+ // and locate the rest in BYTE+1, BYTE+2, etc.
45
+ inline void WriteBits(int n_bits,
46
+ uint64_t bits,
47
+ int * __restrict pos,
48
+ uint8_t * __restrict array) {
49
+ #ifdef BIT_WRITER_DEBUG
50
+ printf("WriteBits %2d 0x%016llx %10d\n", n_bits, bits, *pos);
51
+ #endif
52
+ assert(bits < 1ULL << n_bits);
53
+ #ifdef IS_LITTLE_ENDIAN
54
+ // This branch of the code can write up to 56 bits at a time,
55
+ // 7 bits are lost by being perhaps already in *p and at least
56
+ // 1 bit is needed to initialize the bit-stream ahead (i.e. if 7
57
+ // bits are in *p and we write 57 bits, then the next write will
58
+ // access a byte that was never initialized).
59
+ uint8_t *p = &array[*pos >> 3];
60
+ uint64_t v = *p;
61
+ v |= bits << (*pos & 7);
62
+ BROTLI_UNALIGNED_STORE64(p, v); // Set some bits.
63
+ *pos += n_bits;
64
+ #else
65
+ // implicit & 0xff is assumed for uint8_t arithmetics
66
+ uint8_t *array_pos = &array[*pos >> 3];
67
+ const int bits_reserved_in_first_byte = (*pos & 7);
68
+ bits <<= bits_reserved_in_first_byte;
69
+ *array_pos++ |= bits;
70
+ for (int bits_left_to_write = n_bits - 8 + bits_reserved_in_first_byte;
71
+ bits_left_to_write >= 1;
72
+ bits_left_to_write -= 8) {
73
+ bits >>= 8;
74
+ *array_pos++ = bits;
75
+ }
76
+ *array_pos = 0;
77
+ *pos += n_bits;
78
+ #endif
79
+ }
80
+
81
+ inline void WriteBitsPrepareStorage(int pos, uint8_t *array) {
82
+ #ifdef BIT_WRITER_DEBUG
83
+ printf("WriteBitsPrepareStorage %10d\n", pos);
84
+ #endif
85
+ assert((pos & 7) == 0);
86
+ array[pos >> 3] = 0;
87
+ }
88
+
89
+ } // namespace brotli
90
+
91
+ #endif // BROTLI_ENC_WRITE_BITS_H_
data/ext/extbrotli.cc ADDED
@@ -0,0 +1,24 @@
1
+ #include "extbrotli.h"
2
+
3
+ VALUE mBrotli;
4
+ VALUE mConst;
5
+ VALUE eError;
6
+ VALUE eNeedsMoreInput;
7
+ VALUE eNeedsMoreOutput;
8
+
9
+ EXTBROTLI_CEXTERN
10
+ void
11
+ Init_extbrotli(void)
12
+ {
13
+ mBrotli = rb_define_module("Brotli");
14
+
15
+ mConst = rb_define_module_under(mBrotli, "Constants");
16
+ rb_include_module(mBrotli, mConst);
17
+
18
+ eError = rb_define_class_under(mBrotli, "Error", rb_eRuntimeError);
19
+ eNeedsMoreInput = rb_define_class_under(mBrotli, "NeedsMoreInput", eError);
20
+ eNeedsMoreOutput = rb_define_class_under(mBrotli, "NeedsMoreOutput", eError);
21
+
22
+ extbrotli_init_lowlevelencoder();
23
+ extbrotli_init_lowleveldecoder();
24
+ }
data/ext/extbrotli.h ADDED
@@ -0,0 +1,73 @@
1
+ #ifndef EXTBROTLI_H
2
+ #define EXTBROTLI_H 1
3
+
4
+ #include <ruby.h>
5
+
6
+ #ifdef __cplusplus
7
+ # define EXTBROTLI_CEXTERN extern "C"
8
+ # define EXTBROTLI_BEGIN_CEXTERN EXTBROTLI_CEXTERN {
9
+ # define EXTBROTLI_END_CEXTERN }
10
+ #else
11
+ # define EXTBROTLI_CEXTERN
12
+ # define EXTBROTLI_BEGIN_CEXTERN
13
+ # define EXTBROTLI_END_CEXTERN
14
+ #endif
15
+
16
+ #define RDOCFAKE(...)
17
+
18
+ EXTBROTLI_BEGIN_CEXTERN
19
+
20
+ extern VALUE mBrotli;
21
+ RDOCFAKE(mBrotli = rb_define_module("Brotli"));
22
+
23
+ extern VALUE mConst;
24
+ RDOCFAKE(mConst = rb_define_module_under(mBrotli, "Constants"));
25
+
26
+ extern VALUE eError;
27
+ RDOCFAKE(eError = rb_define_class_under(mBrotli, "Error", rb_eRuntimeError));
28
+
29
+ extern VALUE eNeedsMoreInput;
30
+ RDOCFAKE(eNeedsMoreInput = rb_define_class_under(mBrotli, "NeedsMoreInput", eError));
31
+
32
+ extern VALUE eNeedsMoreOutput;
33
+ RDOCFAKE(eNeedsMoreOutput = rb_define_class_under(mBrotli, "NeedsMoreOutput", eError));
34
+
35
+ void extbrotli_init_lowlevelencoder(void);
36
+ void extbrotli_init_lowleveldecoder(void);
37
+
38
+ static inline void
39
+ referr(VALUE obj)
40
+ {
41
+ rb_raise(rb_eTypeError,
42
+ "invalid reference - #<%s:%p>",
43
+ rb_obj_classname(obj), (void *)obj);
44
+ }
45
+
46
+ static inline void
47
+ reiniterror(VALUE obj)
48
+ {
49
+ rb_raise(rb_eTypeError,
50
+ "initialized object already - #<%s:%p>",
51
+ rb_obj_classname(obj), (void *)obj);
52
+ }
53
+
54
+ static inline void *
55
+ getrefp(VALUE obj, const rb_data_type_t *type)
56
+ {
57
+ void *p;
58
+ TypedData_Get_Struct(obj, void, type, p);
59
+ return p;
60
+ }
61
+
62
+ static inline void *
63
+ getref(VALUE obj, const rb_data_type_t *type)
64
+ {
65
+ void *p = getrefp(obj, type);
66
+ if (!p) { referr(obj); }
67
+ return p;
68
+ }
69
+
70
+
71
+ EXTBROTLI_END_CEXTERN
72
+
73
+ #endif /* EXTBROTLI_H */
data/ext/extconf.rb ADDED
@@ -0,0 +1,35 @@
1
+ #!ruby
2
+ #vim: set fileencoding:utf-8
3
+
4
+ require "mkmf"
5
+
6
+ dir = File.dirname(__FILE__).gsub(/[\[\{\?\*]/, "[\\0]")
7
+
8
+ encfiles = Dir.glob(File.join(dir, "../contrib/brotli/enc/*.cc")).sort
9
+ encsrc = "brotli_encoder.cc"
10
+ File.write encsrc, (encfiles.map {|n| "#include \"#{n}\"" }.join("\n") << "\n")
11
+ decfiles = Dir.glob(File.join(dir, "../contrib/brotli/dec/*.c")).sort
12
+ decsrc = "brotli_decoder.c"
13
+ File.write decsrc, (decfiles.map {|n| "#include \"#{n}\"" }.join("\n") << "\n")
14
+
15
+ filepattern = "*.c{,c,pp}"
16
+ target = File.join(dir, filepattern)
17
+ files = Dir.glob(target).map { |n| File.basename n }
18
+ rejects = (RbConfig::CONFIG["arch"] =~ /mswin|mingw/) ? /_pthreads_/ : /_win32_/
19
+ files.reject! { |n| n =~ rejects }
20
+ $srcs = files + [encsrc, decsrc]
21
+
22
+ #$VPATH.push "$(srcdir)/../contrib"
23
+
24
+ #find_header "brotli/enc/encode.h", "$(srcdir)/../contrib" or abort
25
+ find_header "brotli/dec/decode.h", "$(srcdir)/../contrib" or abort
26
+
27
+ if RbConfig::CONFIG["arch"] =~ /mingw/
28
+ $CPPFLAGS << " -D__forceinline=__attribute__\\(\\(always_inline\\)\\)"
29
+ $CXXFLAGS << " -std=gnu++11"
30
+ $LDFLAGS << " -static-libgcc -static-libstdc++"
31
+ end
32
+
33
+ try_link "void main(void){}", " -Wl,-Bsymbolic " and $LDFLAGS << " -Wl,-Bsymbolic "
34
+
35
+ create_makefile("extbrotli")
data/ext/lldecoder.c ADDED
@@ -0,0 +1,220 @@
1
+ #include "extbrotli.h"
2
+ #include <brotli/dec/decode.h>
3
+
4
+ static VALUE cLLDecoder; /* class Brotli::LowLevelDecoder */
5
+
6
+ static void
7
+ lldec_free(void *pp)
8
+ {
9
+ if (pp) {
10
+ BrotliState *p = (BrotliState *)pp;
11
+ BrotliStateCleanup(p);
12
+ }
13
+ }
14
+
15
+ static const rb_data_type_t lldecoder_type = {
16
+ .wrap_struct_name = "extbrotli.lowleveldecoder",
17
+ .function.dmark = NULL,
18
+ .function.dfree = lldec_free,
19
+ .function.dsize = NULL,
20
+ };
21
+
22
+ static inline BrotliState *
23
+ getlldecoderp(VALUE obj)
24
+ {
25
+ return (BrotliState *)getrefp(obj, &lldecoder_type);
26
+ }
27
+
28
+ static inline BrotliState *
29
+ getlldecoder(VALUE obj)
30
+ {
31
+ return (BrotliState *)getref(obj, &lldecoder_type);
32
+ }
33
+
34
+ static VALUE
35
+ lldec_alloc(VALUE mod)
36
+ {
37
+ return TypedData_Wrap_Struct(mod, &lldecoder_type, NULL);
38
+ }
39
+
40
+ static VALUE
41
+ lldec_init(VALUE dec)
42
+ {
43
+ BrotliState *p = getlldecoderp(dec);
44
+ if (p) { reiniterror(dec); }
45
+ DATA_PTR(dec) = p = ALLOC(BrotliState);
46
+ BrotliStateInit(p);
47
+ return dec;
48
+ }
49
+
50
+ /*
51
+ * call-seq:
52
+ * decode(src, dest, maxdest, finish) -> [result_code, src, dest]
53
+ */
54
+ static VALUE
55
+ lldec_decode(VALUE dec, VALUE src, VALUE dest, VALUE maxdest, VALUE finish)
56
+ {
57
+ BrotliState *p = getlldecoder(dec);
58
+
59
+ char *srcp, *destp;
60
+ size_t srcsize, destsize;
61
+ if (NIL_P(src)) {
62
+ srcp = NULL;
63
+ srcsize = 0;
64
+ } else {
65
+ rb_obj_infect(dec, src);
66
+ rb_check_type(src, RUBY_T_STRING);
67
+ RSTRING_GETMEM(src, srcp, srcsize);
68
+ }
69
+ if (NIL_P(dest)) {
70
+ destp = NULL;
71
+ destsize = 0;
72
+ } else {
73
+ rb_check_type(dest, RUBY_T_STRING);
74
+ destsize = NUM2SIZET(maxdest);
75
+ rb_str_modify(dest);
76
+ rb_str_set_len(dest, 0);
77
+ rb_str_modify_expand(dest, destsize);
78
+ destp = RSTRING_PTR(dest);
79
+ }
80
+ char *srcpp = srcp, *destpp = destp;
81
+ size_t total;
82
+ BrotliResult s = BrotliDecompressBufferStreaming(
83
+ &srcsize, (const uint8_t **)&srcp, NUM2INT(finish),
84
+ &destsize, (uint8_t **)&destp, &total, p);
85
+ switch (s) {
86
+ case BROTLI_RESULT_ERROR:
87
+ rb_raise(eError, "failed BrotliDecompressBufferStreaming() - #<%s:%p>", rb_obj_classname(dec), (void *)dec);
88
+ case BROTLI_RESULT_SUCCESS:
89
+ case BROTLI_RESULT_NEEDS_MORE_INPUT:
90
+ case BROTLI_RESULT_NEEDS_MORE_OUTPUT:
91
+ if (destp > destpp) {
92
+ rb_obj_infect(dest, dec);
93
+ rb_str_set_len(dest, destp - destpp);
94
+ }
95
+ if (srcsize > 0) {
96
+ src = rb_str_substr(src, srcp - srcpp, srcsize);
97
+ } else {
98
+ src = Qnil;
99
+ }
100
+ break;
101
+ default:
102
+ rb_raise(eError,
103
+ "failed BrotliDecompressBufferStreaming() - unknown result code (%d) - #<%s:%p>",
104
+ s, rb_obj_classname(dec), (void *)dec);
105
+ }
106
+
107
+ const VALUE vec[3] = { INT2NUM(s), src, dest };
108
+ return rb_ary_new4(3, vec);
109
+ }
110
+
111
+ static void
112
+ lldec_s_decode_args(int argc, VALUE argv[], VALUE *src, VALUE *dest, size_t *srcsize, size_t *destsize, int *partial)
113
+ {
114
+ if (argc > 0) {
115
+ *src = argv[0];
116
+ rb_check_type(*src, RUBY_T_STRING);
117
+ *srcsize = RSTRING_LEN(*src);
118
+
119
+ int s;
120
+ switch (argc) {
121
+ case 1:
122
+ s = BrotliDecompressedSize(*srcsize, (const uint8_t *)RSTRING_PTR(*src), destsize);
123
+ if (s == 0) { rb_raise(eError, "failed BrotliDecompressedSize()"); }
124
+ *dest = rb_str_buf_new(*destsize);
125
+ *partial = 0;
126
+ return;
127
+ case 2:
128
+ *dest = argv[1];
129
+ if (rb_type_p(*dest, RUBY_T_STRING)) {
130
+ s = BrotliDecompressedSize(*srcsize, (const uint8_t *)RSTRING_PTR(*src), destsize);
131
+ if (s == 0) { rb_raise(eError, "failed BrotliDecompressedSize()"); }
132
+ rb_check_type(*dest, RUBY_T_STRING);
133
+ rb_str_modify(*dest);
134
+ rb_str_set_len(*dest, 0);
135
+ rb_str_modify_expand(*dest, *destsize);
136
+ } else {
137
+ *destsize = NUM2SIZET(*dest);
138
+ *dest = rb_str_buf_new(*destsize);
139
+ }
140
+ *partial = 0;
141
+ return;
142
+ case 3:
143
+ *destsize = NUM2SIZET(argv[1]);
144
+ *dest = argv[2];
145
+ if (*dest == Qtrue || *dest == Qfalse) {
146
+ *partial = RTEST(*dest) ? 1 : 0;
147
+ *dest = rb_str_buf_new(*destsize);
148
+ } else {
149
+ rb_check_type(*dest, RUBY_T_STRING);
150
+ rb_str_modify(*dest);
151
+ rb_str_set_len(*dest, 0);
152
+ rb_str_modify_expand(*dest, *destsize);
153
+ *partial = 0;
154
+ }
155
+ return;
156
+ case 4:
157
+ *destsize = NUM2SIZET(argv[1]);
158
+ *dest = argv[2];
159
+ rb_check_type(*dest, RUBY_T_STRING);
160
+ rb_str_modify(*dest);
161
+ rb_str_set_len(*dest, 0);
162
+ rb_str_modify_expand(*dest, *destsize);
163
+ *partial = RTEST(argv[3]) ? 1 : 0;
164
+ return;
165
+ }
166
+ }
167
+
168
+ rb_error_arity(argc, 1, 4);
169
+ }
170
+
171
+ /*
172
+ * call-seq:
173
+ * decode(src, dest = "") -> decoded binary string
174
+ * decode(src, destsize, partial = false) -> decoded binary string
175
+ * decode(src, destsize, dest = "", partial = false) -> decoded binary string
176
+ */
177
+ static VALUE
178
+ lldec_s_decode(int argc, VALUE argv[], VALUE mod)
179
+ {
180
+ VALUE src, dest;
181
+ size_t srcsize, destsize;
182
+ int partial;
183
+ lldec_s_decode_args(argc, argv, &src, &dest, &srcsize, &destsize, &partial);
184
+
185
+ int s = BrotliDecompressBuffer(srcsize, (const uint8_t *)RSTRING_PTR(src),
186
+ &destsize, (uint8_t *)RSTRING_PTR(dest));
187
+
188
+ if (s == BROTLI_RESULT_NEEDS_MORE_INPUT) {
189
+ if (!partial) {
190
+ rb_raise(eNeedsMoreInput, "failed BrotliDecompressBuffer()");
191
+ }
192
+ } else if (s == BROTLI_RESULT_NEEDS_MORE_OUTPUT) {
193
+ if (!partial) {
194
+ rb_raise(eNeedsMoreOutput, "failed BrotliDecompressBuffer()");
195
+ }
196
+ } else if (s != BROTLI_RESULT_SUCCESS) {
197
+ rb_raise(eError, "failed BrotliDecompressBuffer() - status=%d", s);
198
+ }
199
+
200
+ rb_obj_infect(dest, src);
201
+ rb_str_set_len(dest, destsize);
202
+ return dest;
203
+ }
204
+
205
+ void
206
+ extbrotli_init_lowleveldecoder(void)
207
+ {
208
+ cLLDecoder = rb_define_class_under(mBrotli, "LowLevelDecoder", rb_cObject);
209
+ rb_define_singleton_method(cLLDecoder, "decode", RUBY_METHOD_FUNC(lldec_s_decode), -1);
210
+
211
+ rb_define_alloc_func(cLLDecoder, lldec_alloc);
212
+ rb_define_method(cLLDecoder, "initialize", RUBY_METHOD_FUNC(lldec_init), 0);
213
+ rb_define_method(cLLDecoder, "decode", RUBY_METHOD_FUNC(lldec_decode), 4);
214
+
215
+ rb_include_module(cLLDecoder, mConst);
216
+ rb_define_const(mConst, "RESULT_ERROR", INT2FIX(BROTLI_RESULT_ERROR));
217
+ rb_define_const(mConst, "RESULT_SUCCESS", INT2FIX(BROTLI_RESULT_SUCCESS));
218
+ rb_define_const(mConst, "RESULT_NEEDS_MORE_INPUT", INT2FIX(BROTLI_RESULT_NEEDS_MORE_INPUT));
219
+ rb_define_const(mConst, "RESULT_NEEDS_MORE_OUTPUT", INT2FIX(BROTLI_RESULT_NEEDS_MORE_OUTPUT));
220
+ }