fl-thrift 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. data/CHANGELOG +4 -0
  2. data/Manifest +81 -0
  3. data/README +43 -0
  4. data/Rakefile +102 -0
  5. data/benchmark/Benchmark.thrift +24 -0
  6. data/benchmark/benchmark.rb +271 -0
  7. data/benchmark/client.rb +74 -0
  8. data/benchmark/server.rb +82 -0
  9. data/benchmark/thin_server.rb +44 -0
  10. data/ext/binary_protocol_accelerated.c +474 -0
  11. data/ext/binary_protocol_accelerated.h +20 -0
  12. data/ext/compact_protocol.c +665 -0
  13. data/ext/compact_protocol.h +20 -0
  14. data/ext/constants.h +95 -0
  15. data/ext/extconf.rb +26 -0
  16. data/ext/macros.h +41 -0
  17. data/ext/memory_buffer.c +76 -0
  18. data/ext/memory_buffer.h +20 -0
  19. data/ext/protocol.c +185 -0
  20. data/ext/protocol.h +20 -0
  21. data/ext/struct.c +606 -0
  22. data/ext/struct.h +67 -0
  23. data/ext/thrift_native.c +194 -0
  24. data/lib/thrift.rb +59 -0
  25. data/lib/thrift/client.rb +62 -0
  26. data/lib/thrift/core_ext.rb +23 -0
  27. data/lib/thrift/core_ext/fixnum.rb +29 -0
  28. data/lib/thrift/exceptions.rb +82 -0
  29. data/lib/thrift/processor.rb +57 -0
  30. data/lib/thrift/protocol/base_protocol.rb +290 -0
  31. data/lib/thrift/protocol/binary_protocol.rb +225 -0
  32. data/lib/thrift/protocol/binary_protocol_accelerated.rb +35 -0
  33. data/lib/thrift/protocol/compact_protocol.rb +422 -0
  34. data/lib/thrift/serializer/deserializer.rb +33 -0
  35. data/lib/thrift/serializer/serializer.rb +34 -0
  36. data/lib/thrift/server/base_server.rb +31 -0
  37. data/lib/thrift/server/mongrel_http_server.rb +58 -0
  38. data/lib/thrift/server/nonblocking_server.rb +296 -0
  39. data/lib/thrift/server/simple_server.rb +43 -0
  40. data/lib/thrift/server/thread_pool_server.rb +75 -0
  41. data/lib/thrift/server/threaded_server.rb +47 -0
  42. data/lib/thrift/struct.rb +298 -0
  43. data/lib/thrift/thrift_native.rb +24 -0
  44. data/lib/thrift/transport/base_server_transport.rb +37 -0
  45. data/lib/thrift/transport/base_transport.rb +70 -0
  46. data/lib/thrift/transport/buffered_transport.rb +77 -0
  47. data/lib/thrift/transport/framed_transport.rb +90 -0
  48. data/lib/thrift/transport/http_client_transport.rb +45 -0
  49. data/lib/thrift/transport/io_stream_transport.rb +39 -0
  50. data/lib/thrift/transport/memory_buffer_transport.rb +96 -0
  51. data/lib/thrift/transport/server_socket.rb +63 -0
  52. data/lib/thrift/transport/socket.rb +136 -0
  53. data/lib/thrift/transport/unix_server_socket.rb +60 -0
  54. data/lib/thrift/transport/unix_socket.rb +40 -0
  55. data/lib/thrift/types.rb +101 -0
  56. data/script/proto_benchmark.rb +121 -0
  57. data/script/read_struct.rb +43 -0
  58. data/script/write_struct.rb +30 -0
  59. data/setup.rb +1585 -0
  60. data/spec/ThriftSpec.thrift +84 -0
  61. data/spec/base_protocol_spec.rb +160 -0
  62. data/spec/base_transport_spec.rb +351 -0
  63. data/spec/binary_protocol_accelerated_spec.rb +41 -0
  64. data/spec/binary_protocol_spec.rb +63 -0
  65. data/spec/binary_protocol_spec_shared.rb +375 -0
  66. data/spec/client_spec.rb +100 -0
  67. data/spec/compact_protocol_spec.rb +117 -0
  68. data/spec/exception_spec.rb +142 -0
  69. data/spec/http_client_spec.rb +49 -0
  70. data/spec/mongrel_http_server_spec.rb +117 -0
  71. data/spec/nonblocking_server_spec.rb +265 -0
  72. data/spec/processor_spec.rb +83 -0
  73. data/spec/serializer_spec.rb +69 -0
  74. data/spec/server_socket_spec.rb +80 -0
  75. data/spec/server_spec.rb +160 -0
  76. data/spec/socket_spec.rb +61 -0
  77. data/spec/socket_spec_shared.rb +104 -0
  78. data/spec/spec_helper.rb +60 -0
  79. data/spec/struct_spec.rb +252 -0
  80. data/spec/types_spec.rb +116 -0
  81. data/spec/unix_socket_spec.rb +108 -0
  82. data/thrift.gemspec +32 -0
  83. metadata +202 -0
@@ -0,0 +1,35 @@
1
+ #
2
+ # Licensed to the Apache Software Foundation (ASF) under one
3
+ # or more contributor license agreements. See the NOTICE file
4
+ # distributed with this work for additional information
5
+ # regarding copyright ownership. The ASF licenses this file
6
+ # to you under the Apache License, Version 2.0 (the
7
+ # "License"); you may not use this file except in compliance
8
+ # with the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing,
13
+ # software distributed under the License is distributed on an
14
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ # KIND, either express or implied. See the License for the
16
+ # specific language governing permissions and limitations
17
+ # under the License.
18
+ #
19
+
20
+ =begin
21
+ The only change required for a transport to support BinaryProtocolAccelerated is to implement 2 methods:
22
+ * borrow(size), which takes an optional argument and returns atleast _size_ bytes from the transport,
23
+ or the default buffer size if no argument is given
24
+ * consume!(size), which removes size bytes from the front of the buffer
25
+
26
+ See MemoryBuffer and BufferedTransport for examples.
27
+ =end
28
+
29
+ module Thrift
30
+ class BinaryProtocolAcceleratedFactory < BaseProtocolFactory
31
+ def get_protocol(trans)
32
+ BinaryProtocolAccelerated.new(trans)
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,422 @@
1
+ #
2
+ # Licensed to the Apache Software Foundation (ASF) under one
3
+ # or more contributor license agreements. See the NOTICE file
4
+ # distributed with this work for additional information
5
+ # regarding copyright ownership. The ASF licenses this file
6
+ # to you under the Apache License, Version 2.0 (the
7
+ # "License"); you may not use this file except in compliance
8
+ # with the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing,
13
+ # software distributed under the License is distributed on an
14
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ # KIND, either express or implied. See the License for the
16
+ # specific language governing permissions and limitations
17
+ # under the License.
18
+ #
19
+
20
+ module Thrift
21
+ class CompactProtocol < BaseProtocol
22
+
23
+ PROTOCOL_ID = [0x82].pack('c').unpack('c').first
24
+ VERSION = 1
25
+ VERSION_MASK = 0x1f
26
+ TYPE_MASK = 0xE0
27
+ TYPE_SHIFT_AMOUNT = 5
28
+
29
+ TSTOP = ["", Types::STOP, 0]
30
+
31
+ #
32
+ # All of the on-wire type codes.
33
+ #
34
+ class CompactTypes
35
+ BOOLEAN_TRUE = 0x01
36
+ BOOLEAN_FALSE = 0x02
37
+ BYTE = 0x03
38
+ I16 = 0x04
39
+ I32 = 0x05
40
+ I64 = 0x06
41
+ DOUBLE = 0x07
42
+ BINARY = 0x08
43
+ LIST = 0x09
44
+ SET = 0x0A
45
+ MAP = 0x0B
46
+ STRUCT = 0x0C
47
+
48
+ def self.is_bool_type?(b)
49
+ (b & 0x0f) == BOOLEAN_TRUE || (b & 0x0f) == BOOLEAN_FALSE
50
+ end
51
+
52
+ COMPACT_TO_TTYPE = {
53
+ Types::STOP => Types::STOP,
54
+ BOOLEAN_FALSE => Types::BOOL,
55
+ BOOLEAN_TRUE => Types::BOOL,
56
+ BYTE => Types::BYTE,
57
+ I16 => Types::I16,
58
+ I32 => Types::I32,
59
+ I64 => Types::I64,
60
+ DOUBLE => Types::DOUBLE,
61
+ BINARY => Types::STRING,
62
+ LIST => Types::LIST,
63
+ SET => Types::SET,
64
+ MAP => Types::MAP,
65
+ STRUCT => Types::STRUCT
66
+ }
67
+
68
+ TTYPE_TO_COMPACT = {
69
+ Types::STOP => Types::STOP,
70
+ Types::BOOL => BOOLEAN_TRUE,
71
+ Types::BYTE => BYTE,
72
+ Types::I16 => I16,
73
+ Types::I32 => I32,
74
+ Types::I64 => I64,
75
+ Types::DOUBLE => DOUBLE,
76
+ Types::STRING => BINARY,
77
+ Types::LIST => LIST,
78
+ Types::SET => SET,
79
+ Types::MAP => MAP,
80
+ Types::STRUCT => STRUCT
81
+ }
82
+
83
+ def self.get_ttype(compact_type)
84
+ val = COMPACT_TO_TTYPE[compact_type & 0x0f]
85
+ raise "don't know what type: #{compact_type & 0x0f}" unless val
86
+ val
87
+ end
88
+
89
+ def self.get_compact_type(ttype)
90
+ val = TTYPE_TO_COMPACT[ttype]
91
+ raise "don't know what type: #{ttype & 0x0f}" unless val
92
+ val
93
+ end
94
+ end
95
+
96
+ def initialize(transport)
97
+ super(transport)
98
+
99
+ @last_field = [0]
100
+ @boolean_value = nil
101
+ end
102
+
103
+ def write_message_begin(name, type, seqid)
104
+ write_byte(PROTOCOL_ID)
105
+ write_byte((VERSION & VERSION_MASK) | ((type << TYPE_SHIFT_AMOUNT) & TYPE_MASK))
106
+ write_varint32(seqid)
107
+ write_string(name)
108
+ nil
109
+ end
110
+
111
+ def write_struct_begin(name)
112
+ @last_field.push(0)
113
+ nil
114
+ end
115
+
116
+ def write_struct_end
117
+ @last_field.pop
118
+ nil
119
+ end
120
+
121
+ def write_field_begin(name, type, id)
122
+ if type == Types::BOOL
123
+ # we want to possibly include the value, so we'll wait.
124
+ @boolean_field = [type, id]
125
+ else
126
+ write_field_begin_internal(type, id)
127
+ end
128
+ nil
129
+ end
130
+
131
+ #
132
+ # The workhorse of writeFieldBegin. It has the option of doing a
133
+ # 'type override' of the type header. This is used specifically in the
134
+ # boolean field case.
135
+ #
136
+ def write_field_begin_internal(type, id, type_override=nil)
137
+ last_id = @last_field.pop
138
+
139
+ # if there's a type override, use that.
140
+ typeToWrite = type_override || CompactTypes.get_compact_type(type)
141
+
142
+ # check if we can use delta encoding for the field id
143
+ if id > last_id && id - last_id <= 15
144
+ # write them together
145
+ write_byte((id - last_id) << 4 | typeToWrite)
146
+ else
147
+ # write them separate
148
+ write_byte(typeToWrite)
149
+ write_i16(id)
150
+ end
151
+
152
+ @last_field.push(id)
153
+ nil
154
+ end
155
+
156
+ def write_field_stop
157
+ write_byte(Types::STOP)
158
+ end
159
+
160
+ def write_map_begin(ktype, vtype, size)
161
+ if (size == 0)
162
+ write_byte(0)
163
+ else
164
+ write_varint32(size)
165
+ write_byte(CompactTypes.get_compact_type(ktype) << 4 | CompactTypes.get_compact_type(vtype))
166
+ end
167
+ end
168
+
169
+ def write_list_begin(etype, size)
170
+ write_collection_begin(etype, size)
171
+ end
172
+
173
+ def write_set_begin(etype, size)
174
+ write_collection_begin(etype, size);
175
+ end
176
+
177
+ def write_bool(bool)
178
+ type = bool ? CompactTypes::BOOLEAN_TRUE : CompactTypes::BOOLEAN_FALSE
179
+ unless @boolean_field.nil?
180
+ # we haven't written the field header yet
181
+ write_field_begin_internal(@boolean_field.first, @boolean_field.last, type)
182
+ @boolean_field = nil
183
+ else
184
+ # we're not part of a field, so just write the value.
185
+ write_byte(type)
186
+ end
187
+ end
188
+
189
+ def write_byte(byte)
190
+ @trans.write([byte].pack('c'))
191
+ end
192
+
193
+ def write_i16(i16)
194
+ write_varint32(int_to_zig_zag(i16))
195
+ end
196
+
197
+ def write_i32(i32)
198
+ write_varint32(int_to_zig_zag(i32))
199
+ end
200
+
201
+ def write_i64(i64)
202
+ write_varint64(long_to_zig_zag(i64))
203
+ end
204
+
205
+ def write_double(dub)
206
+ @trans.write([dub].pack("G").reverse)
207
+ end
208
+
209
+ def write_string(str)
210
+ write_varint32(str.length)
211
+ @trans.write(str)
212
+ end
213
+
214
+ def read_message_begin
215
+ protocol_id = read_byte()
216
+ if protocol_id != PROTOCOL_ID
217
+ raise ProtocolException.new("Expected protocol id #{PROTOCOL_ID} but got #{protocol_id}")
218
+ end
219
+
220
+ version_and_type = read_byte()
221
+ version = version_and_type & VERSION_MASK
222
+ if (version != VERSION)
223
+ raise ProtocolException.new("Expected version #{VERSION} but got #{version}");
224
+ end
225
+
226
+ type = (version_and_type >> TYPE_SHIFT_AMOUNT) & 0x03
227
+ seqid = read_varint32()
228
+ messageName = read_string()
229
+ [messageName, type, seqid]
230
+ end
231
+
232
+ def read_struct_begin
233
+ @last_field.push(0)
234
+ ""
235
+ end
236
+
237
+ def read_struct_end
238
+ @last_field.pop()
239
+ nil
240
+ end
241
+
242
+ def read_field_begin
243
+ type = read_byte()
244
+
245
+ # if it's a stop, then we can return immediately, as the struct is over.
246
+ if (type & 0x0f) == Types::STOP
247
+ TSTOP
248
+ else
249
+ field_id = nil
250
+
251
+ # mask off the 4 MSB of the type header. it could contain a field id delta.
252
+ modifier = (type & 0xf0) >> 4
253
+ if modifier == 0
254
+ # not a delta. look ahead for the zigzag varint field id.
255
+ field_id = read_i16()
256
+ else
257
+ # has a delta. add the delta to the last read field id.
258
+ field_id = @last_field.pop + modifier
259
+ end
260
+
261
+ # if this happens to be a boolean field, the value is encoded in the type
262
+ if CompactTypes.is_bool_type?(type)
263
+ # save the boolean value in a special instance variable.
264
+ @bool_value = (type & 0x0f) == CompactTypes::BOOLEAN_TRUE
265
+ end
266
+
267
+ # push the new field onto the field stack so we can keep the deltas going.
268
+ @last_field.push(field_id)
269
+ ["", CompactTypes.get_ttype(type & 0x0f), field_id]
270
+ end
271
+ end
272
+
273
+ def read_map_begin
274
+ size = read_varint32()
275
+ key_and_value_type = size == 0 ? 0 : read_byte()
276
+ [CompactTypes.get_ttype(key_and_value_type >> 4), CompactTypes.get_ttype(key_and_value_type & 0xf), size]
277
+ end
278
+
279
+ def read_list_begin
280
+ size_and_type = read_byte()
281
+ size = (size_and_type >> 4) & 0x0f
282
+ if size == 15
283
+ size = read_varint32()
284
+ end
285
+ type = CompactTypes.get_ttype(size_and_type)
286
+ [type, size]
287
+ end
288
+
289
+ def read_set_begin
290
+ read_list_begin
291
+ end
292
+
293
+ def read_bool
294
+ unless @bool_value.nil?
295
+ bv = @bool_value
296
+ @bool_value = nil
297
+ bv
298
+ else
299
+ read_byte() == CompactTypes::BOOLEAN_TRUE
300
+ end
301
+ end
302
+
303
+ def read_byte
304
+ dat = trans.read_all(1)
305
+ val = dat[0]
306
+ if (val > 0x7f)
307
+ val = 0 - ((val - 1) ^ 0xff)
308
+ end
309
+ val
310
+ end
311
+
312
+ def read_i16
313
+ zig_zag_to_int(read_varint32())
314
+ end
315
+
316
+ def read_i32
317
+ zig_zag_to_int(read_varint32())
318
+ end
319
+
320
+ def read_i64
321
+ zig_zag_to_long(read_varint64())
322
+ end
323
+
324
+ def read_double
325
+ dat = trans.read_all(8)
326
+ val = dat.reverse.unpack('G').first
327
+ val
328
+ end
329
+
330
+ def read_string
331
+ size = read_varint32()
332
+ trans.read_all(size)
333
+ end
334
+
335
+
336
+ private
337
+
338
+ #
339
+ # Abstract method for writing the start of lists and sets. List and sets on
340
+ # the wire differ only by the type indicator.
341
+ #
342
+ def write_collection_begin(elem_type, size)
343
+ if size <= 14
344
+ write_byte(size << 4 | CompactTypes.get_compact_type(elem_type))
345
+ else
346
+ write_byte(0xf0 | CompactTypes.get_compact_type(elem_type))
347
+ write_varint32(size)
348
+ end
349
+ end
350
+
351
+ def write_varint32(n)
352
+ # int idx = 0;
353
+ while true
354
+ if (n & ~0x7F) == 0
355
+ # i32buf[idx++] = (byte)n;
356
+ write_byte(n)
357
+ break
358
+ # return;
359
+ else
360
+ # i32buf[idx++] = (byte)((n & 0x7F) | 0x80);
361
+ write_byte((n & 0x7F) | 0x80)
362
+ n = n >> 7
363
+ end
364
+ end
365
+ # trans_.write(i32buf, 0, idx);
366
+ end
367
+
368
+ SEVEN_BIT_MASK = 0x7F
369
+ EVERYTHING_ELSE_MASK = ~SEVEN_BIT_MASK
370
+
371
+ def write_varint64(n)
372
+ while true
373
+ if (n & EVERYTHING_ELSE_MASK) == 0 #TODO need to find a way to make this into a long...
374
+ write_byte(n)
375
+ break
376
+ else
377
+ write_byte((n & SEVEN_BIT_MASK) | 0x80)
378
+ n >>= 7
379
+ end
380
+ end
381
+ end
382
+
383
+ def read_varint32()
384
+ read_varint64()
385
+ end
386
+
387
+ def read_varint64()
388
+ shift = 0
389
+ result = 0
390
+ while true
391
+ b = read_byte()
392
+ result |= (b & 0x7f) << shift
393
+ break if (b & 0x80) != 0x80
394
+ shift += 7
395
+ end
396
+ result
397
+ end
398
+
399
+ def int_to_zig_zag(n)
400
+ (n << 1) ^ (n >> 31)
401
+ end
402
+
403
+ def long_to_zig_zag(l)
404
+ # puts "zz encoded #{l} to #{(l << 1) ^ (l >> 63)}"
405
+ (l << 1) ^ (l >> 63)
406
+ end
407
+
408
+ def zig_zag_to_int(n)
409
+ (n >> 1) ^ -(n & 1)
410
+ end
411
+
412
+ def zig_zag_to_long(n)
413
+ (n >> 1) ^ -(n & 1)
414
+ end
415
+ end
416
+
417
+ class CompactProtocolFactory < BaseProtocolFactory
418
+ def get_protocol(trans)
419
+ CompactProtocol.new(trans)
420
+ end
421
+ end
422
+ end