sessionm-thrift 0.8.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. data/CHANGELOG +1 -0
  2. data/README +43 -0
  3. data/benchmark/Benchmark.thrift +24 -0
  4. data/benchmark/benchmark.rb +271 -0
  5. data/benchmark/client.rb +74 -0
  6. data/benchmark/server.rb +82 -0
  7. data/benchmark/thin_server.rb +44 -0
  8. data/ext/binary_protocol_accelerated.c +441 -0
  9. data/ext/binary_protocol_accelerated.h +20 -0
  10. data/ext/compact_protocol.c +618 -0
  11. data/ext/compact_protocol.h +20 -0
  12. data/ext/constants.h +96 -0
  13. data/ext/extconf.rb +30 -0
  14. data/ext/macros.h +41 -0
  15. data/ext/memory_buffer.c +131 -0
  16. data/ext/memory_buffer.h +20 -0
  17. data/ext/protocol.c +185 -0
  18. data/ext/protocol.h +20 -0
  19. data/ext/strlcpy.c +41 -0
  20. data/ext/strlcpy.h +30 -0
  21. data/ext/struct.c +691 -0
  22. data/ext/struct.h +25 -0
  23. data/ext/thrift_native.c +196 -0
  24. data/lib/thrift.rb +64 -0
  25. data/lib/thrift/client.rb +62 -0
  26. data/lib/thrift/core_ext.rb +23 -0
  27. data/lib/thrift/core_ext/fixnum.rb +29 -0
  28. data/lib/thrift/exceptions.rb +84 -0
  29. data/lib/thrift/processor.rb +57 -0
  30. data/lib/thrift/protocol/base_protocol.rb +290 -0
  31. data/lib/thrift/protocol/binary_protocol.rb +229 -0
  32. data/lib/thrift/protocol/binary_protocol_accelerated.rb +39 -0
  33. data/lib/thrift/protocol/compact_protocol.rb +426 -0
  34. data/lib/thrift/serializer/deserializer.rb +33 -0
  35. data/lib/thrift/serializer/serializer.rb +34 -0
  36. data/lib/thrift/server/base_server.rb +31 -0
  37. data/lib/thrift/server/mongrel_http_server.rb +58 -0
  38. data/lib/thrift/server/nonblocking_server.rb +305 -0
  39. data/lib/thrift/server/simple_server.rb +43 -0
  40. data/lib/thrift/server/thread_pool_server.rb +75 -0
  41. data/lib/thrift/server/threaded_server.rb +47 -0
  42. data/lib/thrift/struct.rb +237 -0
  43. data/lib/thrift/struct_union.rb +192 -0
  44. data/lib/thrift/thrift_native.rb +24 -0
  45. data/lib/thrift/transport/base_server_transport.rb +37 -0
  46. data/lib/thrift/transport/base_transport.rb +107 -0
  47. data/lib/thrift/transport/buffered_transport.rb +108 -0
  48. data/lib/thrift/transport/framed_transport.rb +116 -0
  49. data/lib/thrift/transport/http_client_transport.rb +51 -0
  50. data/lib/thrift/transport/io_stream_transport.rb +39 -0
  51. data/lib/thrift/transport/memory_buffer_transport.rb +125 -0
  52. data/lib/thrift/transport/server_socket.rb +63 -0
  53. data/lib/thrift/transport/socket.rb +137 -0
  54. data/lib/thrift/transport/unix_server_socket.rb +60 -0
  55. data/lib/thrift/transport/unix_socket.rb +40 -0
  56. data/lib/thrift/types.rb +101 -0
  57. data/lib/thrift/union.rb +179 -0
  58. data/spec/ThriftSpec.thrift +132 -0
  59. data/spec/base_protocol_spec.rb +160 -0
  60. data/spec/base_transport_spec.rb +351 -0
  61. data/spec/binary_protocol_accelerated_spec.rb +46 -0
  62. data/spec/binary_protocol_spec.rb +61 -0
  63. data/spec/binary_protocol_spec_shared.rb +375 -0
  64. data/spec/client_spec.rb +100 -0
  65. data/spec/compact_protocol_spec.rb +144 -0
  66. data/spec/exception_spec.rb +142 -0
  67. data/spec/http_client_spec.rb +64 -0
  68. data/spec/mongrel_http_server_spec.rb +117 -0
  69. data/spec/nonblocking_server_spec.rb +265 -0
  70. data/spec/processor_spec.rb +83 -0
  71. data/spec/serializer_spec.rb +69 -0
  72. data/spec/server_socket_spec.rb +80 -0
  73. data/spec/server_spec.rb +159 -0
  74. data/spec/socket_spec.rb +61 -0
  75. data/spec/socket_spec_shared.rb +104 -0
  76. data/spec/spec_helper.rb +58 -0
  77. data/spec/struct_spec.rb +295 -0
  78. data/spec/types_spec.rb +116 -0
  79. data/spec/union_spec.rb +193 -0
  80. data/spec/unix_socket_spec.rb +108 -0
  81. metadata +247 -0
@@ -0,0 +1,33 @@
1
+ #
2
+ # Licensed to the Apache Software Foundation (ASF) under one
3
+ # or more contributor license agreements. See the NOTICE file
4
+ # distributed with this work for additional information
5
+ # regarding copyright ownership. The ASF licenses this file
6
+ # to you under the Apache License, Version 2.0 (the
7
+ # "License"); you may not use this file except in compliance
8
+ # with the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing,
13
+ # software distributed under the License is distributed on an
14
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ # KIND, either express or implied. See the License for the
16
+ # specific language governing permissions and limitations
17
+ # under the License.
18
+ #
19
+
20
+ module Thrift
21
+ class Deserializer
22
+ def initialize(protocol_factory = BinaryProtocolFactory.new)
23
+ @transport = MemoryBufferTransport.new
24
+ @protocol = protocol_factory.get_protocol(@transport)
25
+ end
26
+
27
+ def deserialize(base, buffer)
28
+ @transport.reset_buffer(buffer)
29
+ base.read(@protocol)
30
+ base
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,34 @@
1
+ #
2
+ # Licensed to the Apache Software Foundation (ASF) under one
3
+ # or more contributor license agreements. See the NOTICE file
4
+ # distributed with this work for additional information
5
+ # regarding copyright ownership. The ASF licenses this file
6
+ # to you under the Apache License, Version 2.0 (the
7
+ # "License"); you may not use this file except in compliance
8
+ # with the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing,
13
+ # software distributed under the License is distributed on an
14
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ # KIND, either express or implied. See the License for the
16
+ # specific language governing permissions and limitations
17
+ # under the License.
18
+ #
19
+
20
+ module Thrift
21
+ class Serializer
22
+ def initialize(protocol_factory = BinaryProtocolFactory.new)
23
+ @transport = MemoryBufferTransport.new
24
+ @protocol = protocol_factory.get_protocol(@transport)
25
+ end
26
+
27
+ def serialize(base)
28
+ @transport.reset_buffer
29
+ base.write(@protocol)
30
+ @transport.read(@transport.available)
31
+ end
32
+ end
33
+ end
34
+
@@ -0,0 +1,31 @@
1
+ #
2
+ # Licensed to the Apache Software Foundation (ASF) under one
3
+ # or more contributor license agreements. See the NOTICE file
4
+ # distributed with this work for additional information
5
+ # regarding copyright ownership. The ASF licenses this file
6
+ # to you under the Apache License, Version 2.0 (the
7
+ # "License"); you may not use this file except in compliance
8
+ # with the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing,
13
+ # software distributed under the License is distributed on an
14
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ # KIND, either express or implied. See the License for the
16
+ # specific language governing permissions and limitations
17
+ # under the License.
18
+ #
19
+
20
+ module Thrift
21
+ class BaseServer
22
+ def initialize(processor, server_transport, transport_factory=nil, protocol_factory=nil)
23
+ @processor = processor
24
+ @server_transport = server_transport
25
+ @transport_factory = transport_factory ? transport_factory : Thrift::BaseTransportFactory.new
26
+ @protocol_factory = protocol_factory ? protocol_factory : Thrift::BinaryProtocolFactory.new
27
+ end
28
+
29
+ def serve; nil; end
30
+ end
31
+ end
@@ -0,0 +1,58 @@
1
+ #
2
+ # Licensed to the Apache Software Foundation (ASF) under one
3
+ # or more contributor license agreements. See the NOTICE file
4
+ # distributed with this work for additional information
5
+ # regarding copyright ownership. The ASF licenses this file
6
+ # to you under the Apache License, Version 2.0 (the
7
+ # "License"); you may not use this file except in compliance
8
+ # with the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing,
13
+ # software distributed under the License is distributed on an
14
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ # KIND, either express or implied. See the License for the
16
+ # specific language governing permissions and limitations
17
+ # under the License.
18
+ #
19
+
20
+ require 'mongrel'
21
+
22
+ ## Sticks a service on a URL, using mongrel to do the HTTP work
23
+ module Thrift
24
+ class MongrelHTTPServer < BaseServer
25
+ class Handler < Mongrel::HttpHandler
26
+ def initialize(processor, protocol_factory)
27
+ @processor = processor
28
+ @protocol_factory = protocol_factory
29
+ end
30
+
31
+ def process(request, response)
32
+ if request.params["REQUEST_METHOD"] == "POST"
33
+ response.start(200) do |head, out|
34
+ head["Content-Type"] = "application/x-thrift"
35
+ transport = IOStreamTransport.new request.body, out
36
+ protocol = @protocol_factory.get_protocol transport
37
+ @processor.process protocol, protocol
38
+ end
39
+ else
40
+ response.start(404) { }
41
+ end
42
+ end
43
+ end
44
+
45
+ def initialize(processor, opts={})
46
+ port = opts[:port] || 80
47
+ ip = opts[:ip] || "0.0.0.0"
48
+ path = opts[:path] || ""
49
+ protocol_factory = opts[:protocol_factory] || BinaryProtocolFactory.new
50
+ @server = Mongrel::HttpServer.new ip, port
51
+ @server.register "/#{path}", Handler.new(processor, protocol_factory)
52
+ end
53
+
54
+ def serve
55
+ @server.run.join
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,305 @@
1
+ #
2
+ # Licensed to the Apache Software Foundation (ASF) under one
3
+ # or more contributor license agreements. See the NOTICE file
4
+ # distributed with this work for additional information
5
+ # regarding copyright ownership. The ASF licenses this file
6
+ # to you under the Apache License, Version 2.0 (the
7
+ # "License"); you may not use this file except in compliance
8
+ # with the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing,
13
+ # software distributed under the License is distributed on an
14
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ # KIND, either express or implied. See the License for the
16
+ # specific language governing permissions and limitations
17
+ # under the License.
18
+ #
19
+
20
+ require 'logger'
21
+ require 'thread'
22
+
23
+ module Thrift
24
+ # this class expects to always use a FramedTransport for reading messages
25
+ class NonblockingServer < BaseServer
26
+ def initialize(processor, server_transport, transport_factory=nil, protocol_factory=nil, num=20, logger=nil)
27
+ super(processor, server_transport, transport_factory, protocol_factory)
28
+ @num_threads = num
29
+ if logger.nil?
30
+ @logger = Logger.new(STDERR)
31
+ @logger.level = Logger::WARN
32
+ else
33
+ @logger = logger
34
+ end
35
+ @shutdown_semaphore = Mutex.new
36
+ @transport_semaphore = Mutex.new
37
+ end
38
+
39
+ def serve
40
+ @logger.info "Starting #{self}"
41
+ @server_transport.listen
42
+ @io_manager = start_io_manager
43
+
44
+ begin
45
+ loop do
46
+ break if @server_transport.closed?
47
+ begin
48
+ rd, = select([@server_transport], nil, nil, 0.1)
49
+ rescue Errno::EBADF => e
50
+ # In Ruby 1.9, calling @server_transport.close in shutdown paths causes the select() to raise an
51
+ # Errno::EBADF. If this happens, ignore it and retry the loop.
52
+ break
53
+ end
54
+ next if rd.nil?
55
+ socket = @server_transport.accept
56
+ @logger.debug "Accepted socket: #{socket.inspect}"
57
+ @io_manager.add_connection socket
58
+ end
59
+ rescue IOError => e
60
+ end
61
+ # we must be shutting down
62
+ @logger.info "#{self} is shutting down, goodbye"
63
+ ensure
64
+ @transport_semaphore.synchronize do
65
+ @server_transport.close
66
+ end
67
+ @io_manager.ensure_closed unless @io_manager.nil?
68
+ end
69
+
70
+ def shutdown(timeout = 0, block = true)
71
+ @shutdown_semaphore.synchronize do
72
+ return if @is_shutdown
73
+ @is_shutdown = true
74
+ end
75
+ # nonblocking is intended for calling from within a Handler
76
+ # but we can't change the order of operations here, so lets thread
77
+ shutdown_proc = lambda do
78
+ @io_manager.shutdown(timeout)
79
+ @transport_semaphore.synchronize do
80
+ @server_transport.close # this will break the accept loop
81
+ end
82
+ end
83
+ if block
84
+ shutdown_proc.call
85
+ else
86
+ Thread.new &shutdown_proc
87
+ end
88
+ end
89
+
90
+ private
91
+
92
+ def start_io_manager
93
+ iom = IOManager.new(@processor, @server_transport, @transport_factory, @protocol_factory, @num_threads, @logger)
94
+ iom.spawn
95
+ iom
96
+ end
97
+
98
+ class IOManager # :nodoc:
99
+ DEFAULT_BUFFER = 2**20
100
+
101
+ def initialize(processor, server_transport, transport_factory, protocol_factory, num, logger)
102
+ @processor = processor
103
+ @server_transport = server_transport
104
+ @transport_factory = transport_factory
105
+ @protocol_factory = protocol_factory
106
+ @num_threads = num
107
+ @logger = logger
108
+ @connections = []
109
+ @buffers = Hash.new { |h,k| h[k] = '' }
110
+ @signal_queue = Queue.new
111
+ @signal_pipes = IO.pipe
112
+ @signal_pipes[1].sync = true
113
+ @worker_queue = Queue.new
114
+ @shutdown_queue = Queue.new
115
+ end
116
+
117
+ def add_connection(socket)
118
+ signal [:connection, socket]
119
+ end
120
+
121
+ def spawn
122
+ @iom_thread = Thread.new do
123
+ @logger.debug "Starting #{self}"
124
+ run
125
+ end
126
+ end
127
+
128
+ def shutdown(timeout = 0)
129
+ @logger.debug "#{self} is shutting down workers"
130
+ @worker_queue.clear
131
+ @num_threads.times { @worker_queue.push [:shutdown] }
132
+ signal [:shutdown, timeout]
133
+ @shutdown_queue.pop
134
+ @signal_pipes[0].close
135
+ @signal_pipes[1].close
136
+ @logger.debug "#{self} is shutting down, goodbye"
137
+ end
138
+
139
+ def ensure_closed
140
+ kill_worker_threads if @worker_threads
141
+ @iom_thread.kill
142
+ end
143
+
144
+ private
145
+
146
+ def run
147
+ spin_worker_threads
148
+
149
+ loop do
150
+ rd, = select([@signal_pipes[0], *@connections])
151
+ if rd.delete @signal_pipes[0]
152
+ break if read_signals == :shutdown
153
+ end
154
+ rd.each do |fd|
155
+ begin
156
+ if fd.handle.eof?
157
+ remove_connection fd
158
+ else
159
+ read_connection fd
160
+ end
161
+ rescue Errno::ECONNRESET
162
+ remove_connection fd
163
+ end
164
+ end
165
+ end
166
+ join_worker_threads(@shutdown_timeout)
167
+ ensure
168
+ @shutdown_queue.push :shutdown
169
+ end
170
+
171
+ def read_connection(fd)
172
+ @buffers[fd] << fd.read(DEFAULT_BUFFER)
173
+ while(frame = slice_frame!(@buffers[fd]))
174
+ @logger.debug "#{self} is processing a frame"
175
+ @worker_queue.push [:frame, fd, frame]
176
+ end
177
+ end
178
+
179
+ def spin_worker_threads
180
+ @logger.debug "#{self} is spinning up worker threads"
181
+ @worker_threads = []
182
+ @num_threads.times do
183
+ @worker_threads << spin_thread
184
+ end
185
+ end
186
+
187
+ def spin_thread
188
+ Worker.new(@processor, @transport_factory, @protocol_factory, @logger, @worker_queue).spawn
189
+ end
190
+
191
+ def signal(msg)
192
+ @signal_queue << msg
193
+ @signal_pipes[1].write " "
194
+ end
195
+
196
+ def read_signals
197
+ # clear the signal pipe
198
+ # note that since read_nonblock is broken in jruby,
199
+ # we can only read up to a set number of signals at once
200
+ sigstr = @signal_pipes[0].readpartial(1024)
201
+ # now read the signals
202
+ begin
203
+ sigstr.length.times do
204
+ signal, obj = @signal_queue.pop(true)
205
+ case signal
206
+ when :connection
207
+ @connections << obj
208
+ when :shutdown
209
+ @shutdown_timeout = obj
210
+ return :shutdown
211
+ end
212
+ end
213
+ rescue ThreadError
214
+ # out of signals
215
+ # note that in a perfect world this would never happen, since we're
216
+ # only reading the number of signals pushed on the pipe, but given the lack
217
+ # of locks, in theory we could clear the pipe/queue while a new signal is being
218
+ # placed on the pipe, at which point our next read_signals would hit this error
219
+ end
220
+ end
221
+
222
+ def remove_connection(fd)
223
+ # don't explicitly close it, a thread may still be writing to it
224
+ @connections.delete fd
225
+ @buffers.delete fd
226
+ end
227
+
228
+ def join_worker_threads(shutdown_timeout)
229
+ start = Time.now
230
+ @worker_threads.each do |t|
231
+ if shutdown_timeout > 0
232
+ timeout = (start + shutdown_timeout) - Time.now
233
+ break if timeout <= 0
234
+ t.join(timeout)
235
+ else
236
+ t.join
237
+ end
238
+ end
239
+ kill_worker_threads
240
+ end
241
+
242
+ def kill_worker_threads
243
+ @worker_threads.each do |t|
244
+ t.kill if t.status
245
+ end
246
+ @worker_threads.clear
247
+ end
248
+
249
+ def slice_frame!(buf)
250
+ if buf.length >= 4
251
+ size = buf.unpack('N').first
252
+ if buf.length >= size + 4
253
+ buf.slice!(0, size + 4)
254
+ else
255
+ nil
256
+ end
257
+ else
258
+ nil
259
+ end
260
+ end
261
+
262
+ class Worker # :nodoc:
263
+ def initialize(processor, transport_factory, protocol_factory, logger, queue)
264
+ @processor = processor
265
+ @transport_factory = transport_factory
266
+ @protocol_factory = protocol_factory
267
+ @logger = logger
268
+ @queue = queue
269
+ end
270
+
271
+ def spawn
272
+ Thread.new do
273
+ @logger.debug "#{self} is spawning"
274
+ run
275
+ end
276
+ end
277
+
278
+ private
279
+
280
+ def run
281
+ loop do
282
+ cmd, *args = @queue.pop
283
+ case cmd
284
+ when :shutdown
285
+ @logger.debug "#{self} is shutting down, goodbye"
286
+ break
287
+ when :frame
288
+ fd, frame = args
289
+ begin
290
+ otrans = @transport_factory.get_transport(fd)
291
+ oprot = @protocol_factory.get_protocol(otrans)
292
+ membuf = MemoryBufferTransport.new(frame)
293
+ itrans = @transport_factory.get_transport(membuf)
294
+ iprot = @protocol_factory.get_protocol(itrans)
295
+ @processor.process(iprot, oprot)
296
+ rescue => e
297
+ @logger.error "#{Thread.current.inspect} raised error: #{e.inspect}\n#{e.backtrace.join("\n")}"
298
+ end
299
+ end
300
+ end
301
+ end
302
+ end
303
+ end
304
+ end
305
+ end