jml_thrift 1.0.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (89) hide show
  1. checksums.yaml +7 -0
  2. data/README +43 -0
  3. data/benchmark/Benchmark.thrift +24 -0
  4. data/benchmark/benchmark.rb +271 -0
  5. data/benchmark/client.rb +74 -0
  6. data/benchmark/server.rb +82 -0
  7. data/benchmark/thin_server.rb +44 -0
  8. data/ext/binary_protocol_accelerated.c +460 -0
  9. data/ext/binary_protocol_accelerated.h +20 -0
  10. data/ext/bytes.c +36 -0
  11. data/ext/bytes.h +31 -0
  12. data/ext/compact_protocol.c +635 -0
  13. data/ext/compact_protocol.h +20 -0
  14. data/ext/constants.h +96 -0
  15. data/ext/extconf.rb +32 -0
  16. data/ext/macros.h +41 -0
  17. data/ext/memory_buffer.c +134 -0
  18. data/ext/memory_buffer.h +20 -0
  19. data/ext/protocol.c +0 -0
  20. data/ext/protocol.h +0 -0
  21. data/ext/strlcpy.c +41 -0
  22. data/ext/strlcpy.h +34 -0
  23. data/ext/struct.c +688 -0
  24. data/ext/struct.h +25 -0
  25. data/ext/thrift_native.c +195 -0
  26. data/lib/thrift.rb +66 -0
  27. data/lib/thrift/bytes.rb +131 -0
  28. data/lib/thrift/client.rb +62 -0
  29. data/lib/thrift/core_ext.rb +23 -0
  30. data/lib/thrift/core_ext/fixnum.rb +29 -0
  31. data/lib/thrift/exceptions.rb +87 -0
  32. data/lib/thrift/processor.rb +57 -0
  33. data/lib/thrift/protocol/base_protocol.rb +377 -0
  34. data/lib/thrift/protocol/binary_protocol.rb +237 -0
  35. data/lib/thrift/protocol/binary_protocol_accelerated.rb +39 -0
  36. data/lib/thrift/protocol/compact_protocol.rb +434 -0
  37. data/lib/thrift/protocol/json_protocol.rb +769 -0
  38. data/lib/thrift/serializer/deserializer.rb +33 -0
  39. data/lib/thrift/serializer/serializer.rb +34 -0
  40. data/lib/thrift/server/base_server.rb +31 -0
  41. data/lib/thrift/server/mongrel_http_server.rb +60 -0
  42. data/lib/thrift/server/nonblocking_server.rb +305 -0
  43. data/lib/thrift/server/simple_server.rb +43 -0
  44. data/lib/thrift/server/thin_http_server.rb +91 -0
  45. data/lib/thrift/server/thread_pool_server.rb +75 -0
  46. data/lib/thrift/server/threaded_server.rb +47 -0
  47. data/lib/thrift/struct.rb +237 -0
  48. data/lib/thrift/struct_union.rb +192 -0
  49. data/lib/thrift/thrift_native.rb +24 -0
  50. data/lib/thrift/transport/base_server_transport.rb +37 -0
  51. data/lib/thrift/transport/base_transport.rb +109 -0
  52. data/lib/thrift/transport/buffered_transport.rb +114 -0
  53. data/lib/thrift/transport/framed_transport.rb +117 -0
  54. data/lib/thrift/transport/http_client_transport.rb +56 -0
  55. data/lib/thrift/transport/io_stream_transport.rb +39 -0
  56. data/lib/thrift/transport/memory_buffer_transport.rb +125 -0
  57. data/lib/thrift/transport/server_socket.rb +63 -0
  58. data/lib/thrift/transport/socket.rb +139 -0
  59. data/lib/thrift/transport/unix_server_socket.rb +60 -0
  60. data/lib/thrift/transport/unix_socket.rb +40 -0
  61. data/lib/thrift/types.rb +101 -0
  62. data/lib/thrift/union.rb +179 -0
  63. data/spec/ThriftSpec.thrift +183 -0
  64. data/spec/base_protocol_spec.rb +217 -0
  65. data/spec/base_transport_spec.rb +350 -0
  66. data/spec/binary_protocol_accelerated_spec.rb +42 -0
  67. data/spec/binary_protocol_spec.rb +66 -0
  68. data/spec/binary_protocol_spec_shared.rb +455 -0
  69. data/spec/bytes_spec.rb +160 -0
  70. data/spec/client_spec.rb +99 -0
  71. data/spec/compact_protocol_spec.rb +143 -0
  72. data/spec/exception_spec.rb +141 -0
  73. data/spec/http_client_spec.rb +120 -0
  74. data/spec/json_protocol_spec.rb +513 -0
  75. data/spec/nonblocking_server_spec.rb +263 -0
  76. data/spec/processor_spec.rb +80 -0
  77. data/spec/serializer_spec.rb +67 -0
  78. data/spec/server_socket_spec.rb +79 -0
  79. data/spec/server_spec.rb +147 -0
  80. data/spec/socket_spec.rb +61 -0
  81. data/spec/socket_spec_shared.rb +104 -0
  82. data/spec/spec_helper.rb +61 -0
  83. data/spec/struct_nested_containers_spec.rb +191 -0
  84. data/spec/struct_spec.rb +293 -0
  85. data/spec/thin_http_server_spec.rb +141 -0
  86. data/spec/types_spec.rb +115 -0
  87. data/spec/union_spec.rb +203 -0
  88. data/spec/unix_socket_spec.rb +107 -0
  89. metadata +310 -0
@@ -0,0 +1,117 @@
1
+ # encoding: ascii-8bit
2
+ #
3
+ # Licensed to the Apache Software Foundation (ASF) under one
4
+ # or more contributor license agreements. See the NOTICE file
5
+ # distributed with this work for additional information
6
+ # regarding copyright ownership. The ASF licenses this file
7
+ # to you under the Apache License, Version 2.0 (the
8
+ # "License"); you may not use this file except in compliance
9
+ # with the License. You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing,
14
+ # software distributed under the License is distributed on an
15
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16
+ # KIND, either express or implied. See the License for the
17
+ # specific language governing permissions and limitations
18
+ # under the License.
19
+ #
20
+
21
+ module Thrift
22
+ class FramedTransport < BaseTransport
23
+ def initialize(transport, read=true, write=true)
24
+ @transport = transport
25
+ @rbuf = Bytes.empty_byte_buffer
26
+ @wbuf = Bytes.empty_byte_buffer
27
+ @read = read
28
+ @write = write
29
+ @index = 0
30
+ end
31
+
32
+ def open?
33
+ @transport.open?
34
+ end
35
+
36
+ def open
37
+ @transport.open
38
+ end
39
+
40
+ def close
41
+ @transport.close
42
+ end
43
+
44
+ def read(sz)
45
+ return @transport.read(sz) unless @read
46
+
47
+ return Bytes.empty_byte_buffer if sz <= 0
48
+
49
+ read_frame if @index >= @rbuf.length
50
+
51
+ @index += sz
52
+ @rbuf.slice(@index - sz, sz) || Bytes.empty_byte_buffer
53
+ end
54
+
55
+ def read_byte
56
+ return @transport.read_byte() unless @read
57
+
58
+ read_frame if @index >= @rbuf.length
59
+
60
+ # The read buffer has some data now, read a single byte. Using get_string_byte() avoids
61
+ # allocating a temp string of size 1 unnecessarily.
62
+ @index += 1
63
+ return Bytes.get_string_byte(@rbuf, @index - 1)
64
+ end
65
+
66
+ def read_into_buffer(buffer, size)
67
+ i = 0
68
+ while i < size
69
+ read_frame if @index >= @rbuf.length
70
+
71
+ # The read buffer has some data now, so copy bytes over to the output buffer.
72
+ byte = Bytes.get_string_byte(@rbuf, @index)
73
+ Bytes.set_string_byte(buffer, i, byte)
74
+ @index += 1
75
+ i += 1
76
+ end
77
+ i
78
+ end
79
+
80
+ def write(buf, sz=nil)
81
+ return @transport.write(buf) unless @write
82
+
83
+ buf = Bytes.force_binary_encoding(buf)
84
+ @wbuf << (sz ? buf[0...sz] : buf)
85
+ end
86
+
87
+ #
88
+ # Writes the output buffer to the stream in the format of a 4-byte length
89
+ # followed by the actual data.
90
+ #
91
+ def flush
92
+ return @transport.flush unless @write
93
+
94
+ out = [@wbuf.length].pack('N')
95
+ # Array#pack should return a BINARY encoded String, so it shouldn't be necessary to force encoding
96
+ out << @wbuf
97
+ @transport.write(out)
98
+ @transport.flush
99
+ @wbuf = Bytes.empty_byte_buffer
100
+ end
101
+
102
+ private
103
+
104
+ def read_frame
105
+ sz = @transport.read_all(4).unpack('N').first
106
+
107
+ @index = 0
108
+ @rbuf = @transport.read_all(sz)
109
+ end
110
+ end
111
+
112
+ class FramedTransportFactory < BaseTransportFactory
113
+ def get_transport(transport)
114
+ return FramedTransport.new(transport)
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,56 @@
1
+ # encoding: ascii-8bit
2
+ #
3
+ # Licensed to the Apache Software Foundation (ASF) under one
4
+ # or more contributor license agreements. See the NOTICE file
5
+ # distributed with this work for additional information
6
+ # regarding copyright ownership. The ASF licenses this file
7
+ # to you under the Apache License, Version 2.0 (the
8
+ # "License"); you may not use this file except in compliance
9
+ # with the License. You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing,
14
+ # software distributed under the License is distributed on an
15
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16
+ # KIND, either express or implied. See the License for the
17
+ # specific language governing permissions and limitations
18
+ # under the License.
19
+ #
20
+
21
+ require 'net/http'
22
+ require 'net/https'
23
+ require 'openssl'
24
+ require 'uri'
25
+ require 'stringio'
26
+
27
+ module Thrift
28
+ class HTTPClientTransport < BaseTransport
29
+
30
+ def initialize(url, opts = {})
31
+ @url = URI url
32
+ @headers = {'Content-Type' => 'application/x-thrift'}
33
+ @outbuf = Bytes.empty_byte_buffer
34
+ @ssl_verify_mode = opts.fetch(:ssl_verify_mode, OpenSSL::SSL::VERIFY_PEER)
35
+ end
36
+
37
+ def open?; true end
38
+ def read(sz); @inbuf.read sz end
39
+ def write(buf); @outbuf << Bytes.force_binary_encoding(buf) end
40
+
41
+ def add_headers(headers)
42
+ @headers = @headers.merge(headers)
43
+ end
44
+
45
+ def flush
46
+ http = Net::HTTP.new @url.host, @url.port
47
+ http.use_ssl = @url.scheme == 'https'
48
+ http.verify_mode = @ssl_verify_mode if @url.scheme == 'https'
49
+ resp = http.post(@url.request_uri, @outbuf, @headers)
50
+ data = resp.body
51
+ data = Bytes.force_binary_encoding(data)
52
+ @inbuf = StringIO.new data
53
+ @outbuf = Bytes.empty_byte_buffer
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,39 @@
1
+ # encoding: ascii-8bit
2
+ #
3
+ # Licensed to the Apache Software Foundation (ASF) under one
4
+ # or more contributor license agreements. See the NOTICE file
5
+ # distributed with this work for additional information
6
+ # regarding copyright ownership. The ASF licenses this file
7
+ # to you under the Apache License, Version 2.0 (the
8
+ # "License"); you may not use this file except in compliance
9
+ # with the License. You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing,
14
+ # software distributed under the License is distributed on an
15
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16
+ # KIND, either express or implied. See the License for the
17
+ # specific language governing permissions and limitations
18
+ # under the License.
19
+ #
20
+
21
+ # Very very simple implementation of wrapping two objects, one with a #read
22
+ # method and one with a #write method, into a transport for thrift.
23
+ #
24
+ # Assumes both objects are open, remain open, don't require flushing, etc.
25
+ #
26
+ module Thrift
27
+ class IOStreamTransport < BaseTransport
28
+ def initialize(input, output)
29
+ @input = input
30
+ @output = output
31
+ end
32
+
33
+ def open?; not @input.closed? or not @output.closed? end
34
+ def read(sz); @input.read(sz) end
35
+ def write(buf); @output.write(Bytes.force_binary_encoding(buf)) end
36
+ def close; @input.close; @output.close end
37
+ def to_io; @input end # we're assuming this is used in a IO.select for reading
38
+ end
39
+ end
@@ -0,0 +1,125 @@
1
+ # encoding: ascii-8bit
2
+ #
3
+ # Licensed to the Apache Software Foundation (ASF) under one
4
+ # or more contributor license agreements. See the NOTICE file
5
+ # distributed with this work for additional information
6
+ # regarding copyright ownership. The ASF licenses this file
7
+ # to you under the Apache License, Version 2.0 (the
8
+ # "License"); you may not use this file except in compliance
9
+ # with the License. You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing,
14
+ # software distributed under the License is distributed on an
15
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16
+ # KIND, either express or implied. See the License for the
17
+ # specific language governing permissions and limitations
18
+ # under the License.
19
+ #
20
+
21
+ module Thrift
22
+ class MemoryBufferTransport < BaseTransport
23
+ GARBAGE_BUFFER_SIZE = 4*(2**10) # 4kB
24
+
25
+ # If you pass a string to this, you should #dup that string
26
+ # unless you want it to be modified by #read and #write
27
+ #--
28
+ # this behavior is no longer required. If you wish to change it
29
+ # go ahead, just make sure the specs pass
30
+ def initialize(buffer = nil)
31
+ @buf = buffer ? Bytes.force_binary_encoding(buffer) : Bytes.empty_byte_buffer
32
+ @index = 0
33
+ end
34
+
35
+ def open?
36
+ return true
37
+ end
38
+
39
+ def open
40
+ end
41
+
42
+ def close
43
+ end
44
+
45
+ def peek
46
+ @index < @buf.size
47
+ end
48
+
49
+ # this method does not use the passed object directly but copies it
50
+ def reset_buffer(new_buf = '')
51
+ @buf.replace Bytes.force_binary_encoding(new_buf)
52
+ @index = 0
53
+ end
54
+
55
+ def available
56
+ @buf.length - @index
57
+ end
58
+
59
+ def read(len)
60
+ data = @buf.slice(@index, len)
61
+ @index += len
62
+ @index = @buf.size if @index > @buf.size
63
+ if @index >= GARBAGE_BUFFER_SIZE
64
+ @buf = @buf.slice(@index..-1)
65
+ @index = 0
66
+ end
67
+ if data.size < len
68
+ raise EOFError, "Not enough bytes remain in buffer"
69
+ end
70
+ data
71
+ end
72
+
73
+ def read_byte
74
+ raise EOFError.new("Not enough bytes remain in buffer") if @index >= @buf.size
75
+ val = Bytes.get_string_byte(@buf, @index)
76
+ @index += 1
77
+ if @index >= GARBAGE_BUFFER_SIZE
78
+ @buf = @buf.slice(@index..-1)
79
+ @index = 0
80
+ end
81
+ val
82
+ end
83
+
84
+ def read_into_buffer(buffer, size)
85
+ i = 0
86
+ while i < size
87
+ raise EOFError.new("Not enough bytes remain in buffer") if @index >= @buf.size
88
+
89
+ # The read buffer has some data now, so copy bytes over to the output buffer.
90
+ byte = Bytes.get_string_byte(@buf, @index)
91
+ Bytes.set_string_byte(buffer, i, byte)
92
+ @index += 1
93
+ i += 1
94
+ end
95
+ if @index >= GARBAGE_BUFFER_SIZE
96
+ @buf = @buf.slice(@index..-1)
97
+ @index = 0
98
+ end
99
+ i
100
+ end
101
+
102
+ def write(wbuf)
103
+ @buf << Bytes.force_binary_encoding(wbuf)
104
+ end
105
+
106
+ def flush
107
+ end
108
+
109
+ def inspect_buffer
110
+ out = []
111
+ for idx in 0...(@buf.size)
112
+ # if idx != 0
113
+ # out << " "
114
+ # end
115
+
116
+ if idx == @index
117
+ out << ">"
118
+ end
119
+
120
+ out << @buf[idx].ord.to_s(16)
121
+ end
122
+ out.join(" ")
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,63 @@
1
+ # encoding: ascii-8bit
2
+ #
3
+ # Licensed to the Apache Software Foundation (ASF) under one
4
+ # or more contributor license agreements. See the NOTICE file
5
+ # distributed with this work for additional information
6
+ # regarding copyright ownership. The ASF licenses this file
7
+ # to you under the Apache License, Version 2.0 (the
8
+ # "License"); you may not use this file except in compliance
9
+ # with the License. You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing,
14
+ # software distributed under the License is distributed on an
15
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16
+ # KIND, either express or implied. See the License for the
17
+ # specific language governing permissions and limitations
18
+ # under the License.
19
+ #
20
+
21
+ require 'socket'
22
+
23
+ module Thrift
24
+ class ServerSocket < BaseServerTransport
25
+ # call-seq: initialize(host = nil, port)
26
+ def initialize(host_or_port, port = nil)
27
+ if port
28
+ @host = host_or_port
29
+ @port = port
30
+ else
31
+ @host = nil
32
+ @port = host_or_port
33
+ end
34
+ @handle = nil
35
+ end
36
+
37
+ attr_reader :handle
38
+
39
+ def listen
40
+ @handle = TCPServer.new(@host, @port)
41
+ end
42
+
43
+ def accept
44
+ unless @handle.nil?
45
+ sock = @handle.accept
46
+ trans = Socket.new
47
+ trans.handle = sock
48
+ trans
49
+ end
50
+ end
51
+
52
+ def close
53
+ @handle.close unless @handle.nil? or @handle.closed?
54
+ @handle = nil
55
+ end
56
+
57
+ def closed?
58
+ @handle.nil? or @handle.closed?
59
+ end
60
+
61
+ alias to_io handle
62
+ end
63
+ end
@@ -0,0 +1,139 @@
1
+ # encoding: ascii-8bit
2
+ #
3
+ # Licensed to the Apache Software Foundation (ASF) under one
4
+ # or more contributor license agreements. See the NOTICE file
5
+ # distributed with this work for additional information
6
+ # regarding copyright ownership. The ASF licenses this file
7
+ # to you under the Apache License, Version 2.0 (the
8
+ # "License"); you may not use this file except in compliance
9
+ # with the License. You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing,
14
+ # software distributed under the License is distributed on an
15
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16
+ # KIND, either express or implied. See the License for the
17
+ # specific language governing permissions and limitations
18
+ # under the License.
19
+ #
20
+
21
+ require 'socket'
22
+
23
+ module Thrift
24
+ class Socket < BaseTransport
25
+ def initialize(host='localhost', port=9090, timeout=nil)
26
+ @host = host
27
+ @port = port
28
+ @timeout = timeout
29
+ @desc = "#{host}:#{port}"
30
+ @handle = nil
31
+ end
32
+
33
+ attr_accessor :handle, :timeout
34
+
35
+ def open
36
+ begin
37
+ addrinfo = ::Socket::getaddrinfo(@host, @port, nil, ::Socket::SOCK_STREAM).first
38
+ @handle = ::Socket.new(addrinfo[4], ::Socket::SOCK_STREAM, 0)
39
+ @handle.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1)
40
+ sockaddr = ::Socket.sockaddr_in(addrinfo[1], addrinfo[3])
41
+ begin
42
+ @handle.connect_nonblock(sockaddr)
43
+ rescue Errno::EINPROGRESS
44
+ unless IO.select(nil, [ @handle ], nil, @timeout)
45
+ raise TransportException.new(TransportException::NOT_OPEN, "Connection timeout to #{@desc}")
46
+ end
47
+ begin
48
+ @handle.connect_nonblock(sockaddr)
49
+ rescue Errno::EISCONN
50
+ end
51
+ end
52
+ @handle
53
+ rescue StandardError => e
54
+ raise TransportException.new(TransportException::NOT_OPEN, "Could not connect to #{@desc}: #{e}")
55
+ end
56
+ end
57
+
58
+ def open?
59
+ !@handle.nil? and !@handle.closed?
60
+ end
61
+
62
+ def write(str)
63
+ raise IOError, "closed stream" unless open?
64
+ str = Bytes.force_binary_encoding(str)
65
+ begin
66
+ if @timeout.nil? or @timeout == 0
67
+ @handle.write(str)
68
+ else
69
+ len = 0
70
+ start = Time.now
71
+ while Time.now - start < @timeout
72
+ rd, wr, = IO.select(nil, [@handle], nil, @timeout)
73
+ if wr and not wr.empty?
74
+ len += @handle.write_nonblock(str[len..-1])
75
+ break if len >= str.length
76
+ end
77
+ end
78
+ if len < str.length
79
+ raise TransportException.new(TransportException::TIMED_OUT, "Socket: Timed out writing #{str.length} bytes to #{@desc}")
80
+ else
81
+ len
82
+ end
83
+ end
84
+ rescue TransportException => e
85
+ # pass this on
86
+ raise e
87
+ rescue StandardError => e
88
+ @handle.close
89
+ @handle = nil
90
+ raise TransportException.new(TransportException::NOT_OPEN, e.message)
91
+ end
92
+ end
93
+
94
+ def read(sz)
95
+ raise IOError, "closed stream" unless open?
96
+
97
+ begin
98
+ if @timeout.nil? or @timeout == 0
99
+ data = @handle.readpartial(sz)
100
+ else
101
+ # it's possible to interrupt select for something other than the timeout
102
+ # so we need to ensure we've waited long enough, but not too long
103
+ start = Time.now
104
+ timespent = 0
105
+ rd = loop do
106
+ rd, = IO.select([@handle], nil, nil, @timeout - timespent)
107
+ timespent = Time.now - start
108
+ break rd if (rd and not rd.empty?) or timespent >= @timeout
109
+ end
110
+ if rd.nil? or rd.empty?
111
+ raise TransportException.new(TransportException::TIMED_OUT, "Socket: Timed out reading #{sz} bytes from #{@desc}")
112
+ else
113
+ data = @handle.readpartial(sz)
114
+ end
115
+ end
116
+ rescue TransportException => e
117
+ # don't let this get caught by the StandardError handler
118
+ raise e
119
+ rescue StandardError => e
120
+ @handle.close unless @handle.closed?
121
+ @handle = nil
122
+ raise TransportException.new(TransportException::NOT_OPEN, e.message)
123
+ end
124
+ if (data.nil? or data.length == 0)
125
+ raise TransportException.new(TransportException::UNKNOWN, "Socket: Could not read #{sz} bytes from #{@desc}")
126
+ end
127
+ data
128
+ end
129
+
130
+ def close
131
+ @handle.close unless @handle.nil? or @handle.closed?
132
+ @handle = nil
133
+ end
134
+
135
+ def to_io
136
+ @handle
137
+ end
138
+ end
139
+ end