thrift_client-adamd 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,272 @@
1
+ require 'socket'
2
+ require 'getoptlong'
3
+
4
+ class ThriftClient
5
+
6
+ # This is a simplified form of thrift, useful for clients only, and not
7
+ # making any attempt to have good performance. It's intended to be used by
8
+ # small command-line tools that don't want to install a dozen ruby files.
9
+ module Simple
10
+ VERSION_1 = 0x8001
11
+
12
+ # message types
13
+ CALL, REPLY, EXCEPTION = (1..3).to_a
14
+
15
+ # value types
16
+ STOP, VOID, BOOL, BYTE, DOUBLE, _, I16, _, I32, _, I64, STRING, STRUCT, MAP, SET, LIST = (0..15).to_a
17
+
18
+ FORMATS = {
19
+ BYTE => "c",
20
+ DOUBLE => "G",
21
+ I16 => "n",
22
+ I32 => "N",
23
+ }
24
+
25
+ SIZES = {
26
+ BYTE => 1,
27
+ DOUBLE => 8,
28
+ I16 => 2,
29
+ I32 => 4,
30
+ }
31
+
32
+ module ComplexType
33
+ module Extends
34
+ def type_id=(n)
35
+ @type_id = n
36
+ end
37
+
38
+ def type_id
39
+ @type_id
40
+ end
41
+ end
42
+
43
+ module Includes
44
+ def to_i
45
+ self.class.type_id
46
+ end
47
+
48
+ def to_s
49
+ args = self.values.map { |v| self.class.type_id == STRUCT ? v.name : v.to_s }.join(", ")
50
+ "#{self.class.name}.new(#{args})"
51
+ end
52
+ end
53
+ end
54
+
55
+ def self.make_type(type_id, name, *args)
56
+ klass = Struct.new("STT_#{name}", *args)
57
+ klass.send(:extend, ComplexType::Extends)
58
+ klass.send(:include, ComplexType::Includes)
59
+ klass.type_id = type_id
60
+ klass
61
+ end
62
+
63
+ ListType = make_type(LIST, "ListType", :element_type)
64
+ MapType = make_type(MAP, "MapType", :key_type, :value_type)
65
+ SetType = make_type(SET, "SetType", :element_type)
66
+ StructType = make_type(STRUCT, "StructType", :struct_class)
67
+
68
+ class << self
69
+ def pack_value(type, value)
70
+ case type
71
+ when BOOL
72
+ [ value ? 1 : 0 ].pack("c")
73
+ when STRING
74
+ [ value.size, value ].pack("Na*")
75
+ when I64
76
+ [ value >> 32, value & 0xffffffff ].pack("NN")
77
+ when ListType
78
+ [ type.element_type.to_i, value.size ].pack("cN") + value.map { |item| pack_value(type.element_type, item) }.join("")
79
+ when MapType
80
+ [ type.key_type.to_i, type.value_type.to_i, value.size ].pack("ccN") + value.map { |k, v| pack_value(type.key_type, k) + pack_value(type.value_type, v) }.join("")
81
+ when SetType
82
+ [ type.element_type.to_i, value.size ].pack("cN") + value.map { |item| pack_value(type.element_type, item) }.join("")
83
+ when StructType
84
+ value._pack
85
+ else
86
+ [ value ].pack(FORMATS[type])
87
+ end
88
+ end
89
+
90
+ def pack_request(method_name, arg_struct, request_id=0)
91
+ [ VERSION_1, CALL, method_name.to_s.size, method_name.to_s, request_id, arg_struct._pack ].pack("nnNa*Na*")
92
+ end
93
+
94
+ def read_value(s, type)
95
+ case type
96
+ when BOOL
97
+ s.read(1).unpack("c").first != 0
98
+ when STRING
99
+ len = s.read(4).unpack("N").first
100
+ s.read(len)
101
+ when I64
102
+ hi, lo = s.read(8).unpack("NN")
103
+ rv = (hi << 32) | lo
104
+ (rv >= (1 << 63)) ? (rv - (1 << 64)) : rv
105
+ when LIST
106
+ read_list(s)
107
+ when MAP
108
+ read_map(s)
109
+ when STRUCT
110
+ read_struct(s)
111
+ when ListType
112
+ read_list(s, type.element_type)
113
+ when MapType
114
+ read_map(s, type.key_type, type.value_type)
115
+ when StructType
116
+ read_struct(s, type.struct_class)
117
+ else
118
+ rv = s.read(SIZES[type]).unpack(FORMATS[type]).first
119
+ case type
120
+ when I16
121
+ (rv >= (1 << 15)) ? (rv - (1 << 16)) : rv
122
+ when I32
123
+ (rv >= (1 << 31)) ? (rv - (1 << 32)) : rv
124
+ else
125
+ rv
126
+ end
127
+ end
128
+ end
129
+
130
+ def read_list(s, element_type=nil)
131
+ etype, len = s.read(5).unpack("cN")
132
+ expected_type = (element_type and element_type.to_i == etype.to_i) ? element_type : etype
133
+ rv = []
134
+ len.times do
135
+ rv << read_value(s, expected_type)
136
+ end
137
+ rv
138
+ end
139
+
140
+ def read_map(s, key_type=nil, value_type=nil)
141
+ ktype, vtype, len = s.read(6).unpack("ccN")
142
+ rv = {}
143
+ expected_key_type, expected_value_type = if key_type and value_type and key_type.to_i == ktype and value_type.to_i == vtype
144
+ [ key_type, value_type ]
145
+ else
146
+ [ ktype, vtype ]
147
+ end
148
+ len.times do
149
+ key = read_value(s, expected_key_type)
150
+ value = read_value(s, expected_value_type)
151
+ rv[key] = value
152
+ end
153
+ rv
154
+ end
155
+
156
+ def read_struct(s, struct_class=nil)
157
+ rv = struct_class.new()
158
+ while true
159
+ type = s.read(1).unpack("c").first
160
+ return rv if type == STOP
161
+ fid = s.read(2).unpack("n").first
162
+ field = struct_class ? struct_class._fields.find { |f| (f.fid == fid) and (f.type.to_i == type) } : nil
163
+ value = read_value(s, field ? field.type : type)
164
+ rv[field.name] = value if field
165
+ end
166
+ end
167
+
168
+ def read_response(s, rv_class)
169
+ version, message_type, method_name_len = s.read(8).unpack("nnN")
170
+ method_name = s.read(method_name_len)
171
+ seq_id = s.read(4).unpack("N").first
172
+ [ method_name, seq_id, read_struct(s, rv_class).rv ]
173
+ end
174
+ end
175
+
176
+ ## ----------------------------------------
177
+
178
+ class Field
179
+ attr_accessor :name, :type, :fid
180
+
181
+ def initialize(name, type, fid)
182
+ @name = name
183
+ @type = type
184
+ @fid = fid
185
+ end
186
+
187
+ def pack(value)
188
+ value.nil? ? "" : [ type.to_i, fid, ThriftClient::Simple.pack_value(type, value) ].pack("cna*")
189
+ end
190
+ end
191
+
192
+ class ThriftException < RuntimeError
193
+ def initialize(reason)
194
+ @reason = reason
195
+ end
196
+
197
+ def to_s
198
+ "ThriftException(#{@reason.inspect})"
199
+ end
200
+ end
201
+
202
+ module ThriftStruct
203
+ module Include
204
+ def _pack
205
+ self.class._fields.map { |f| f.pack(self[f.name]) }.join + [ STOP ].pack("c")
206
+ end
207
+ end
208
+
209
+ module Extend
210
+ def _fields
211
+ @fields
212
+ end
213
+
214
+ def _fields=(f)
215
+ @fields = f
216
+ end
217
+ end
218
+ end
219
+
220
+ def self.make_struct(name, *fields)
221
+ st_name = "ST_#{name}"
222
+ if Struct.constants.include?(st_name)
223
+ warn "#{caller[0]}: Struct::#{st_name} is already defined; returning original class."
224
+ Struct.const_get(st_name)
225
+ else
226
+ names = fields.map { |f| f.name.to_sym }
227
+ klass = Struct.new(st_name, *names)
228
+ klass.send(:include, ThriftStruct::Include)
229
+ klass.send(:extend, ThriftStruct::Extend)
230
+ klass._fields = fields
231
+ klass
232
+ end
233
+ end
234
+
235
+ class ThriftService
236
+ def initialize(sock)
237
+ @sock = sock
238
+ end
239
+
240
+ def self._arg_structs
241
+ @_arg_structs = {} if @_arg_structs.nil?
242
+ @_arg_structs
243
+ end
244
+
245
+ def self.thrift_method(name, rtype, *args)
246
+ arg_struct = ThriftClient::Simple.make_struct("Args__#{self.name}__#{name}", *args)
247
+ rv_struct = ThriftClient::Simple.make_struct("Retval__#{self.name}__#{name}", ThriftClient::Simple::Field.new(:rv, rtype, 0))
248
+ _arg_structs[name.to_sym] = [ arg_struct, rv_struct ]
249
+
250
+ arg_names = args.map { |a| a.name.to_s }.join(", ")
251
+ class_eval "def #{name}(#{arg_names}); _proxy(:#{name}#{args.size > 0 ? ', ' : ''}#{arg_names}); end"
252
+ end
253
+
254
+ def _proxy(method_name, *args)
255
+ cls = self.class.ancestors.find { |cls| cls.respond_to?(:_arg_structs) and cls._arg_structs[method_name.to_sym] }
256
+ arg_class, rv_class = cls._arg_structs[method_name.to_sym]
257
+ arg_struct = arg_class.new(*args)
258
+ @sock.write(ThriftClient::Simple.pack_request(method_name, arg_struct))
259
+ rv = ThriftClient::Simple.read_response(@sock, rv_class)
260
+ rv[2]
261
+ end
262
+
263
+ # convenience. robey is lazy.
264
+ [[ :field, "Field.new" ], [ :struct, "StructType.new" ],
265
+ [ :list, "ListType.new" ], [ :map, "MapType.new" ]].each do |new_name, old_name|
266
+ class_eval "def self.#{new_name}(*args); ThriftClient::Simple::#{old_name}(*args); end"
267
+ end
268
+
269
+ [ :void, :bool, :byte, :double, :i16, :i32, :i64, :string ].each { |sym| class_eval "def self.#{sym}; ThriftClient::Simple::#{sym.to_s.upcase}; end" }
270
+ end
271
+ end
272
+ end
@@ -0,0 +1,40 @@
1
+ module Thrift
2
+ class BaseTransport
3
+ def timeout=(timeout)
4
+ end
5
+
6
+ def timeout
7
+ nil
8
+ end
9
+ end
10
+
11
+ class BufferedTransport
12
+ def timeout=(timeout)
13
+ @transport.timeout = timeout
14
+ end
15
+
16
+ def timeout
17
+ @transport.timeout
18
+ end
19
+ end
20
+
21
+ class FramedTransport
22
+ def timeout=(timeout)
23
+ @transport.timeout = timeout
24
+ end
25
+
26
+ def timeout
27
+ @transport.timeout
28
+ end
29
+ end
30
+
31
+ module Client
32
+ def timeout=(timeout)
33
+ @iprot.trans.timeout = timeout
34
+ end
35
+
36
+ def timeout
37
+ @iprot.trans.timeout
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,34 @@
1
+ require 'thrift'
2
+ require 'thrift_client/thrift'
3
+ require 'thrift_client/server'
4
+ require 'thrift_client/abstract_thrift_client'
5
+
6
+ class ThriftClient < AbstractThriftClient
7
+ class NoServersAvailable < StandardError; end
8
+
9
+ =begin rdoc
10
+ Create a new ThriftClient instance. Accepts an internal Thrift client class (such as CassandraRb::Client), a list of servers with ports, and optional parameters.
11
+
12
+ Valid optional parameters are:
13
+
14
+ <tt>:protocol</tt>:: Which Thrift protocol to use. Defaults to <tt>Thrift::BinaryProtocol</tt>.
15
+ <tt>:protocol_extra_params</tt>:: An array of additional parameters to pass to the protocol initialization call. Defaults to <tt>[]</tt>.
16
+ <tt>:transport</tt>:: Which Thrift transport to use. Defaults to <tt>Thrift::Socket</tt>.
17
+ <tt>:transport_wrapper</tt>:: Which Thrift transport wrapper to use. Defaults to <tt>Thrift::FramedTransport</tt>.
18
+ <tt>:exception_classes</tt>:: Which exceptions to catch and retry when sending a request. Defaults to <tt>[IOError, Thrift::Exception, Thrift::ApplicationException, Thrift::TransportException, NoServersAvailable]</tt>
19
+ <tt>:exception_class_overrides</tt>:: For specifying children of classes in exception_classes for which you don't want to retry or reconnect.
20
+ <tt>:raise</tt>:: Whether to reraise errors if no responsive servers are found. Defaults to <tt>true</tt>.
21
+ <tt>:retries</tt>:: How many times to retry a request. Defaults to 0.
22
+ <tt>:server_retry_period</tt>:: How many seconds to wait before trying to reconnect to a dead server. Defaults to <tt>1</tt>. Set to <tt>nil</tt> to disable.
23
+ <tt>:server_max_requests</tt>:: How many requests to perform before moving on to the next server in the pool, regardless of error status. Defaults to <tt>nil</tt> (no limit).
24
+ <tt>:timeout</tt>:: Specify the default timeout in seconds. Defaults to <tt>1</tt>.
25
+ <tt>:connect_timeout</tt>:: Specify the connection timeout in seconds. Defaults to <tt>0.1</tt>.
26
+ <tt>:timeout_overrides</tt>:: Specify additional timeouts on a per-method basis, in seconds. Only works with <tt>Thrift::BufferedTransport</tt>.
27
+ <tt>:cached_connections</tt>:: Cache connections between requests. Trades connect() costs for open sockets. Defaults to <tt>false</tt>.
28
+ <tt>:defaults</tt>:: Specify default values to return on a per-method basis, if <tt>:raise</tt> is set to false.
29
+ =end rdoc
30
+
31
+ def initialize(client_class, servers, options = {})
32
+ super
33
+ end
34
+ end
@@ -0,0 +1,121 @@
1
+ #
2
+ # Autogenerated by Thrift
3
+ #
4
+ # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
5
+ #
6
+
7
+ require 'thrift'
8
+
9
+ module Greeter
10
+ class Client
11
+ include ::Thrift::Client
12
+
13
+ def greeting(name)
14
+ send_greeting(name)
15
+ return recv_greeting()
16
+ end
17
+
18
+ def send_greeting(name)
19
+ send_message('greeting', Greeting_args, :name => name)
20
+ end
21
+
22
+ def recv_greeting()
23
+ result = receive_message(Greeting_result)
24
+ return result.success unless result.success.nil?
25
+ raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'greeting failed: unknown result')
26
+ end
27
+
28
+ def yo(name)
29
+ send_yo(name)
30
+ end
31
+
32
+ def send_yo(name)
33
+ send_message('yo', Yo_args, :name => name)
34
+ end
35
+ end
36
+
37
+ class Processor
38
+ include ::Thrift::Processor
39
+
40
+ def process_greeting(seqid, iprot, oprot)
41
+ args = read_args(iprot, Greeting_args)
42
+ result = Greeting_result.new()
43
+ result.success = @handler.greeting(args.name)
44
+ write_result(result, oprot, 'greeting', seqid)
45
+ end
46
+
47
+ def process_yo(seqid, iprot, oprot)
48
+ args = read_args(iprot, Yo_args)
49
+ @handler.yo(args.name)
50
+ return
51
+ end
52
+
53
+ end
54
+
55
+ # HELPER FUNCTIONS AND STRUCTURES
56
+
57
+ class Greeting_args
58
+ include ::Thrift::Struct, ::Thrift::Struct_Union
59
+ NAME = 1
60
+
61
+ FIELDS = {
62
+ NAME => {:type => ::Thrift::Types::STRING, :name => 'name'}
63
+ }
64
+
65
+ def struct_fields; FIELDS; end
66
+
67
+ def validate
68
+ end
69
+
70
+ ::Thrift::Struct.generate_accessors self
71
+ end
72
+
73
+ class Greeting_result
74
+ include ::Thrift::Struct, ::Thrift::Struct_Union
75
+ SUCCESS = 0
76
+
77
+ FIELDS = {
78
+ SUCCESS => {:type => ::Thrift::Types::STRING, :name => 'success'}
79
+ }
80
+
81
+ def struct_fields; FIELDS; end
82
+
83
+ def validate
84
+ end
85
+
86
+ ::Thrift::Struct.generate_accessors self
87
+ end
88
+
89
+ class Yo_args
90
+ include ::Thrift::Struct, ::Thrift::Struct_Union
91
+ NAME = 1
92
+
93
+ FIELDS = {
94
+ NAME => {:type => ::Thrift::Types::STRING, :name => 'name'}
95
+ }
96
+
97
+ def struct_fields; FIELDS; end
98
+
99
+ def validate
100
+ end
101
+
102
+ ::Thrift::Struct.generate_accessors self
103
+ end
104
+
105
+ class Yo_result
106
+ include ::Thrift::Struct, ::Thrift::Struct_Union
107
+
108
+ FIELDS = {
109
+
110
+ }
111
+
112
+ def struct_fields; FIELDS; end
113
+
114
+ def validate
115
+ end
116
+
117
+ ::Thrift::Struct.generate_accessors self
118
+ end
119
+
120
+ end
121
+
@@ -0,0 +1,44 @@
1
+ module Greeter
2
+ class Handler
3
+ def greeting(name)
4
+ "hello there #{name}!"
5
+ end
6
+
7
+ def yo(name)
8
+ #whee
9
+ end
10
+ end
11
+
12
+ class Server
13
+ def initialize(port)
14
+ @port = port
15
+ handler = Greeter::Handler.new
16
+ processor = Greeter::Processor.new(handler)
17
+ transport = Thrift::ServerSocket.new("127.0.0.1", port)
18
+ transportFactory = Thrift::FramedTransportFactory.new()
19
+ @server = Thrift::SimpleServer.new(processor, transport, transportFactory)
20
+ end
21
+
22
+ def serve
23
+ @server.serve()
24
+ end
25
+ end
26
+
27
+ # client:
28
+ # trans = Thrift::HTTPClientTransport.new("http://127.0.0.1:9292/greeter")
29
+ # prot = Thrift::BinaryProtocol.new(trans)
30
+ # c = Greeter::Client.new(prot)
31
+ class HTTPServer
32
+ def initialize(uri)
33
+ uri = URI.parse(uri)
34
+ handler = Greeter::Handler.new
35
+ processor = Greeter::Processor.new(handler)
36
+ path = uri.path
37
+ @server = Thrift::ThinHTTPServer.new(processor, :port => uri.port, :ip => uri.host, :path => path)
38
+ end
39
+
40
+ def serve
41
+ @server.serve()
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,112 @@
1
+ require File.expand_path('test_helper.rb', File.dirname(__FILE__))
2
+
3
+ class MultipleWorkingServersTest < Test::Unit::TestCase
4
+ def setup
5
+ @servers = ["127.0.0.1:1461", "127.0.0.1:1462", "127.0.0.1:1463"]
6
+ @socket = 1461
7
+ @timeout = 0.2
8
+ @options = {:protocol_extra_params => [false]}
9
+ @pids = []
10
+ @servers.each do |s|
11
+ @pids << Process.fork do
12
+ Signal.trap("INT") { exit }
13
+ Greeter::Server.new(s.split(':').last).serve
14
+ end
15
+ end
16
+ # Need to give the child process a moment to open the listening socket or
17
+ # we get occasional "could not connect" errors in tests.
18
+ sleep 0.05
19
+ end
20
+
21
+ def teardown
22
+ @pids.each do |pid|
23
+ Process.kill("INT", pid)
24
+ Process.wait(pid)
25
+ end
26
+ end
27
+
28
+ def test_server_creates_new_client_that_can_talk_to_all_servers_after_disconnect
29
+ client = ThriftClient.new(Greeter::Client, @servers, @options)
30
+ client.greeting("someone")
31
+ last_client = client.last_client
32
+ client.greeting("someone")
33
+ assert_equal last_client, client.last_client # Sanity check
34
+
35
+ client.disconnect!
36
+ client.greeting("someone")
37
+ last_client = client.last_client
38
+ client.greeting("someone")
39
+ assert_equal last_client, client.last_client
40
+ last_client = client.last_client
41
+ client.greeting("someone")
42
+ assert_equal last_client, client.last_client
43
+
44
+ # Moves on to the second server
45
+ assert_nothing_raised {
46
+ client.greeting("someone")
47
+ client.greeting("someone")
48
+ }
49
+ end
50
+
51
+ def test_server_doesnt_max_out_after_explicit_disconnect
52
+ client = ThriftClient.new(Greeter::Client, @servers, @options.merge(:server_max_requests => 2))
53
+ client.greeting("someone")
54
+ last_client = client.last_client
55
+ client.greeting("someone")
56
+ assert_equal last_client, client.last_client # Sanity check
57
+
58
+ client.disconnect!
59
+
60
+ client.greeting("someone")
61
+ last_client = client.last_client
62
+ client.greeting("someone")
63
+ assert_equal last_client, client.last_client, "ThriftClient should not have reset the internal client if the counter was reset on disconnect"
64
+ end
65
+
66
+ def test_server_disconnect_doesnt_drop_servers_with_retry_period
67
+ client = ThriftClient.new(Greeter::Client, @servers, @options.merge(:server_max_requests => 2, :retry_period => 1))
68
+ 3.times {
69
+ client.greeting("someone")
70
+ last_client = client.last_client
71
+ client.greeting("someone")
72
+ assert_equal last_client, client.last_client # Sanity check
73
+
74
+ client.disconnect!
75
+
76
+ client.greeting("someone")
77
+ last_client = client.last_client
78
+ client.greeting("someone")
79
+ assert_equal last_client, client.last_client, "ThriftClient should not have reset the internal client if the counter was reset on disconnect"
80
+ }
81
+ end
82
+
83
+
84
+ def test_server_max_requests
85
+ client = ThriftClient.new(Greeter::Client, @servers, @options.merge(:server_max_requests => 2))
86
+
87
+ client.greeting("someone")
88
+ last_client = client.last_client
89
+
90
+ client.greeting("someone")
91
+ assert_equal last_client, client.last_client
92
+
93
+ # This next call maxes out the requests for that "client" object
94
+ # and moves on to the next.
95
+ client.greeting("someone")
96
+ assert_not_equal last_client, new_client = client.last_client
97
+
98
+ # And here we should still have the same client as the last one...
99
+ client.greeting("someone")
100
+ assert_equal new_client, client.last_client
101
+
102
+ # Until we max it out, too.
103
+ client.greeting("someone")
104
+ assert_not_equal new_client, client.last_client
105
+ assert_not_nil client.last_client
106
+
107
+ new_new_client = client.last_client
108
+ # And we should still have one server left
109
+ client.greeting("someone")
110
+ assert_equal new_new_client, client.last_client
111
+ end
112
+ end