thtp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.rspec +3 -0
- data/.rubocop.yml +190 -0
- data/.travis.yml +5 -0
- data/Gemfile +3 -0
- data/LICENSE.txt +21 -0
- data/README.md +39 -0
- data/Rakefile +6 -0
- data/bin/console +7 -0
- data/bin/setup +8 -0
- data/lib/thtp.rb +10 -0
- data/lib/thtp/client.rb +123 -0
- data/lib/thtp/client/instrumentation.rb +46 -0
- data/lib/thtp/client/middleware.rb +27 -0
- data/lib/thtp/encoding.rb +32 -0
- data/lib/thtp/errors.rb +143 -0
- data/lib/thtp/middleware_stack.rb +54 -0
- data/lib/thtp/server.rb +122 -0
- data/lib/thtp/server/instrumentation.rb +241 -0
- data/lib/thtp/server/middleware.rb +63 -0
- data/lib/thtp/server/null_route.rb +22 -0
- data/lib/thtp/server/pub_sub.rb +25 -0
- data/lib/thtp/status.rb +10 -0
- data/lib/thtp/utils.rb +120 -0
- data/lib/thtp/version.rb +3 -0
- data/spec/spec_helper.rb +36 -0
- data/spec/support/helpers/rack_test.rb +3 -0
- data/spec/support/helpers/thrift.rb +163 -0
- data/spec/thtp/server/null_route_spec.rb +27 -0
- data/thtp.gemspec +36 -0
- metadata +233 -0
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'thrift'
|
2
|
+
require 'thtp/errors'
|
3
|
+
|
4
|
+
module THTP
|
5
|
+
class Server
|
6
|
+
module Middleware
|
7
|
+
# Raise Thrift validation issues as their own detectable error type, rather
|
8
|
+
# than just ProtocolException.
|
9
|
+
class SchemaValidation
|
10
|
+
def initialize(app)
|
11
|
+
require 'thrift/validator' # if you don't have it, you'll need it
|
12
|
+
@app = app
|
13
|
+
@validator = Thrift::Validator.new
|
14
|
+
end
|
15
|
+
|
16
|
+
# Raises a ValidationError if any part of the request or response did not
|
17
|
+
# match the schema
|
18
|
+
def call(rpc, *rpc_args, **rpc_opts)
|
19
|
+
@validator.validate(rpc_args)
|
20
|
+
@app.call(rpc, *rpc_args, **rpc_opts).tap { |resp| @validator.validate(resp) }
|
21
|
+
rescue Thrift::ProtocolException => e
|
22
|
+
raise ServerValidationError, e.message
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# Performs explicit rather than implicit AR connection management to ensure
|
27
|
+
# we don't run out of SQL connections. Note that this approach is
|
28
|
+
# suboptimal from a contention standpoint (better to check out once per
|
29
|
+
# thread), but that sync time should be irrelevant if we size our pool
|
30
|
+
# correctly, which we do. It is also suboptimal if we have any handler
|
31
|
+
# methods that do not hit the database at all, but that's unlikely.
|
32
|
+
#
|
33
|
+
# For more details, check out (get it?):
|
34
|
+
# https://bibwild.wordpress.com/2014/07/17/activerecord-concurrency-in-rails4-avoid-leaked-connections/
|
35
|
+
#
|
36
|
+
# This is probably only useful on servers.
|
37
|
+
class ActiveRecordPool
|
38
|
+
def initialize(app)
|
39
|
+
require 'active_record' # if you don't have it, why do you want this?
|
40
|
+
@app = app
|
41
|
+
end
|
42
|
+
|
43
|
+
def call(rpc, *rpc_args_and_opts)
|
44
|
+
ActiveRecord::Base.connection_pool.with_connection { @app.call(rpc, *rpc_args_and_opts) }
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Instruments RPCs for Skylight; requires Skylight to be initialised properly elsewhere
|
49
|
+
class Skylight
|
50
|
+
def initialize(app)
|
51
|
+
require 'skylight'
|
52
|
+
@app = app
|
53
|
+
end
|
54
|
+
|
55
|
+
def call(rpc, *rpc_args_and_opts)
|
56
|
+
::Skylight.trace(rpc, 'rpc') do
|
57
|
+
@app.call(rpc, *rpc_args_and_opts)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'rack'
|
2
|
+
require 'thrift'
|
3
|
+
|
4
|
+
require 'thtp/encoding'
|
5
|
+
require 'thtp/errors'
|
6
|
+
require 'thtp/status'
|
7
|
+
require 'thtp/utils'
|
8
|
+
|
9
|
+
module THTP
|
10
|
+
# A THTP middleware stack terminator, telling clients they've done
|
11
|
+
# something wrong if their requests weren't captured by a running server
|
12
|
+
class NullRoute
|
13
|
+
# if a request makes it here, it's bad; tell the caller what it should have done
|
14
|
+
def call(env)
|
15
|
+
# default to CompactProtocol because we need a protocol with which to send back errors
|
16
|
+
protocol = Encoding.protocol(Rack::Request.new(env).media_type) || Thrift::JsonProtocol
|
17
|
+
headers = { Rack::CONTENT_TYPE => Encoding.content_type(protocol) }
|
18
|
+
body = Utils.serialize_buffer(BadRequestError.new.to_thrift, protocol)
|
19
|
+
[Status::EXCEPTION, headers, [body]]
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module THTP
|
2
|
+
class Server
|
3
|
+
# A truly trivial pub/sub implementation for instrumentation. Note, NOT
|
4
|
+
# threadsafe; make sure all subscribers are added before publishing anything.
|
5
|
+
module PubSub
|
6
|
+
# Add listeners to be run in the order of subscription
|
7
|
+
def subscribe(subscriber)
|
8
|
+
subscribers << subscriber
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
# If a subscriber raises an exception, any future ones won't run: this is
|
14
|
+
# not considered a bug. Don't raise.
|
15
|
+
def publish(event, *args)
|
16
|
+
# freeze to prevent any subscriber changes after usage
|
17
|
+
subscribers.freeze.each { |l| l.send(event, *args) if l.respond_to?(event) }
|
18
|
+
end
|
19
|
+
|
20
|
+
def subscribers
|
21
|
+
@subscribers ||= []
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/lib/thtp/status.rb
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
module THTP
|
2
|
+
# These HTTP status codes correspond to Thrift::MessageTypes, which tell
|
3
|
+
# clients what kind of message body to expect.
|
4
|
+
module Status
|
5
|
+
# CALL is not needed because HTTP has explicit requests
|
6
|
+
# ONEWAY is not supported in Ruby, but regardless, it has the same semantics as CALL
|
7
|
+
REPLY = 200
|
8
|
+
EXCEPTION = 500
|
9
|
+
end
|
10
|
+
end
|
data/lib/thtp/utils.rb
ADDED
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'thrift'
|
2
|
+
|
3
|
+
require 'thtp/errors'
|
4
|
+
|
5
|
+
module THTP
|
6
|
+
# Methods for interacting with generated Thrift files
|
7
|
+
module Utils
|
8
|
+
extend self
|
9
|
+
|
10
|
+
### Timing
|
11
|
+
|
12
|
+
# get the current time, for benchmarking purposes. monotonic time is better
|
13
|
+
# than Time.now for this purposes. note, only works on Ruby 2.1.8+
|
14
|
+
def get_time
|
15
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
16
|
+
end
|
17
|
+
|
18
|
+
# useful for benchmarking. returns wall clock ms since start time
|
19
|
+
def elapsed_ms(start_time)
|
20
|
+
((get_time - start_time) * 1000).round
|
21
|
+
end
|
22
|
+
|
23
|
+
### Thrift definition hacks
|
24
|
+
|
25
|
+
# @return array<Symbol> for a given thrift Service using reflection
|
26
|
+
# because the Thrift compiler's generated definitions do not lend
|
27
|
+
# themselves to external use
|
28
|
+
def extract_rpcs(thrift_service)
|
29
|
+
# it's really the Processor we want (the Client would work too)
|
30
|
+
root = thrift_service < Thrift::Processor ? thrift_service : thrift_service::Processor
|
31
|
+
# get all candidate classes that may contribute RPCs to this service
|
32
|
+
root.ancestors.flat_map do |klass|
|
33
|
+
next [] unless klass < Thrift::Processor
|
34
|
+
klass.instance_methods(false).
|
35
|
+
select { |method_name| method_name =~ /^process_/ }.
|
36
|
+
map { |method_name| method_name.to_s.sub(/^process_/, '').to_sym }
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# args class is named after RPC, e.g., #get_things => Get_things_args
|
41
|
+
def args_class(service, rpc)
|
42
|
+
service.const_get("#{rpc.capitalize}_args")
|
43
|
+
end
|
44
|
+
|
45
|
+
# result class is named after RPC, e.g., #do_stuff => Do_stuff_result
|
46
|
+
def result_class(service, rpc)
|
47
|
+
service.const_get("#{rpc.capitalize}_result")
|
48
|
+
end
|
49
|
+
|
50
|
+
### Thrift deserialisation
|
51
|
+
|
52
|
+
def deserialize(transport, struct, protocol)
|
53
|
+
struct.read(protocol.new(transport)) # read off the stream into Thrift objects
|
54
|
+
struct # return input object with all fields filled out
|
55
|
+
rescue Thrift::ProtocolException, EOFError => e
|
56
|
+
raise DeserializationError, e
|
57
|
+
end
|
58
|
+
|
59
|
+
def deserialize_buffer(buffer, struct, protocol)
|
60
|
+
deserialize(Thrift::MemoryBufferTransport.new(buffer), struct, protocol)
|
61
|
+
end
|
62
|
+
|
63
|
+
def deserialize_stream(in_stream, struct, protocol)
|
64
|
+
deserialize(Thrift::IOStreamTransport.new(in_stream, nil), struct, protocol)
|
65
|
+
end
|
66
|
+
|
67
|
+
# Thrift serialisation
|
68
|
+
|
69
|
+
def serialize(base_struct, transport, protocol)
|
70
|
+
base_struct.write(protocol.new(transport))
|
71
|
+
transport.tap(&:flush)
|
72
|
+
rescue Thrift::ProtocolException, EOFError => e
|
73
|
+
raise SerializationError, e
|
74
|
+
end
|
75
|
+
|
76
|
+
def serialize_buffer(base_struct, protocol)
|
77
|
+
transport = Thrift::MemoryBufferTransport.new
|
78
|
+
serialize(base_struct, transport, protocol) # write to output transport, an in-memory buffer
|
79
|
+
transport.read(transport.available) # return serialised thrift
|
80
|
+
end
|
81
|
+
|
82
|
+
def serialize_stream(base_struct, out_stream, protocol)
|
83
|
+
serialize(base_struct, Thrift::IOStreamTransport.new(nil, out_stream), protocol)
|
84
|
+
end
|
85
|
+
|
86
|
+
# String and object transforms
|
87
|
+
|
88
|
+
# Used to turn classes to canonical strings for routing and logging, e.g.,
|
89
|
+
# - MyServices::Thing::ThingService -> MyService.Thing.ThingService
|
90
|
+
# - MyServices::Thing::ThingRequest -> MyService.Thing.ThingRequest
|
91
|
+
# - MyServices::Thing::BadException -> MyService.Thing.BadException
|
92
|
+
# @param klass [Class]
|
93
|
+
def canonical_name(klass)
|
94
|
+
klass.name.gsub('::', '.')
|
95
|
+
end
|
96
|
+
|
97
|
+
# Recursive object serialiser compatible with anything likely to appear
|
98
|
+
# in Thrift, in case ActiveSupport's #as_json monkeypatch is unavailable
|
99
|
+
def jsonify(obj)
|
100
|
+
return obj.as_json if obj.respond_to?(:as_json) # ActiveSupport shortcut
|
101
|
+
|
102
|
+
case obj
|
103
|
+
when nil, false, true, Numeric # directly representable as JSON
|
104
|
+
obj
|
105
|
+
when Hash
|
106
|
+
obj.each_with_object({}) { |(k, v), h| h[jsonify(k)] = jsonify(v) }
|
107
|
+
when Enumerable
|
108
|
+
obj.map { |v| jsonify(v) }
|
109
|
+
else
|
110
|
+
if obj.instance_variables.any?
|
111
|
+
obj.instance_variables.each_with_object({}) do |iv, h|
|
112
|
+
h[iv.to_s[1..-1].to_sym] = jsonify(obj.instance_variable_get(iv))
|
113
|
+
end
|
114
|
+
else
|
115
|
+
obj.to_s # lowest common denominator serialisation
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
data/lib/thtp/version.rb
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'bundler/setup'
|
2
|
+
require 'thtp'
|
3
|
+
|
4
|
+
# test helpers
|
5
|
+
require 'rack/test'
|
6
|
+
require 'webmock'
|
7
|
+
|
8
|
+
# require all spec helpers
|
9
|
+
Dir[File.expand_path('support/**/*.rb', __dir__)].each do |f|
|
10
|
+
require f
|
11
|
+
end
|
12
|
+
|
13
|
+
RSpec.configure do |config|
|
14
|
+
# Enable flags like --only-failures and --next-failure
|
15
|
+
config.example_status_persistence_file_path = '.rspec_status'
|
16
|
+
|
17
|
+
# Disable RSpec exposing methods globally on `Module` and `main`
|
18
|
+
config.disable_monkey_patching!
|
19
|
+
|
20
|
+
config.expect_with :rspec do |c|
|
21
|
+
c.syntax = :expect
|
22
|
+
end
|
23
|
+
|
24
|
+
# Prevents mocking or stubbing a method that does not exist
|
25
|
+
config.mock_with :rspec do |mocks|
|
26
|
+
mocks.verify_partial_doubles = true
|
27
|
+
end
|
28
|
+
|
29
|
+
config.warnings = true
|
30
|
+
|
31
|
+
config.profile_examples = 10
|
32
|
+
|
33
|
+
# randomise test order, allowing repeatable runs via --seed
|
34
|
+
config.order = :random
|
35
|
+
Kernel.srand config.seed
|
36
|
+
end
|
@@ -0,0 +1,163 @@
|
|
1
|
+
module ThriftHelpers
|
2
|
+
include THTP::Utils
|
3
|
+
|
4
|
+
def read_struct(buffer, struct, protocol)
|
5
|
+
deserialize_buffer(buffer, struct, protocol)
|
6
|
+
end
|
7
|
+
|
8
|
+
def read_exception(buffer, protocol)
|
9
|
+
read_struct(buffer, Thrift::ApplicationException.new, protocol)
|
10
|
+
end
|
11
|
+
|
12
|
+
def read_result_response(response, service, rpc)
|
13
|
+
protocol = THTP::Encoding.protocol(response.media_type)
|
14
|
+
result_struct = THTP::Utils.result_class(service, rpc).new
|
15
|
+
read_struct(response.body, result_struct, protocol)
|
16
|
+
end
|
17
|
+
|
18
|
+
def read_exception_response(response)
|
19
|
+
protocol = THTP::Encoding.protocol(response.media_type)
|
20
|
+
read_exception(response.body, protocol)
|
21
|
+
end
|
22
|
+
|
23
|
+
def get_result_type(result_struct)
|
24
|
+
result_struct.struct_fields.each_value do |field|
|
25
|
+
return field[:name] unless result_struct.send(field[:name]).nil?
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def get_result_response(response, service, rpc)
|
30
|
+
result = read_result_response(response, service, rpc)
|
31
|
+
field = get_result_type(result)
|
32
|
+
return result.send(field) if field
|
33
|
+
return nil unless result.respond_to?(:success)
|
34
|
+
raise THTP::BadResponseError, rpc
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
RSpec.configure do |config|
|
39
|
+
config.include ThriftHelpers
|
40
|
+
end
|
41
|
+
|
42
|
+
RSpec::Matchers.define :be_thrift_struct do |klass, attrs|
|
43
|
+
match do |struct|
|
44
|
+
expect(struct).to be_a klass
|
45
|
+
expect(struct).to have_attributes(attrs) unless attrs.nil?
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
RSpec::Matchers.define :be_thrift_result_response do |service, rpc|
|
50
|
+
match do |response|
|
51
|
+
return false if service.nil? || rpc.nil?
|
52
|
+
return false if response.status != THTP::Status::REPLY
|
53
|
+
read_result_response(response, service, rpc)
|
54
|
+
true
|
55
|
+
rescue THTP::SerializationError
|
56
|
+
false
|
57
|
+
end
|
58
|
+
|
59
|
+
failure_message do |response|
|
60
|
+
return 'service and rpc must be specified' if service.nil? || rpc.nil?
|
61
|
+
|
62
|
+
read_result_response(response, service, rpc)
|
63
|
+
|
64
|
+
if response.status != THTP::Status::REPLY
|
65
|
+
"expected HTTP status #{THTP::Status::REPLY}, but was #{response.status}"
|
66
|
+
end
|
67
|
+
rescue THTP::SerializationError
|
68
|
+
'expected a Thrift ApplicationException, but deserialisation failed'
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
RSpec::Matchers.define :be_a_thrift_success_response do |service, rpc, klass, attrs|
|
73
|
+
match do |response|
|
74
|
+
return false if service.nil? || rpc.nil?
|
75
|
+
return false if response.status != THTP::Status::REPLY
|
76
|
+
result = read_result_response(response, service, rpc)
|
77
|
+
result_type = get_result_type(result)
|
78
|
+
return false unless result_type.nil? || result_type == 'success'
|
79
|
+
expect(result.success).to be_thrift_struct(klass, attrs) if klass && result_type
|
80
|
+
true
|
81
|
+
rescue THTP::SerializationError, Thrift::BadResponseError
|
82
|
+
false
|
83
|
+
end
|
84
|
+
|
85
|
+
failure_message do |response|
|
86
|
+
return 'service and rpc must be specified' if service.nil? || rpc.nil?
|
87
|
+
|
88
|
+
result = read_result_response(response, service, rpc)
|
89
|
+
result_type = get_result_type(result)
|
90
|
+
|
91
|
+
if response.status != THTP::Status::REPLY
|
92
|
+
"expected HTTP status #{THTP::Status::REPLY}, but was #{response.status}"
|
93
|
+
elsif !result_type.nil? && result_type != 'success'
|
94
|
+
"expected successful Thrift result, but was #{result_type} instead"
|
95
|
+
elsif klass && result_type
|
96
|
+
expect(result.success).to be_thrift_struct(klass, attrs)
|
97
|
+
end
|
98
|
+
rescue THTP::SerializationError
|
99
|
+
'expected a Thrift ApplicationException, but deserialisation failed'
|
100
|
+
rescue Thrift::BadResponseError
|
101
|
+
'expected a result, but no result field was set'
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
RSpec::Matchers.define :be_a_thrift_exception_response do |service, rpc, klass, attrs|
|
106
|
+
match do |response|
|
107
|
+
return false if service.nil? || rpc.nil?
|
108
|
+
return false if response.status != THTP::Status::REPLY
|
109
|
+
result = read_result_response(response, service, rpc)
|
110
|
+
result_type = get_result_type(result)
|
111
|
+
return false if result_type.nil? || result_type == 'success'
|
112
|
+
expect(result.send(result_type)).to be_thrift_struct(klass, attrs) if klass
|
113
|
+
true
|
114
|
+
rescue THTP::SerializationError, Thrift::BadResponseError
|
115
|
+
false
|
116
|
+
end
|
117
|
+
|
118
|
+
failure_message do |response|
|
119
|
+
return 'service and rpc must be specified' if service.nil? || rpc.nil?
|
120
|
+
|
121
|
+
result = read_result_response(response, service, rpc)
|
122
|
+
result_type = get_result_type(result)
|
123
|
+
|
124
|
+
if response.status != THTP::Status::REPLY
|
125
|
+
"expected HTTP status #{THTP::Status::REPLY}, but was #{response.status}"
|
126
|
+
elsif result_type.nil? || result_type == 'success'
|
127
|
+
"expected Thrift exception, but was #{result_type} instead"
|
128
|
+
elsif klass
|
129
|
+
expect(result.send(result_type)).to be_thrift_struct(klass, attrs)
|
130
|
+
end
|
131
|
+
rescue THTP::SerializationError
|
132
|
+
'expected a Thrift ApplicationException, but deserialisation failed'
|
133
|
+
rescue Thrift::BadResponseError
|
134
|
+
'expected a result, but no result field was set'
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
RSpec::Matchers.define :be_thrift_error_response do |error_klass, message|
|
139
|
+
match do |response|
|
140
|
+
error = read_exception_response(response)
|
141
|
+
|
142
|
+
return false if response.status != THTP::Status::EXCEPTION
|
143
|
+
return false if !error_klass.nil? && error.type != error_klass.type
|
144
|
+
expect(error.message).to match(message) unless message.nil?
|
145
|
+
true
|
146
|
+
rescue THTP::SerializationError
|
147
|
+
false
|
148
|
+
end
|
149
|
+
|
150
|
+
failure_message do |response|
|
151
|
+
error = read_exception_response(response)
|
152
|
+
|
153
|
+
if response.status != THTP::Status::EXCEPTION
|
154
|
+
"expected HTTP status #{THTP::Status::EXCEPTION}, but was #{response.status}"
|
155
|
+
elsif !error_klass.nil? && error.type != error_klass.type
|
156
|
+
"expected ApplicationExceptionType #{error_klass.type}, but was #{error.type}"
|
157
|
+
elsif !message.nil?
|
158
|
+
expect(error.message).to match(message)
|
159
|
+
end
|
160
|
+
rescue THTP::SerializationError
|
161
|
+
'expected a Thrift ApplicationException, but deserialisation failed'
|
162
|
+
end
|
163
|
+
end
|