thtp 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,63 @@
1
+ require 'thrift'
2
+ require 'thtp/errors'
3
+
4
+ module THTP
5
+ class Server
6
+ module Middleware
7
+ # Raise Thrift validation issues as their own detectable error type, rather
8
+ # than just ProtocolException.
9
+ class SchemaValidation
10
+ def initialize(app)
11
+ require 'thrift/validator' # if you don't have it, you'll need it
12
+ @app = app
13
+ @validator = Thrift::Validator.new
14
+ end
15
+
16
+ # Raises a ValidationError if any part of the request or response did not
17
+ # match the schema
18
+ def call(rpc, *rpc_args, **rpc_opts)
19
+ @validator.validate(rpc_args)
20
+ @app.call(rpc, *rpc_args, **rpc_opts).tap { |resp| @validator.validate(resp) }
21
+ rescue Thrift::ProtocolException => e
22
+ raise ServerValidationError, e.message
23
+ end
24
+ end
25
+
26
+ # Performs explicit rather than implicit AR connection management to ensure
27
+ # we don't run out of SQL connections. Note that this approach is
28
+ # suboptimal from a contention standpoint (better to check out once per
29
+ # thread), but that sync time should be irrelevant if we size our pool
30
+ # correctly, which we do. It is also suboptimal if we have any handler
31
+ # methods that do not hit the database at all, but that's unlikely.
32
+ #
33
+ # For more details, check out (get it?):
34
+ # https://bibwild.wordpress.com/2014/07/17/activerecord-concurrency-in-rails4-avoid-leaked-connections/
35
+ #
36
+ # This is probably only useful on servers.
37
+ class ActiveRecordPool
38
+ def initialize(app)
39
+ require 'active_record' # if you don't have it, why do you want this?
40
+ @app = app
41
+ end
42
+
43
+ def call(rpc, *rpc_args_and_opts)
44
+ ActiveRecord::Base.connection_pool.with_connection { @app.call(rpc, *rpc_args_and_opts) }
45
+ end
46
+ end
47
+
48
+ # Instruments RPCs for Skylight; requires Skylight to be initialised properly elsewhere
49
+ class Skylight
50
+ def initialize(app)
51
+ require 'skylight'
52
+ @app = app
53
+ end
54
+
55
+ def call(rpc, *rpc_args_and_opts)
56
+ ::Skylight.trace(rpc, 'rpc') do
57
+ @app.call(rpc, *rpc_args_and_opts)
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,22 @@
1
+ require 'rack'
2
+ require 'thrift'
3
+
4
+ require 'thtp/encoding'
5
+ require 'thtp/errors'
6
+ require 'thtp/status'
7
+ require 'thtp/utils'
8
+
9
+ module THTP
10
+ # A THTP middleware stack terminator, telling clients they've done
11
+ # something wrong if their requests weren't captured by a running server
12
+ class NullRoute
13
+ # if a request makes it here, it's bad; tell the caller what it should have done
14
+ def call(env)
15
+ # default to CompactProtocol because we need a protocol with which to send back errors
16
+ protocol = Encoding.protocol(Rack::Request.new(env).media_type) || Thrift::JsonProtocol
17
+ headers = { Rack::CONTENT_TYPE => Encoding.content_type(protocol) }
18
+ body = Utils.serialize_buffer(BadRequestError.new.to_thrift, protocol)
19
+ [Status::EXCEPTION, headers, [body]]
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,25 @@
1
+ module THTP
2
+ class Server
3
+ # A truly trivial pub/sub implementation for instrumentation. Note, NOT
4
+ # threadsafe; make sure all subscribers are added before publishing anything.
5
+ module PubSub
6
+ # Add listeners to be run in the order of subscription
7
+ def subscribe(subscriber)
8
+ subscribers << subscriber
9
+ end
10
+
11
+ private
12
+
13
+ # If a subscriber raises an exception, any future ones won't run: this is
14
+ # not considered a bug. Don't raise.
15
+ def publish(event, *args)
16
+ # freeze to prevent any subscriber changes after usage
17
+ subscribers.freeze.each { |l| l.send(event, *args) if l.respond_to?(event) }
18
+ end
19
+
20
+ def subscribers
21
+ @subscribers ||= []
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,10 @@
1
+ module THTP
2
+ # These HTTP status codes correspond to Thrift::MessageTypes, which tell
3
+ # clients what kind of message body to expect.
4
+ module Status
5
+ # CALL is not needed because HTTP has explicit requests
6
+ # ONEWAY is not supported in Ruby, but regardless, it has the same semantics as CALL
7
+ REPLY = 200
8
+ EXCEPTION = 500
9
+ end
10
+ end
data/lib/thtp/utils.rb ADDED
@@ -0,0 +1,120 @@
1
+ require 'thrift'
2
+
3
+ require 'thtp/errors'
4
+
5
+ module THTP
6
+ # Methods for interacting with generated Thrift files
7
+ module Utils
8
+ extend self
9
+
10
+ ### Timing
11
+
12
+ # get the current time, for benchmarking purposes. monotonic time is better
13
+ # than Time.now for this purposes. note, only works on Ruby 2.1.8+
14
+ def get_time
15
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
16
+ end
17
+
18
+ # useful for benchmarking. returns wall clock ms since start time
19
+ def elapsed_ms(start_time)
20
+ ((get_time - start_time) * 1000).round
21
+ end
22
+
23
+ ### Thrift definition hacks
24
+
25
+ # @return array<Symbol> for a given thrift Service using reflection
26
+ # because the Thrift compiler's generated definitions do not lend
27
+ # themselves to external use
28
+ def extract_rpcs(thrift_service)
29
+ # it's really the Processor we want (the Client would work too)
30
+ root = thrift_service < Thrift::Processor ? thrift_service : thrift_service::Processor
31
+ # get all candidate classes that may contribute RPCs to this service
32
+ root.ancestors.flat_map do |klass|
33
+ next [] unless klass < Thrift::Processor
34
+ klass.instance_methods(false).
35
+ select { |method_name| method_name =~ /^process_/ }.
36
+ map { |method_name| method_name.to_s.sub(/^process_/, '').to_sym }
37
+ end
38
+ end
39
+
40
+ # args class is named after RPC, e.g., #get_things => Get_things_args
41
+ def args_class(service, rpc)
42
+ service.const_get("#{rpc.capitalize}_args")
43
+ end
44
+
45
+ # result class is named after RPC, e.g., #do_stuff => Do_stuff_result
46
+ def result_class(service, rpc)
47
+ service.const_get("#{rpc.capitalize}_result")
48
+ end
49
+
50
+ ### Thrift deserialisation
51
+
52
+ def deserialize(transport, struct, protocol)
53
+ struct.read(protocol.new(transport)) # read off the stream into Thrift objects
54
+ struct # return input object with all fields filled out
55
+ rescue Thrift::ProtocolException, EOFError => e
56
+ raise DeserializationError, e
57
+ end
58
+
59
+ def deserialize_buffer(buffer, struct, protocol)
60
+ deserialize(Thrift::MemoryBufferTransport.new(buffer), struct, protocol)
61
+ end
62
+
63
+ def deserialize_stream(in_stream, struct, protocol)
64
+ deserialize(Thrift::IOStreamTransport.new(in_stream, nil), struct, protocol)
65
+ end
66
+
67
+ # Thrift serialisation
68
+
69
+ def serialize(base_struct, transport, protocol)
70
+ base_struct.write(protocol.new(transport))
71
+ transport.tap(&:flush)
72
+ rescue Thrift::ProtocolException, EOFError => e
73
+ raise SerializationError, e
74
+ end
75
+
76
+ def serialize_buffer(base_struct, protocol)
77
+ transport = Thrift::MemoryBufferTransport.new
78
+ serialize(base_struct, transport, protocol) # write to output transport, an in-memory buffer
79
+ transport.read(transport.available) # return serialised thrift
80
+ end
81
+
82
+ def serialize_stream(base_struct, out_stream, protocol)
83
+ serialize(base_struct, Thrift::IOStreamTransport.new(nil, out_stream), protocol)
84
+ end
85
+
86
+ # String and object transforms
87
+
88
+ # Used to turn classes to canonical strings for routing and logging, e.g.,
89
+ # - MyServices::Thing::ThingService -> MyService.Thing.ThingService
90
+ # - MyServices::Thing::ThingRequest -> MyService.Thing.ThingRequest
91
+ # - MyServices::Thing::BadException -> MyService.Thing.BadException
92
+ # @param klass [Class]
93
+ def canonical_name(klass)
94
+ klass.name.gsub('::', '.')
95
+ end
96
+
97
+ # Recursive object serialiser compatible with anything likely to appear
98
+ # in Thrift, in case ActiveSupport's #as_json monkeypatch is unavailable
99
+ def jsonify(obj)
100
+ return obj.as_json if obj.respond_to?(:as_json) # ActiveSupport shortcut
101
+
102
+ case obj
103
+ when nil, false, true, Numeric # directly representable as JSON
104
+ obj
105
+ when Hash
106
+ obj.each_with_object({}) { |(k, v), h| h[jsonify(k)] = jsonify(v) }
107
+ when Enumerable
108
+ obj.map { |v| jsonify(v) }
109
+ else
110
+ if obj.instance_variables.any?
111
+ obj.instance_variables.each_with_object({}) do |iv, h|
112
+ h[iv.to_s[1..-1].to_sym] = jsonify(obj.instance_variable_get(iv))
113
+ end
114
+ else
115
+ obj.to_s # lowest common denominator serialisation
116
+ end
117
+ end
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,3 @@
1
+ module THTP
2
+ VERSION = '0.1.0'
3
+ end
@@ -0,0 +1,36 @@
1
+ require 'bundler/setup'
2
+ require 'thtp'
3
+
4
+ # test helpers
5
+ require 'rack/test'
6
+ require 'webmock'
7
+
8
+ # require all spec helpers
9
+ Dir[File.expand_path('support/**/*.rb', __dir__)].each do |f|
10
+ require f
11
+ end
12
+
13
+ RSpec.configure do |config|
14
+ # Enable flags like --only-failures and --next-failure
15
+ config.example_status_persistence_file_path = '.rspec_status'
16
+
17
+ # Disable RSpec exposing methods globally on `Module` and `main`
18
+ config.disable_monkey_patching!
19
+
20
+ config.expect_with :rspec do |c|
21
+ c.syntax = :expect
22
+ end
23
+
24
+ # Prevents mocking or stubbing a method that does not exist
25
+ config.mock_with :rspec do |mocks|
26
+ mocks.verify_partial_doubles = true
27
+ end
28
+
29
+ config.warnings = true
30
+
31
+ config.profile_examples = 10
32
+
33
+ # randomise test order, allowing repeatable runs via --seed
34
+ config.order = :random
35
+ Kernel.srand config.seed
36
+ end
@@ -0,0 +1,3 @@
1
+ RSpec.configure do |config|
2
+ config.include Rack::Test::Methods
3
+ end
@@ -0,0 +1,163 @@
1
+ module ThriftHelpers
2
+ include THTP::Utils
3
+
4
+ def read_struct(buffer, struct, protocol)
5
+ deserialize_buffer(buffer, struct, protocol)
6
+ end
7
+
8
+ def read_exception(buffer, protocol)
9
+ read_struct(buffer, Thrift::ApplicationException.new, protocol)
10
+ end
11
+
12
+ def read_result_response(response, service, rpc)
13
+ protocol = THTP::Encoding.protocol(response.media_type)
14
+ result_struct = THTP::Utils.result_class(service, rpc).new
15
+ read_struct(response.body, result_struct, protocol)
16
+ end
17
+
18
+ def read_exception_response(response)
19
+ protocol = THTP::Encoding.protocol(response.media_type)
20
+ read_exception(response.body, protocol)
21
+ end
22
+
23
+ def get_result_type(result_struct)
24
+ result_struct.struct_fields.each_value do |field|
25
+ return field[:name] unless result_struct.send(field[:name]).nil?
26
+ end
27
+ end
28
+
29
+ def get_result_response(response, service, rpc)
30
+ result = read_result_response(response, service, rpc)
31
+ field = get_result_type(result)
32
+ return result.send(field) if field
33
+ return nil unless result.respond_to?(:success)
34
+ raise THTP::BadResponseError, rpc
35
+ end
36
+ end
37
+
38
+ RSpec.configure do |config|
39
+ config.include ThriftHelpers
40
+ end
41
+
42
+ RSpec::Matchers.define :be_thrift_struct do |klass, attrs|
43
+ match do |struct|
44
+ expect(struct).to be_a klass
45
+ expect(struct).to have_attributes(attrs) unless attrs.nil?
46
+ end
47
+ end
48
+
49
+ RSpec::Matchers.define :be_thrift_result_response do |service, rpc|
50
+ match do |response|
51
+ return false if service.nil? || rpc.nil?
52
+ return false if response.status != THTP::Status::REPLY
53
+ read_result_response(response, service, rpc)
54
+ true
55
+ rescue THTP::SerializationError
56
+ false
57
+ end
58
+
59
+ failure_message do |response|
60
+ return 'service and rpc must be specified' if service.nil? || rpc.nil?
61
+
62
+ read_result_response(response, service, rpc)
63
+
64
+ if response.status != THTP::Status::REPLY
65
+ "expected HTTP status #{THTP::Status::REPLY}, but was #{response.status}"
66
+ end
67
+ rescue THTP::SerializationError
68
+ 'expected a Thrift ApplicationException, but deserialisation failed'
69
+ end
70
+ end
71
+
72
+ RSpec::Matchers.define :be_a_thrift_success_response do |service, rpc, klass, attrs|
73
+ match do |response|
74
+ return false if service.nil? || rpc.nil?
75
+ return false if response.status != THTP::Status::REPLY
76
+ result = read_result_response(response, service, rpc)
77
+ result_type = get_result_type(result)
78
+ return false unless result_type.nil? || result_type == 'success'
79
+ expect(result.success).to be_thrift_struct(klass, attrs) if klass && result_type
80
+ true
81
+ rescue THTP::SerializationError, Thrift::BadResponseError
82
+ false
83
+ end
84
+
85
+ failure_message do |response|
86
+ return 'service and rpc must be specified' if service.nil? || rpc.nil?
87
+
88
+ result = read_result_response(response, service, rpc)
89
+ result_type = get_result_type(result)
90
+
91
+ if response.status != THTP::Status::REPLY
92
+ "expected HTTP status #{THTP::Status::REPLY}, but was #{response.status}"
93
+ elsif !result_type.nil? && result_type != 'success'
94
+ "expected successful Thrift result, but was #{result_type} instead"
95
+ elsif klass && result_type
96
+ expect(result.success).to be_thrift_struct(klass, attrs)
97
+ end
98
+ rescue THTP::SerializationError
99
+ 'expected a Thrift ApplicationException, but deserialisation failed'
100
+ rescue Thrift::BadResponseError
101
+ 'expected a result, but no result field was set'
102
+ end
103
+ end
104
+
105
+ RSpec::Matchers.define :be_a_thrift_exception_response do |service, rpc, klass, attrs|
106
+ match do |response|
107
+ return false if service.nil? || rpc.nil?
108
+ return false if response.status != THTP::Status::REPLY
109
+ result = read_result_response(response, service, rpc)
110
+ result_type = get_result_type(result)
111
+ return false if result_type.nil? || result_type == 'success'
112
+ expect(result.send(result_type)).to be_thrift_struct(klass, attrs) if klass
113
+ true
114
+ rescue THTP::SerializationError, Thrift::BadResponseError
115
+ false
116
+ end
117
+
118
+ failure_message do |response|
119
+ return 'service and rpc must be specified' if service.nil? || rpc.nil?
120
+
121
+ result = read_result_response(response, service, rpc)
122
+ result_type = get_result_type(result)
123
+
124
+ if response.status != THTP::Status::REPLY
125
+ "expected HTTP status #{THTP::Status::REPLY}, but was #{response.status}"
126
+ elsif result_type.nil? || result_type == 'success'
127
+ "expected Thrift exception, but was #{result_type} instead"
128
+ elsif klass
129
+ expect(result.send(result_type)).to be_thrift_struct(klass, attrs)
130
+ end
131
+ rescue THTP::SerializationError
132
+ 'expected a Thrift ApplicationException, but deserialisation failed'
133
+ rescue Thrift::BadResponseError
134
+ 'expected a result, but no result field was set'
135
+ end
136
+ end
137
+
138
+ RSpec::Matchers.define :be_thrift_error_response do |error_klass, message|
139
+ match do |response|
140
+ error = read_exception_response(response)
141
+
142
+ return false if response.status != THTP::Status::EXCEPTION
143
+ return false if !error_klass.nil? && error.type != error_klass.type
144
+ expect(error.message).to match(message) unless message.nil?
145
+ true
146
+ rescue THTP::SerializationError
147
+ false
148
+ end
149
+
150
+ failure_message do |response|
151
+ error = read_exception_response(response)
152
+
153
+ if response.status != THTP::Status::EXCEPTION
154
+ "expected HTTP status #{THTP::Status::EXCEPTION}, but was #{response.status}"
155
+ elsif !error_klass.nil? && error.type != error_klass.type
156
+ "expected ApplicationExceptionType #{error_klass.type}, but was #{error.type}"
157
+ elsif !message.nil?
158
+ expect(error.message).to match(message)
159
+ end
160
+ rescue THTP::SerializationError
161
+ 'expected a Thrift ApplicationException, but deserialisation failed'
162
+ end
163
+ end