makara 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/.rspec +2 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.travis.yml +28 -0
- data/CHANGELOG.md +27 -0
- data/Gemfile +18 -0
- data/LICENSE.txt +22 -0
- data/README.md +278 -0
- data/Rakefile +9 -0
- data/gemfiles/ar30.gemfile +30 -0
- data/gemfiles/ar31.gemfile +29 -0
- data/gemfiles/ar32.gemfile +29 -0
- data/gemfiles/ar40.gemfile +15 -0
- data/gemfiles/ar41.gemfile +15 -0
- data/gemfiles/ar42.gemfile +15 -0
- data/lib/active_record/connection_adapters/jdbcmysql_makara_adapter.rb +25 -0
- data/lib/active_record/connection_adapters/jdbcpostgresql_makara_adapter.rb +25 -0
- data/lib/active_record/connection_adapters/makara_abstract_adapter.rb +209 -0
- data/lib/active_record/connection_adapters/makara_jdbcmysql_adapter.rb +25 -0
- data/lib/active_record/connection_adapters/makara_jdbcpostgresql_adapter.rb +25 -0
- data/lib/active_record/connection_adapters/makara_mysql2_adapter.rb +44 -0
- data/lib/active_record/connection_adapters/makara_postgresql_adapter.rb +44 -0
- data/lib/active_record/connection_adapters/mysql2_makara_adapter.rb +44 -0
- data/lib/active_record/connection_adapters/postgresql_makara_adapter.rb +44 -0
- data/lib/makara.rb +25 -0
- data/lib/makara/cache.rb +53 -0
- data/lib/makara/cache/memory_store.rb +28 -0
- data/lib/makara/cache/noop_store.rb +15 -0
- data/lib/makara/config_parser.rb +200 -0
- data/lib/makara/connection_wrapper.rb +170 -0
- data/lib/makara/context.rb +46 -0
- data/lib/makara/error_handler.rb +39 -0
- data/lib/makara/errors/all_connections_blacklisted.rb +13 -0
- data/lib/makara/errors/blacklist_connection.rb +14 -0
- data/lib/makara/errors/no_connections_available.rb +14 -0
- data/lib/makara/logging/logger.rb +23 -0
- data/lib/makara/logging/subscriber.rb +38 -0
- data/lib/makara/middleware.rb +109 -0
- data/lib/makara/pool.rb +188 -0
- data/lib/makara/proxy.rb +277 -0
- data/lib/makara/railtie.rb +14 -0
- data/lib/makara/version.rb +15 -0
- data/makara.gemspec +19 -0
- data/spec/active_record/connection_adapters/makara_abstract_adapter_error_handling_spec.rb +92 -0
- data/spec/active_record/connection_adapters/makara_abstract_adapter_spec.rb +114 -0
- data/spec/active_record/connection_adapters/makara_mysql2_adapter_spec.rb +183 -0
- data/spec/active_record/connection_adapters/makara_postgresql_adapter_spec.rb +121 -0
- data/spec/cache_spec.rb +59 -0
- data/spec/config_parser_spec.rb +102 -0
- data/spec/connection_wrapper_spec.rb +33 -0
- data/spec/context_spec.rb +107 -0
- data/spec/middleware_spec.rb +84 -0
- data/spec/pool_spec.rb +158 -0
- data/spec/proxy_spec.rb +182 -0
- data/spec/spec_helper.rb +46 -0
- data/spec/support/configurator.rb +13 -0
- data/spec/support/deep_dup.rb +12 -0
- data/spec/support/mock_objects.rb +67 -0
- data/spec/support/mysql2_database.yml +17 -0
- data/spec/support/mysql2_database_with_custom_errors.yml +17 -0
- data/spec/support/pool_extensions.rb +14 -0
- data/spec/support/postgresql_database.yml +13 -0
- data/spec/support/proxy_extensions.rb +33 -0
- data/spec/support/schema.rb +7 -0
- metadata +144 -0
data/lib/makara/pool.rb
ADDED
@@ -0,0 +1,188 @@
|
|
1
|
+
require 'active_support/core_ext/hash/keys'
|
2
|
+
|
3
|
+
# Wraps a collection of similar connections and chooses which one to use
|
4
|
+
# Uses the Makara::Context to determine if the connection needs rotation.
|
5
|
+
# Provides convenience methods for accessing underlying connections
|
6
|
+
|
7
|
+
module Makara
|
8
|
+
class Pool
|
9
|
+
|
10
|
+
# there are cases when we understand the pool is busted and we essentially want to skip
|
11
|
+
# all execution
|
12
|
+
attr_accessor :disabled
|
13
|
+
attr_reader :blacklist_errors
|
14
|
+
attr_reader :role
|
15
|
+
attr_reader :connections
|
16
|
+
|
17
|
+
def initialize(role, proxy)
|
18
|
+
@role = role
|
19
|
+
@proxy = proxy
|
20
|
+
@context = Makara::Context.get_current
|
21
|
+
@connections = []
|
22
|
+
@blacklist_errors = []
|
23
|
+
@current_idx = 0
|
24
|
+
@disabled = false
|
25
|
+
end
|
26
|
+
|
27
|
+
|
28
|
+
def completely_blacklisted?
|
29
|
+
@connections.each do |connection|
|
30
|
+
return false unless connection._makara_blacklisted?
|
31
|
+
end
|
32
|
+
true
|
33
|
+
end
|
34
|
+
|
35
|
+
|
36
|
+
# Add a connection to this pool, wrapping the connection with a Makara::ConnectionWrapper
|
37
|
+
def add(config)
|
38
|
+
config[:name] ||= "#{@role}/#{@connections.length + 1}"
|
39
|
+
|
40
|
+
connection = yield
|
41
|
+
|
42
|
+
# already wrapped because of initial error
|
43
|
+
if connection.is_a?(Makara::ConnectionWrapper)
|
44
|
+
connection.config = config # to add :name
|
45
|
+
wrapper = connection
|
46
|
+
else
|
47
|
+
wrapper = Makara::ConnectionWrapper.new(@proxy, connection, config)
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
# the weight results in N references to the connection, not N connections
|
52
|
+
wrapper._makara_weight.times{ @connections << wrapper }
|
53
|
+
|
54
|
+
if should_shuffle?
|
55
|
+
# randomize the connections so we don't get peaks and valleys of load
|
56
|
+
@connections.shuffle!
|
57
|
+
|
58
|
+
# then start at a random spot in the list
|
59
|
+
@current_idx = rand(@connections.length)
|
60
|
+
end
|
61
|
+
|
62
|
+
wrapper
|
63
|
+
end
|
64
|
+
|
65
|
+
# send this method to all available nodes
|
66
|
+
def send_to_all(method, *args)
|
67
|
+
ret = nil
|
68
|
+
provide_each do |con|
|
69
|
+
ret = con.send(method, *args)
|
70
|
+
end
|
71
|
+
ret
|
72
|
+
end
|
73
|
+
|
74
|
+
# provide all available nodes to the given block
|
75
|
+
def provide_each
|
76
|
+
idx = @current_idx
|
77
|
+
last_idx = nil
|
78
|
+
begin
|
79
|
+
provide(false) do |con|
|
80
|
+
yield con
|
81
|
+
end
|
82
|
+
return if @current_idx == last_idx
|
83
|
+
last_idx = @current_idx
|
84
|
+
end while @current_idx != idx
|
85
|
+
end
|
86
|
+
|
87
|
+
# Provide a connection that is not blacklisted and connected. Handle any errors
|
88
|
+
# that may occur within the block.
|
89
|
+
def provide(allow_stickiness = true)
|
90
|
+
provided_connection = self.next(allow_stickiness)
|
91
|
+
|
92
|
+
# nil implies that it's blacklisted
|
93
|
+
if provided_connection
|
94
|
+
|
95
|
+
value = @proxy.error_handler.handle(provided_connection) do
|
96
|
+
yield provided_connection
|
97
|
+
end
|
98
|
+
|
99
|
+
@blacklist_errors = []
|
100
|
+
|
101
|
+
value
|
102
|
+
|
103
|
+
# if we've made any connections within this pool, we should report the blackout.
|
104
|
+
elsif connection_made?
|
105
|
+
err = Makara::Errors::AllConnectionsBlacklisted.new(self, @blacklist_errors)
|
106
|
+
@blacklist_errors = []
|
107
|
+
raise err
|
108
|
+
else
|
109
|
+
raise Makara::Errors::NoConnectionsAvailable.new(@role) unless @disabled
|
110
|
+
end
|
111
|
+
|
112
|
+
# when a connection causes a blacklist error within the provided block, we blacklist it then retry
|
113
|
+
rescue Makara::Errors::BlacklistConnection => e
|
114
|
+
@blacklist_errors.insert(0, e)
|
115
|
+
provided_connection._makara_blacklist!
|
116
|
+
retry
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
|
121
|
+
protected
|
122
|
+
|
123
|
+
|
124
|
+
# have we connected to any of the underlying connections.
|
125
|
+
def connection_made?
|
126
|
+
@connections.any?(&:_makara_connected?)
|
127
|
+
end
|
128
|
+
|
129
|
+
|
130
|
+
# Get the next non-blacklisted connection. If the proxy is setup
|
131
|
+
# to be sticky, provide back the current connection assuming it is
|
132
|
+
# not blacklisted.
|
133
|
+
def next(allow_stickiness = true)
|
134
|
+
|
135
|
+
if allow_stickiness && @proxy.sticky && Makara::Context.get_current == @context
|
136
|
+
con = safe_value(@current_idx)
|
137
|
+
return con if con
|
138
|
+
end
|
139
|
+
|
140
|
+
idx = @current_idx
|
141
|
+
begin
|
142
|
+
|
143
|
+
idx = next_index(idx)
|
144
|
+
|
145
|
+
# if we've looped all the way around, return our safe value
|
146
|
+
return safe_value(idx, true) if idx == @current_idx
|
147
|
+
|
148
|
+
# while our current safe value is dangerous
|
149
|
+
end while safe_value(idx).nil?
|
150
|
+
|
151
|
+
# store our current spot and return our safe value
|
152
|
+
safe_value(idx, true)
|
153
|
+
end
|
154
|
+
|
155
|
+
|
156
|
+
# next index within the bounds of the connections array
|
157
|
+
# loop around when the end is hit
|
158
|
+
def next_index(idx)
|
159
|
+
idx = idx + 1
|
160
|
+
idx = 0 if idx >= @connections.length
|
161
|
+
idx
|
162
|
+
end
|
163
|
+
|
164
|
+
|
165
|
+
# return the connection if it's not blacklisted
|
166
|
+
# otherwise return nil
|
167
|
+
# optionally, store the position and context we're returning
|
168
|
+
def safe_value(idx, stick = false)
|
169
|
+
con = @connections[idx]
|
170
|
+
return nil unless con
|
171
|
+
return nil if con._makara_blacklisted?
|
172
|
+
|
173
|
+
if stick
|
174
|
+
@current_idx = idx
|
175
|
+
@context = Makara::Context.get_current
|
176
|
+
end
|
177
|
+
|
178
|
+
con
|
179
|
+
end
|
180
|
+
|
181
|
+
|
182
|
+
# stub in test mode to ensure consistency
|
183
|
+
def should_shuffle?
|
184
|
+
true
|
185
|
+
end
|
186
|
+
|
187
|
+
end
|
188
|
+
end
|
data/lib/makara/proxy.rb
ADDED
@@ -0,0 +1,277 @@
|
|
1
|
+
require 'delegate'
|
2
|
+
require 'active_support/core_ext/class/attribute'
|
3
|
+
require 'active_support/core_ext/hash/keys'
|
4
|
+
|
5
|
+
# The entry point of Makara. It contains a master and slave pool which are chosen based on the invocation
|
6
|
+
# being proxied. Makara::Proxy implementations should declare which methods they are hijacking via the
|
7
|
+
# `hijack_method` class method.
|
8
|
+
# While debugging this class use prepend debug calls with Kernel. (Kernel.byebug for example)
|
9
|
+
# to avoid getting into method_missing stuff.
|
10
|
+
|
11
|
+
module Makara
|
12
|
+
class Proxy < ::SimpleDelegator
|
13
|
+
|
14
|
+
METHOD_MISSING_SKIP = [ :byebug ]
|
15
|
+
|
16
|
+
class_attribute :hijack_methods
|
17
|
+
self.hijack_methods = []
|
18
|
+
|
19
|
+
class << self
|
20
|
+
def hijack_method(*method_names)
|
21
|
+
self.hijack_methods = self.hijack_methods || []
|
22
|
+
self.hijack_methods |= method_names
|
23
|
+
|
24
|
+
method_names.each do |method_name|
|
25
|
+
define_method method_name do |*args, &block|
|
26
|
+
appropriate_connection(method_name, args) do |con|
|
27
|
+
con.send(method_name, *args, &block)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def send_to_all(*method_names)
|
34
|
+
method_names.each do |method_name|
|
35
|
+
define_method method_name do |*args|
|
36
|
+
send_to_all method_name, *args
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
|
43
|
+
attr_reader :error_handler
|
44
|
+
attr_reader :sticky
|
45
|
+
|
46
|
+
def initialize(config)
|
47
|
+
@config = config.symbolize_keys
|
48
|
+
@config_parser = Makara::ConfigParser.new(@config)
|
49
|
+
@id = @config_parser.id
|
50
|
+
@ttl = @config_parser.makara_config[:master_ttl]
|
51
|
+
@sticky = @config_parser.makara_config[:sticky]
|
52
|
+
@hijacked = false
|
53
|
+
@error_handler ||= ::Makara::ErrorHandler.new
|
54
|
+
@skip_sticking = false
|
55
|
+
instantiate_connections
|
56
|
+
super(config)
|
57
|
+
end
|
58
|
+
|
59
|
+
def without_sticking
|
60
|
+
@skip_sticking = true
|
61
|
+
yield
|
62
|
+
ensure
|
63
|
+
@skip_sticking = false
|
64
|
+
end
|
65
|
+
|
66
|
+
def hijacked?
|
67
|
+
@hijacked
|
68
|
+
end
|
69
|
+
|
70
|
+
def stick_to_master!(write_to_cache = true)
|
71
|
+
@master_context = Makara::Context.get_current
|
72
|
+
Makara::Cache.write("makara::#{@master_context}-#{@id}", '1', @ttl) if write_to_cache
|
73
|
+
end
|
74
|
+
|
75
|
+
def method_missing(m, *args, &block)
|
76
|
+
if METHOD_MISSING_SKIP.include?(m)
|
77
|
+
return super(m, *args, &block)
|
78
|
+
end
|
79
|
+
|
80
|
+
any_connection do |con|
|
81
|
+
if con.respond_to?(m)
|
82
|
+
con.public_send(m, *args, &block)
|
83
|
+
elsif con.respond_to?(m, true)
|
84
|
+
con.__send__(m, *args, &block)
|
85
|
+
else
|
86
|
+
super(m, *args, &block)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
class_eval <<-RUBY_EVAL, __FILE__, __LINE__ + 1
|
92
|
+
def respond_to#{RUBY_VERSION.to_s =~ /^1.8/ ? nil : '_missing'}?(m, include_private = false)
|
93
|
+
any_connection do |con|
|
94
|
+
con._makara_connection.respond_to?(m, true)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
RUBY_EVAL
|
98
|
+
|
99
|
+
def graceful_connection_for(config)
|
100
|
+
fake_wrapper = Makara::ConnectionWrapper.new(self, nil, config)
|
101
|
+
|
102
|
+
@error_handler.handle(fake_wrapper) do
|
103
|
+
connection_for(config)
|
104
|
+
end
|
105
|
+
rescue Makara::Errors::BlacklistConnection => e
|
106
|
+
fake_wrapper.initial_error = e.original_error
|
107
|
+
fake_wrapper
|
108
|
+
end
|
109
|
+
|
110
|
+
def disconnect!
|
111
|
+
send_to_all(:disconnect!)
|
112
|
+
rescue ::Makara::Errors::AllConnectionsBlacklisted, ::Makara::Errors::NoConnectionsAvailable
|
113
|
+
# all connections are already down, nothing to do here
|
114
|
+
end
|
115
|
+
|
116
|
+
protected
|
117
|
+
|
118
|
+
|
119
|
+
def send_to_all(method_name, *args)
|
120
|
+
# slave pool must run first to allow for slave-->master failover without running operations on master twice.
|
121
|
+
handling_an_all_execution(method_name) do
|
122
|
+
@slave_pool.send_to_all method_name, *args
|
123
|
+
@master_pool.send_to_all method_name, *args
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def any_connection
|
128
|
+
@master_pool.provide(true) do |con|
|
129
|
+
yield con
|
130
|
+
end
|
131
|
+
rescue ::Makara::Errors::AllConnectionsBlacklisted, ::Makara::Errors::NoConnectionsAvailable => e
|
132
|
+
begin
|
133
|
+
@master_pool.disabled = true
|
134
|
+
@slave_pool.provide(true) do |con|
|
135
|
+
yield con
|
136
|
+
end
|
137
|
+
ensure
|
138
|
+
@master_pool.disabled = false
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
# based on the method_name and args, provide the appropriate connection
|
143
|
+
# mark this proxy as hijacked so the underlying connection does not attempt to check
|
144
|
+
# with back with this proxy.
|
145
|
+
def appropriate_connection(method_name, args)
|
146
|
+
appropriate_pool(method_name, args) do |pool|
|
147
|
+
pool.provide do |connection|
|
148
|
+
hijacked do
|
149
|
+
yield connection
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
|
156
|
+
# master or slave
|
157
|
+
def appropriate_pool(method_name, args)
|
158
|
+
|
159
|
+
# for testing purposes
|
160
|
+
pool = _appropriate_pool(method_name, args)
|
161
|
+
|
162
|
+
yield pool
|
163
|
+
|
164
|
+
rescue ::Makara::Errors::AllConnectionsBlacklisted, ::Makara::Errors::NoConnectionsAvailable => e
|
165
|
+
if pool == @master_pool
|
166
|
+
@master_pool.connections.each(&:_makara_whitelist!)
|
167
|
+
@slave_pool.connections.each(&:_makara_whitelist!)
|
168
|
+
Kernel.raise e
|
169
|
+
else
|
170
|
+
@master_pool.blacklist_errors << e
|
171
|
+
retry
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def _appropriate_pool(method_name, args)
|
176
|
+
# the args provided absolutely need master
|
177
|
+
if needs_master?(method_name, args)
|
178
|
+
stick_to_master(method_name, args)
|
179
|
+
@master_pool
|
180
|
+
|
181
|
+
# in this context, we've already stuck to master
|
182
|
+
elsif Makara::Context.get_current == @master_context
|
183
|
+
@master_pool
|
184
|
+
|
185
|
+
# the previous context stuck us to master
|
186
|
+
elsif previously_stuck_to_master?
|
187
|
+
|
188
|
+
# we're only on master because of the previous context so
|
189
|
+
# behave like we're sticking to master but store the current context
|
190
|
+
stick_to_master(method_name, args, false)
|
191
|
+
@master_pool
|
192
|
+
|
193
|
+
# all slaves are down (or empty)
|
194
|
+
elsif @slave_pool.completely_blacklisted?
|
195
|
+
stick_to_master(method_name, args)
|
196
|
+
@master_pool
|
197
|
+
|
198
|
+
# yay! use a slave
|
199
|
+
else
|
200
|
+
@slave_pool
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
# do these args require a master connection
|
205
|
+
def needs_master?(method_name, args)
|
206
|
+
true
|
207
|
+
end
|
208
|
+
|
209
|
+
|
210
|
+
def hijacked
|
211
|
+
@hijacked = true
|
212
|
+
yield
|
213
|
+
ensure
|
214
|
+
@hijacked = false
|
215
|
+
end
|
216
|
+
|
217
|
+
|
218
|
+
def previously_stuck_to_master?
|
219
|
+
return false unless @sticky
|
220
|
+
!!Makara::Cache.read("makara::#{Makara::Context.get_previous}-#{@id}")
|
221
|
+
end
|
222
|
+
|
223
|
+
|
224
|
+
def stick_to_master(method_name, args, write_to_cache = true)
|
225
|
+
# if we're already stuck to master, don't bother doing it again
|
226
|
+
return if @master_context == Makara::Context.get_current
|
227
|
+
|
228
|
+
# check to see if we're configured, bypassed, or some custom implementation has input
|
229
|
+
return unless should_stick?(method_name, args)
|
230
|
+
|
231
|
+
# do the sticking
|
232
|
+
stick_to_master!(write_to_cache)
|
233
|
+
end
|
234
|
+
|
235
|
+
|
236
|
+
# if we are configured to be sticky and we aren't bypassing stickiness
|
237
|
+
def should_stick?(method_name, args)
|
238
|
+
@sticky && !@skip_sticking
|
239
|
+
end
|
240
|
+
|
241
|
+
|
242
|
+
# use the config parser to generate a master and slave pool
|
243
|
+
def instantiate_connections
|
244
|
+
@master_pool = Makara::Pool.new('master', self)
|
245
|
+
@config_parser.master_configs.each do |master_config|
|
246
|
+
@master_pool.add master_config.merge(@config_parser.makara_config) do
|
247
|
+
graceful_connection_for(master_config)
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
@slave_pool = Makara::Pool.new('slave', self)
|
252
|
+
@config_parser.slave_configs.each do |slave_config|
|
253
|
+
@slave_pool.add slave_config.merge(@config_parser.makara_config) do
|
254
|
+
graceful_connection_for(slave_config)
|
255
|
+
end
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
def handling_an_all_execution(method_name)
|
260
|
+
yield
|
261
|
+
rescue ::Makara::Errors::NoConnectionsAvailable => e
|
262
|
+
if e.role == 'master'
|
263
|
+
Kernel.raise ::Makara::Errors::NoConnectionsAvailable.new('master and slave')
|
264
|
+
end
|
265
|
+
@slave_pool.disabled = true
|
266
|
+
yield
|
267
|
+
ensure
|
268
|
+
@slave_pool.disabled = false
|
269
|
+
end
|
270
|
+
|
271
|
+
|
272
|
+
def connection_for(config)
|
273
|
+
Kernel.raise NotImplementedError
|
274
|
+
end
|
275
|
+
|
276
|
+
end
|
277
|
+
end
|