master_slave_adapter 0.2.0 → 1.0.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1,72 @@
1
- require 'master_slave_adapter'
1
+ require 'active_record'
2
+ require 'active_record/connection_adapters/master_slave_adapter'
3
+ require 'active_record/connection_adapters/master_slave_adapter/clock'
4
+ require 'active_record/connection_adapters/mysql_adapter'
5
+
6
+ module ActiveRecord
7
+ class Base
8
+ def self.mysql_master_slave_connection(config)
9
+ ConnectionAdapters::MysqlMasterSlaveAdapter.new(config, logger)
10
+ end
11
+ end
12
+
13
+ module ConnectionAdapters
14
+ class MysqlMasterSlaveAdapter < MasterSlaveAdapter::Base
15
+ CONNECTION_ERRORS = [
16
+ Mysql::Error::CR_CONNECTION_ERROR, # query: not connected
17
+ Mysql::Error::CR_CONN_HOST_ERROR, # Can't connect to MySQL server on '%s' (%d)
18
+ Mysql::Error::CR_SERVER_GONE_ERROR, # MySQL server has gone away
19
+ Mysql::Error::CR_SERVER_LOST, # Lost connection to MySQL server during query
20
+ ]
21
+
22
+ def with_consistency(clock)
23
+ clock =
24
+ case clock
25
+ when Clock then clock
26
+ when String then Clock.parse(clock)
27
+ when nil then Clock.zero
28
+ end
29
+
30
+ super(clock)
31
+ end
32
+
33
+ # TODO: only do the actual conenction specific things here
34
+ def master_clock
35
+ conn = master_connection
36
+ if status = conn.uncached { conn.select_one("SHOW MASTER STATUS") }
37
+ Clock.new(status['File'], status['Position'])
38
+ else
39
+ Clock.infinity
40
+ end
41
+ rescue MasterUnavailable
42
+ Clock.zero
43
+ rescue
44
+ Clock.infinity
45
+ end
46
+
47
+ # TODO: only do the actual conenction specific things here
48
+ def slave_clock(conn)
49
+ if status = conn.uncached { conn.select_one("SHOW SLAVE STATUS") }
50
+ Clock.new(status['Relay_Master_Log_File'], status['Exec_Master_Log_Pos'])
51
+ else
52
+ Clock.zero
53
+ end
54
+ rescue
55
+ Clock.zero
56
+ end
57
+
58
+ private
59
+
60
+ def connection_error?(exception)
61
+ case exception
62
+ when ActiveRecord::StatementInvalid
63
+ CONNECTION_ERRORS.include?(current_connection.raw_connection.errno)
64
+ when Mysql::Error
65
+ CONNECTION_ERRORS.include?(exception.errno)
66
+ else
67
+ false
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
@@ -1,624 +1 @@
1
- require 'active_record'
2
-
3
- module ActiveRecord
4
- class MasterUnavailable < ConnectionNotEstablished; end
5
-
6
- class Base
7
- class << self
8
- def with_consistency(clock, &blk)
9
- if connection.respond_to? :with_consistency
10
- connection.with_consistency(clock, &blk)
11
- else
12
- yield
13
- nil
14
- end
15
- end
16
-
17
- def with_master(&blk)
18
- if connection.respond_to? :with_master
19
- connection.with_master(&blk)
20
- else
21
- yield
22
- end
23
- end
24
-
25
- def with_slave(&blk)
26
- if connection.respond_to? :with_slave
27
- connection.with_slave(&blk)
28
- else
29
- yield
30
- end
31
- end
32
-
33
- def on_commit(&blk)
34
- connection.on_commit(&blk) if connection.respond_to? :on_commit
35
- end
36
-
37
- def on_rollback(&blk)
38
- connection.on_rollback(&blk) if connection.respond_to? :on_rollback
39
- end
40
-
41
- def master_slave_connection(config)
42
- config = massage(config)
43
- load_adapter(config.fetch(:connection_adapter))
44
- ConnectionAdapters::MasterSlaveAdapter.new(config, logger)
45
- end
46
-
47
- def mysql_master_slave_connection(config)
48
- master_slave_connection(config)
49
- end
50
-
51
- private
52
-
53
- def massage(config)
54
- config = config.symbolize_keys
55
- skip = [ :adapter, :connection_adapter, :master, :slaves ]
56
- defaults = config.
57
- reject { |k,_| skip.include?(k) }.
58
- merge(:adapter => config.fetch(:connection_adapter))
59
- ([config.fetch(:master)] + config.fetch(:slaves, [])).map do |cfg|
60
- cfg.symbolize_keys!.reverse_merge!(defaults)
61
- end
62
- config
63
- end
64
-
65
- def load_adapter(adapter_name)
66
- unless respond_to?("#{adapter_name}_connection")
67
- begin
68
- require 'rubygems'
69
- gem "activerecord-#{adapter_name}-adapter"
70
- require "active_record/connection_adapters/#{adapter_name}_adapter"
71
- rescue LoadError
72
- begin
73
- require "active_record/connection_adapters/#{adapter_name}_adapter"
74
- rescue LoadError
75
- raise %Q{Please install the #{adapter_name} adapter:
76
- `gem install activerecord-#{adapter_name}-adapter` (#{$!})"}
77
- end
78
- end
79
- end
80
- end
81
- end
82
- end
83
-
84
- module ConnectionAdapters
85
- class AbstractAdapter
86
- alias_method :orig_log_info, :log_info
87
- def log_info(sql, name, ms)
88
- connection_name =
89
- [ @config[:name], @config[:host], @config[:port] ].compact.join(":")
90
- orig_log_info sql, "[#{connection_name}] #{name || 'SQL'}", ms
91
- end
92
- end
93
-
94
- class MasterSlaveAdapter < AbstractAdapter
95
- class Clock
96
- include Comparable
97
- attr_reader :file, :position
98
-
99
- def initialize(file, position)
100
- raise ArgumentError, "file and postion may not be nil" if file.nil? || position.nil?
101
- @file, @position = file, position.to_i
102
- end
103
-
104
- def <=>(other)
105
- @file == other.file ? @position <=> other.position : @file <=> other.file
106
- end
107
-
108
- def to_s
109
- [ @file, @position ].join('@')
110
- end
111
-
112
- def self.zero
113
- @zero ||= Clock.new('', 0)
114
- end
115
-
116
- def self.infinity
117
- @infinity ||= Clock.new('', Float::MAX.to_i)
118
- end
119
- end
120
-
121
- checkout :active?
122
-
123
- def initialize(config, logger)
124
- super(nil, logger)
125
-
126
- @config = config
127
-
128
- @connections = {}
129
- @connections[:master] = connect_to_master
130
- @connections[:slaves] = @config.fetch(:slaves).map { |cfg| connect(cfg, :slave) }
131
-
132
- @disable_connection_test = @config[:disable_connection_test] == 'true'
133
-
134
- self.current_connection = slave_connection!
135
- end
136
-
137
- # MASTER SLAVE ADAPTER INTERFACE ========================================
138
-
139
- def with_master
140
- with(master_connection) { yield }
141
- end
142
-
143
- def with_slave
144
- with(slave_connection!) { yield }
145
- end
146
-
147
- def with_consistency(clock)
148
- raise ArgumentError, "consistency cannot be nil" if clock.nil?
149
- # try random slave, else fall back to master
150
- slave = slave_connection!
151
- conn =
152
- if !open_transaction? && slave_consistent?(slave, clock)
153
- slave
154
- else
155
- master_connection
156
- end
157
-
158
- with(conn) { yield }
159
-
160
- self.current_clock || clock
161
- end
162
-
163
- def on_commit(&blk)
164
- on_commit_callbacks.push blk
165
- end
166
-
167
- def on_rollback(&blk)
168
- on_rollback_callbacks.push blk
169
- end
170
-
171
-
172
- # backwards compatibility
173
- class << self
174
- def with_master(&blk)
175
- ActiveRecord::Base.with_master(&blk)
176
- end
177
- def with_slave(&blk)
178
- ActiveRecord::Base.with_slave(&blk)
179
- end
180
- def with_consistency(clock, &blk)
181
- ActiveRecord::Base.with_consistency(clock, &blk)
182
- end
183
- def reset!
184
- Thread.current[:master_slave_clock] =
185
- Thread.current[:master_slave_connection] =
186
- Thread.current[:on_commit_callbacks] =
187
- Thread.current[:on_rollback_callbacks] =
188
- nil
189
- end
190
- end
191
-
192
- # ADAPTER INTERFACE OVERRIDES ===========================================
193
-
194
- def insert(*args)
195
- on_write { |conn| conn.insert(*args) }
196
- end
197
-
198
- def update(*args)
199
- on_write { |conn| conn.update(*args) }
200
- end
201
-
202
- def delete(*args)
203
- on_write { |conn| conn.delete(*args) }
204
- end
205
-
206
- def commit_db_transaction
207
- on_write { |conn| conn.commit_db_transaction }
208
- on_commit_callbacks.shift.call(current_clock) until on_commit_callbacks.blank?
209
- end
210
-
211
- def rollback_db_transaction
212
- on_commit_callbacks.clear
213
- with(master_connection) { |conn| conn.rollback_db_transaction }
214
- on_rollback_callbacks.shift.call until on_rollback_callbacks.blank?
215
- end
216
-
217
- def active?
218
- return true if @disable_connection_test
219
- connections.map { |c| c.active? }.all?
220
- end
221
-
222
- def reconnect!
223
- connections.each { |c| c.reconnect! }
224
- end
225
-
226
- def disconnect!
227
- connections.each { |c| c.disconnect! }
228
- end
229
-
230
- def reset!
231
- connections.each { |c| c.reset! }
232
- end
233
-
234
- def cache(&block)
235
- connections.inject(block) do |block, connection|
236
- lambda { connection.cache(&block) }
237
- end.call
238
- end
239
-
240
- def uncached(&block)
241
- connections.inject(block) do |block, connection|
242
- lambda { connection.uncached(&block) }
243
- end.call
244
- end
245
-
246
- def clear_query_cache
247
- connections.each { |connection| connection.clear_query_cache }
248
- end
249
-
250
- # Someone calling execute directly on the connection is likely to be a
251
- # write, respectively some DDL statement. People really shouldn't do that,
252
- # but let's delegate this to master, just to be sure.
253
- def execute(*args)
254
- on_write { |conn| conn.execute(*args) }
255
- end
256
-
257
- # ADAPTER INTERFACE DELEGATES ===========================================
258
-
259
- def self.rescued_delegate(*methods, options)
260
- to, fallback = options.values_at(:to, :fallback)
261
-
262
- file, line = caller.first.split(':', 2)
263
- line = line.to_i
264
-
265
- methods.each do |method|
266
- module_eval(<<-EOS, file, line)
267
- def #{method}(*args, &block)
268
- begin
269
- #{to}.__send__(:#{method}, *args, &block)
270
- rescue MasterUnavailable
271
- #{fallback ? "#{fallback}.__send__(:#{method}, *args, *block)" : "raise"}
272
- rescue => exception
273
- if master_connection?(#{to}) && connection_error?(exception)
274
- reset_master_connection
275
- #{fallback ? "#{fallback}.__send__(:#{method}, *args, *block)" : "raise MasterUnavailable"}
276
- else
277
- raise
278
- end
279
- end
280
- end
281
- EOS
282
- end
283
- end
284
- class << self; private :rescued_delegate; end
285
-
286
- # === must go to master
287
- rescued_delegate :adapter_name,
288
- :supports_migrations?,
289
- :supports_primary_key?,
290
- :supports_savepoints?,
291
- :native_database_types,
292
- :raw_connection,
293
- :open_transactions,
294
- :increment_open_transactions,
295
- :decrement_open_transactions,
296
- :transaction_joinable=,
297
- :create_savepoint,
298
- :rollback_to_savepoint,
299
- :release_savepoint,
300
- :current_savepoint_name,
301
- :begin_db_transaction,
302
- :outside_transaction?,
303
- :add_limit!,
304
- :default_sequence_name,
305
- :reset_sequence!,
306
- :insert_fixture,
307
- :empty_insert_statement,
308
- :case_sensitive_equality_operator,
309
- :limited_update_conditions,
310
- :insert_sql,
311
- :update_sql,
312
- :delete_sql,
313
- :sanitize_limit,
314
- :to => :master_connection
315
- # schema statements
316
- rescued_delegate :table_exists?,
317
- :create_table,
318
- :change_table,
319
- :rename_table,
320
- :drop_table,
321
- :add_column,
322
- :remove_column,
323
- :remove_columns,
324
- :change_column,
325
- :change_column_default,
326
- :rename_column,
327
- :add_index,
328
- :remove_index,
329
- :remove_index!,
330
- :rename_index,
331
- :index_name,
332
- :index_exists?,
333
- :structure_dump,
334
- :dump_schema_information,
335
- :initialize_schema_migrations_table,
336
- :assume_migrated_upto_version,
337
- :type_to_sql,
338
- :add_column_options!,
339
- :distinct,
340
- :add_order_by_for_association_limiting!,
341
- :add_timestamps,
342
- :remove_timestamps,
343
- :quoted_columns_for_index,
344
- :options_include_default?,
345
- :to => :master_connection
346
- # ActiveRecord 3.0
347
- rescued_delegate :visitor,
348
- :to => :master_connection
349
- # no clear interface contract:
350
- rescued_delegate :tables, # commented in SchemaStatements
351
- :truncate_table, # monkeypatching database_cleaner gem
352
- :primary_key, # is Base#primary_key meant to be the contract?
353
- :to => :master_connection
354
- # No need to be so picky about these methods
355
- rescued_delegate :add_limit_offset!, # DatabaseStatements
356
- :add_lock!, #DatabaseStatements
357
- :columns,
358
- :table_alias_for,
359
- :to => :master_connection,
360
- :fallback => :slave_connection!
361
-
362
- # ok, we might have missed more
363
- def method_missing(name, *args, &blk)
364
- master_connection.send(name.to_sym, *args, &blk).tap do
365
- @logger.try(:warn, %Q{
366
- You called the unsupported method '#{name}' on #{self.class.name}.
367
- In order to help us improve master_slave_adapter, please report this
368
- to: https://github.com/soundcloud/master_slave_adapter/issues
369
-
370
- Thank you.
371
- })
372
- end
373
- rescue => exception
374
- if connection_error?(exception)
375
- reset_master_connection
376
- raise MasterUnavailable
377
- else
378
- raise
379
- end
380
- end
381
-
382
- # === determine read connection
383
- rescued_delegate :select_all,
384
- :select_one,
385
- :select_rows,
386
- :select_value,
387
- :select_values,
388
- :to => :connection_for_read
389
-
390
- def connection_for_read
391
- open_transaction? ? master_connection : current_connection
392
- end
393
- private :connection_for_read
394
-
395
- # === doesn't really matter, but must be handled by underlying adapter
396
- delegate *(ActiveRecord::ConnectionAdapters::Quoting.instance_methods + [{
397
- :to => :current_connection }])
398
- # issue #4: current_database is not supported by all adapters, though
399
- delegate :current_database, :to => :current_connection
400
-
401
- # UTIL ==================================================================
402
-
403
- def master_connection
404
- if circuit.is_tripped?
405
- raise MasterUnavailable
406
- end
407
-
408
- @connections[:master] ||= connect_to_master
409
- if @connections[:master]
410
- circuit.success!
411
- @connections[:master]
412
- else
413
- circuit.fail!
414
- raise MasterUnavailable
415
- end
416
- end
417
-
418
- def master_connection?(connection)
419
- @connections[:master] == connection
420
- end
421
-
422
- def master_available?
423
- !@connections[:master].nil?
424
- end
425
-
426
- def reset_master_connection
427
- @connections[:master] = nil
428
- end
429
-
430
- # Returns a random slave connection
431
- # Note: the method is not referentially transparent, hence the bang
432
- def slave_connection!
433
- @connections[:slaves].sample
434
- end
435
-
436
- def connections
437
- @connections.values.inject([]) { |m,c| m << c }.flatten.compact
438
- end
439
-
440
- def current_connection
441
- connection_stack.first
442
- end
443
-
444
- def current_clock
445
- Thread.current[:master_slave_clock]
446
- end
447
-
448
- def current_clock=(clock)
449
- Thread.current[:master_slave_clock] = clock
450
- end
451
-
452
- def master_clock
453
- conn = master_connection
454
- # TODO: should be extracted into adapter specific code
455
- if status = conn.uncached { conn.select_one("SHOW MASTER STATUS") }
456
- Clock.new(status['File'], status['Position'])
457
- end
458
- end
459
-
460
- def slave_clock!
461
- slave_clock(slave_connection!)
462
- end
463
-
464
- def slave_clock(conn)
465
- # TODO: should be extracted into adapter specific code
466
- if status = conn.uncached { conn.select_one("SHOW SLAVE STATUS") }
467
- Clock.new(status['Relay_Master_Log_File'], status['Exec_Master_Log_Pos']).tap do |c|
468
- set_last_seen_slave_clock(conn, c)
469
- end
470
- end
471
- end
472
-
473
- def slave_consistent?(conn, clock)
474
- get_last_seen_slave_clock(conn).try(:>=, clock) ||
475
- slave_clock(conn).try(:>=, clock)
476
- end
477
-
478
- protected
479
-
480
- def on_write
481
- with(master_connection) do |conn|
482
- yield(conn).tap do
483
- unless open_transaction?
484
- if mc = master_clock
485
- self.current_clock = mc unless current_clock.try(:>=, mc)
486
- end
487
- # keep using master after write
488
- connection_stack.replace([ conn ])
489
- end
490
- end
491
- end
492
- end
493
-
494
- def with(conn)
495
- self.current_connection = conn
496
- yield(conn).tap { connection_stack.shift if connection_stack.size > 1 }
497
- end
498
-
499
- private
500
-
501
- def connect(cfg, name)
502
- adapter_method = "#{cfg.fetch(:adapter)}_connection".to_sym
503
- ActiveRecord::Base.send(adapter_method, { :name => name }.merge(cfg))
504
- end
505
-
506
- def open_transaction?
507
- master_available? ? (master_connection.open_transactions > 0) : false
508
- end
509
-
510
- def connection_stack
511
- Thread.current[:master_slave_connection] ||= []
512
- end
513
-
514
- def current_connection=(conn)
515
- connection_stack.unshift(conn)
516
- end
517
-
518
- def on_commit_callbacks
519
- Thread.current[:on_commit_callbacks] ||= []
520
- end
521
-
522
- def on_rollback_callbacks
523
- Thread.current[:on_rollback_callbacks] ||= []
524
- end
525
-
526
- def get_last_seen_slave_clock(conn)
527
- conn.instance_variable_get(:@last_seen_slave_clock)
528
- end
529
-
530
- def set_last_seen_slave_clock(conn, clock)
531
- last_seen = get_last_seen_slave_clock(conn)
532
- if last_seen.nil? || last_seen < clock
533
- conn.instance_variable_set(:@last_seen_slave_clock, clock)
534
- end
535
- end
536
-
537
- def connect_to_master
538
- connect(@config.fetch(:master), :master)
539
- rescue => exception
540
- connection_error?(exception) ? nil : raise
541
- end
542
-
543
- # TODO: should be extracted into adapter specific code
544
- def connection_error?(exception)
545
- connection_errors = [
546
- Mysql::Error::CR_CONNECTION_ERROR, # query: not connected
547
- Mysql::Error::CR_CONN_HOST_ERROR, # Can't connect to MySQL server on '%s' (%d)
548
- Mysql::Error::CR_SERVER_GONE_ERROR, # MySQL server has gone away
549
- Mysql::Error::CR_SERVER_LOST, # Lost connection to MySQL server during query
550
- ]
551
-
552
- case exception
553
- when ActiveRecord::StatementInvalid
554
- connection_errors.include?(current_connection.raw_connection.errno)
555
- when Mysql::Error
556
- connection_errors.include?(exception.errno)
557
- else
558
- false
559
- end
560
- end
561
-
562
- class CircuitBreaker
563
- def initialize(logger = nil, failure_threshold = 5, invokation_timeout = 30)
564
- @logger = logger
565
- @failure_count = 0
566
- @failure_threshold = failure_threshold
567
- @invokation_timeout = invokation_timeout
568
- @state = :closed
569
- end
570
-
571
- def open?
572
- :open == @state
573
- end
574
-
575
- def half_open?
576
- :half_open == @state
577
- end
578
-
579
- def closed?
580
- :closed == @state
581
- end
582
-
583
- def is_tripped?
584
- if open? && timeout_exceeded?
585
- change_state_to :half_open
586
- end
587
-
588
- open?
589
- end
590
-
591
- def success!
592
- if !closed?
593
- @failure_count = 0
594
- change_state_to :closed
595
- end
596
- end
597
-
598
- def fail!
599
- @failure_count += 1
600
- if !open? && @failure_count >= @failure_threshold
601
- @opened_at = Time.now
602
- change_state_to :open
603
- end
604
- end
605
-
606
- private
607
-
608
- def timeout_exceeded?
609
- (Time.now - @opened_at) >= @invokation_timeout
610
- end
611
-
612
- def change_state_to(state)
613
- @state = state
614
- @logger.try(:warn, "circuit is now #{state}")
615
- end
616
- end
617
-
618
- # TODO: pass configuration values
619
- def circuit
620
- @circuit ||= CircuitBreaker.new(@logger)
621
- end
622
- end
623
- end
624
- end
1
+ require 'active_record/connection_adapters/master_slave_adapter'