ar_mysql_flexmaster 0.3.1 → 0.4.0

data/.travis.yml

@@ -1,6 +1,5 @@
 rvm:
   - 1.9.3
-  - 2.0
 gemfile:
   - gemfiles/rails2.gemfile
   - gemfiles/rails3.gemfile

data/Gemfile

@@ -2,6 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in ar_mysql_flexmaster.gemspec
 gemspec
-gem "debugger", :platform => :ruby_19
+gem "debugger", "~>1.5.0"
 gem "appraisal"
-

data/ar_mysql_flexmaster.gemspec

@@ -12,12 +12,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "ar_mysql_flexmaster"
   gem.require_paths = ["lib"]
-  gem.version       = "0.3.1"
+  gem.version       = "0.4.0"
 
   gem.add_runtime_dependency("mysql2")
   gem.add_runtime_dependency("activerecord")
   gem.add_runtime_dependency("activesupport")
   gem.add_development_dependency("appraisal")
   gem.add_development_dependency("yaggy")
-  gem.add_development_dependency("mysql_isolated_server")
+  gem.add_development_dependency("mysql_isolated_server", "~> 0.1.1")
 end

data/gemfiles/rails2.gemfile

@@ -2,7 +2,7 @@
 
 source "https://rubygems.org"
 
-gem "debugger", :platform=>:ruby_19
+gem "debugger", "~>1.5.0"
 gem "appraisal"
 gem "rails", "~> 2.3.15"
 gem "mysql2", "~> 0.2.0"

data/gemfiles/rails2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: /Users/ben/src/ar_mysql_flexmaster
   specs:
-    ar_mysql_flexmaster (0.3.0)
+    ar_mysql_flexmaster (0.3.1)
       activerecord
       activesupport
       mysql2
@@ -9,38 +9,37 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionmailer (2.3.15)
-      actionpack (= 2.3.15)
-    actionpack (2.3.15)
-      activesupport (= 2.3.15)
-      rack (~> 1.1.3)
-    activerecord (2.3.15)
-      activesupport (= 2.3.15)
-    activeresource (2.3.15)
-      activesupport (= 2.3.15)
-    activesupport (2.3.15)
-    appraisal (0.5.1)
+    actionmailer (2.3.18)
+      actionpack (= 2.3.18)
+    actionpack (2.3.18)
+      activesupport (= 2.3.18)
+      rack (~> 1.1.0)
+    activerecord (2.3.18)
+      activesupport (= 2.3.18)
+    activeresource (2.3.18)
+      activesupport (= 2.3.18)
+    activesupport (2.3.18)
+    appraisal (0.5.2)
       bundler
       rake
     columnize (0.3.6)
-    debugger (1.2.3)
+    debugger (1.5.0)
       columnize (>= 0.3.1)
-      debugger-linecache (~> 1.1.1)
-      debugger-ruby_core_source (~> 1.1.5)
-    debugger-linecache (1.1.2)
-      debugger-ruby_core_source (>= 1.1.1)
-    debugger-ruby_core_source (1.1.6)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.2.0)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.2.0)
     mysql2 (0.2.18)
-    mysql_isolated_server (0.0.2)
-    rack (1.1.5)
-    rails (2.3.15)
-      actionmailer (= 2.3.15)
-      actionpack (= 2.3.15)
-      activerecord (= 2.3.15)
-      activeresource (= 2.3.15)
-      activesupport (= 2.3.15)
+    mysql_isolated_server (0.1.1)
+    rack (1.1.6)
+    rails (2.3.18)
+      actionmailer (= 2.3.18)
+      actionpack (= 2.3.18)
+      activerecord (= 2.3.18)
+      activeresource (= 2.3.18)
+      activesupport (= 2.3.18)
       rake (>= 0.8.3)
-    rake (10.0.3)
+    rake (10.0.4)
     yaggy (0.1.7)
 
 PLATFORMS
@@ -49,8 +48,8 @@ PLATFORMS
 DEPENDENCIES
   appraisal
   ar_mysql_flexmaster!
-  debugger
+  debugger (~> 1.5.0)
   mysql2 (~> 0.2.0)
-  mysql_isolated_server
+  mysql_isolated_server (~> 0.1.1)
   rails (~> 2.3.15)
   yaggy

data/gemfiles/rails3.2.gemfile

@@ -2,7 +2,7 @@
 
 source "https://rubygems.org"
 
-gem "debugger", :platform=>:ruby_19
+gem "debugger", "~>1.5.0"
 gem "appraisal"
 gem "rails", "~> 3.2.0"
 gem "mysql2", "~> 0.3.0"

data/gemfiles/rails3.2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: /Users/ben/src/ar_mysql_flexmaster
   specs:
-    ar_mysql_flexmaster (0.3.0)
+    ar_mysql_flexmaster (0.3.1)
       activerecord
       activesupport
       mysql2
@@ -9,96 +9,95 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionmailer (3.2.11)
-      actionpack (= 3.2.11)
-      mail (~> 2.4.4)
-    actionpack (3.2.11)
-      activemodel (= 3.2.11)
-      activesupport (= 3.2.11)
+    actionmailer (3.2.13)
+      actionpack (= 3.2.13)
+      mail (~> 2.5.3)
+    actionpack (3.2.13)
+      activemodel (= 3.2.13)
+      activesupport (= 3.2.13)
       builder (~> 3.0.0)
       erubis (~> 2.7.0)
       journey (~> 1.0.4)
-      rack (~> 1.4.0)
+      rack (~> 1.4.5)
       rack-cache (~> 1.2)
       rack-test (~> 0.6.1)
       sprockets (~> 2.2.1)
-    activemodel (3.2.11)
-      activesupport (= 3.2.11)
+    activemodel (3.2.13)
+      activesupport (= 3.2.13)
       builder (~> 3.0.0)
-    activerecord (3.2.11)
-      activemodel (= 3.2.11)
-      activesupport (= 3.2.11)
+    activerecord (3.2.13)
+      activemodel (= 3.2.13)
+      activesupport (= 3.2.13)
       arel (~> 3.0.2)
       tzinfo (~> 0.3.29)
-    activeresource (3.2.11)
-      activemodel (= 3.2.11)
-      activesupport (= 3.2.11)
-    activesupport (3.2.11)
-      i18n (~> 0.6)
+    activeresource (3.2.13)
+      activemodel (= 3.2.13)
+      activesupport (= 3.2.13)
+    activesupport (3.2.13)
+      i18n (= 0.6.1)
       multi_json (~> 1.0)
-    appraisal (0.5.1)
+    appraisal (0.5.2)
       bundler
       rake
     arel (3.0.2)
     builder (3.0.4)
     columnize (0.3.6)
-    debugger (1.2.3)
+    debugger (1.5.0)
       columnize (>= 0.3.1)
-      debugger-linecache (~> 1.1.1)
-      debugger-ruby_core_source (~> 1.1.5)
-    debugger-linecache (1.1.2)
-      debugger-ruby_core_source (>= 1.1.1)
-    debugger-ruby_core_source (1.1.6)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.2.0)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.2.0)
     erubis (2.7.0)
-    hike (1.2.1)
+    hike (1.2.2)
     i18n (0.6.1)
     journey (1.0.4)
-    json (1.7.6)
-    mail (2.4.4)
+    json (1.7.7)
+    mail (2.5.3)
       i18n (>= 0.4.0)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
-    mime-types (1.19)
-    multi_json (1.5.0)
+    mime-types (1.23)
+    multi_json (1.7.2)
     mysql2 (0.3.11)
-    mysql_isolated_server (0.0.2)
+    mysql_isolated_server (0.1.1)
     polyglot (0.3.3)
-    rack (1.4.4)
+    rack (1.4.5)
     rack-cache (1.2)
       rack (>= 0.4)
-    rack-ssl (1.3.2)
+    rack-ssl (1.3.3)
       rack
     rack-test (0.6.2)
       rack (>= 1.0)
-    rails (3.2.11)
-      actionmailer (= 3.2.11)
-      actionpack (= 3.2.11)
-      activerecord (= 3.2.11)
-      activeresource (= 3.2.11)
-      activesupport (= 3.2.11)
+    rails (3.2.13)
+      actionmailer (= 3.2.13)
+      actionpack (= 3.2.13)
+      activerecord (= 3.2.13)
+      activeresource (= 3.2.13)
+      activesupport (= 3.2.13)
       bundler (~> 1.0)
-      railties (= 3.2.11)
-    railties (3.2.11)
-      actionpack (= 3.2.11)
-      activesupport (= 3.2.11)
+      railties (= 3.2.13)
+    railties (3.2.13)
+      actionpack (= 3.2.13)
+      activesupport (= 3.2.13)
       rack-ssl (~> 1.3.2)
       rake (>= 0.8.7)
       rdoc (~> 3.4)
       thor (>= 0.14.6, < 2.0)
-    rake (10.0.3)
-    rdoc (3.12)
+    rake (10.0.4)
+    rdoc (3.12.2)
       json (~> 1.4)
     sprockets (2.2.2)
       hike (~> 1.2)
       multi_json (~> 1.0)
       rack (~> 1.0)
       tilt (~> 1.1, != 1.3.0)
-    thor (0.16.0)
-    tilt (1.3.3)
+    thor (0.18.1)
+    tilt (1.3.7)
     treetop (1.4.12)
       polyglot
       polyglot (>= 0.3.1)
-    tzinfo (0.3.35)
+    tzinfo (0.3.37)
     yaggy (0.1.7)
 
 PLATFORMS
@@ -107,8 +106,8 @@ PLATFORMS
 DEPENDENCIES
   appraisal
   ar_mysql_flexmaster!
-  debugger
+  debugger (~> 1.5.0)
   mysql2 (~> 0.3.0)
-  mysql_isolated_server
+  mysql_isolated_server (~> 0.1.1)
   rails (~> 3.2.0)
   yaggy

data/gemfiles/rails3.gemfile

@@ -2,7 +2,7 @@
 
 source "https://rubygems.org"
 
-gem "debugger", :platform=>:ruby_19
+gem "debugger", "~>1.5.0"
 gem "appraisal"
 gem "rails", "~> 3.0.0"
 gem "mysql2", "~> 0.2.0"

data/gemfiles/rails3.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: /Users/ben/src/ar_mysql_flexmaster
   specs:
-    ar_mysql_flexmaster (0.3.0)
+    ar_mysql_flexmaster (0.3.1)
       activerecord
       activesupport
       mysql2
@@ -10,12 +10,12 @@ GEM
   remote: https://rubygems.org/
   specs:
     abstract (1.0.0)
-    actionmailer (3.0.19)
-      actionpack (= 3.0.19)
+    actionmailer (3.0.20)
+      actionpack (= 3.0.20)
       mail (~> 2.2.19)
-    actionpack (3.0.19)
-      activemodel (= 3.0.19)
-      activesupport (= 3.0.19)
+    actionpack (3.0.20)
+      activemodel (= 3.0.20)
+      activesupport (= 3.0.20)
       builder (~> 2.1.2)
       erubis (~> 2.6.6)
       i18n (~> 0.5.0)
@@ -23,72 +23,71 @@ GEM
       rack-mount (~> 0.6.14)
       rack-test (~> 0.5.7)
       tzinfo (~> 0.3.23)
-    activemodel (3.0.19)
-      activesupport (= 3.0.19)
+    activemodel (3.0.20)
+      activesupport (= 3.0.20)
       builder (~> 2.1.2)
       i18n (~> 0.5.0)
-    activerecord (3.0.19)
-      activemodel (= 3.0.19)
-      activesupport (= 3.0.19)
+    activerecord (3.0.20)
+      activemodel (= 3.0.20)
+      activesupport (= 3.0.20)
       arel (~> 2.0.10)
       tzinfo (~> 0.3.23)
-    activeresource (3.0.19)
-      activemodel (= 3.0.19)
-      activesupport (= 3.0.19)
-    activesupport (3.0.19)
-    appraisal (0.5.1)
+    activeresource (3.0.20)
+      activemodel (= 3.0.20)
+      activesupport (= 3.0.20)
+    activesupport (3.0.20)
+    appraisal (0.5.2)
       bundler
       rake
     arel (2.0.10)
     builder (2.1.2)
     columnize (0.3.6)
-    debugger (1.2.3)
+    debugger (1.5.0)
       columnize (>= 0.3.1)
-      debugger-linecache (~> 1.1.1)
-      debugger-ruby_core_source (~> 1.1.5)
-    debugger-linecache (1.1.2)
-      debugger-ruby_core_source (>= 1.1.1)
-    debugger-ruby_core_source (1.1.6)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.2.0)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.2.0)
     erubis (2.6.6)
       abstract (>= 1.0.0)
     i18n (0.5.0)
-    json (1.7.6)
+    json (1.7.7)
     mail (2.2.19)
       activesupport (>= 2.3.6)
       i18n (>= 0.4.0)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
-    mime-types (1.19)
+    mime-types (1.23)
     mysql2 (0.2.18)
-    mysql_isolated_server (0.0.2)
+    mysql_isolated_server (0.1.1)
     polyglot (0.3.3)
-    rack (1.2.7)
+    rack (1.2.8)
     rack-mount (0.6.14)
       rack (>= 1.0.0)
     rack-test (0.5.7)
       rack (>= 1.0)
-    rails (3.0.19)
-      actionmailer (= 3.0.19)
-      actionpack (= 3.0.19)
-      activerecord (= 3.0.19)
-      activeresource (= 3.0.19)
-      activesupport (= 3.0.19)
+    rails (3.0.20)
+      actionmailer (= 3.0.20)
+      actionpack (= 3.0.20)
+      activerecord (= 3.0.20)
+      activeresource (= 3.0.20)
+      activesupport (= 3.0.20)
       bundler (~> 1.0)
-      railties (= 3.0.19)
-    railties (3.0.19)
-      actionpack (= 3.0.19)
-      activesupport (= 3.0.19)
+      railties (= 3.0.20)
+    railties (3.0.20)
+      actionpack (= 3.0.20)
+      activesupport (= 3.0.20)
       rake (>= 0.8.7)
       rdoc (~> 3.4)
       thor (~> 0.14.4)
-    rake (10.0.3)
-    rdoc (3.12)
+    rake (10.0.4)
+    rdoc (3.12.2)
       json (~> 1.4)
     thor (0.14.6)
     treetop (1.4.12)
       polyglot
       polyglot (>= 0.3.1)
-    tzinfo (0.3.35)
+    tzinfo (0.3.37)
     yaggy (0.1.7)
 
 PLATFORMS
@@ -97,8 +96,8 @@ PLATFORMS
 DEPENDENCIES
   appraisal
   ar_mysql_flexmaster!
-  debugger
+  debugger (~> 1.5.0)
   mysql2 (~> 0.2.0)
-  mysql_isolated_server
+  mysql_isolated_server (~> 0.1.1)
   rails (~> 3.0.0)
   yaggy

data/lib/active_record/connection_adapters/mysql_flexmaster_adapter.rb

@@ -35,36 +35,37 @@ module ActiveRecord
       def initialize(logger, config)
         @select_counter = 0
         @config = config
-        @is_master = !config[:slave]
+        @rw = config[:slave] ? :read : :write
         @tx_hold_timeout = @config[:tx_hold_timeout] || DEFAULT_TX_HOLD_TIMEOUT
         @connection_timeout = @config[:connection_timeout] || DEFAULT_CONNECT_TIMEOUT
 
-        connection = find_correct_host
+        connection = find_correct_host(@rw)
 
         raise_no_server_available! unless connection
         super(connection, logger, [], config)
       end
 
       def begin_db_transaction
-        if !cx_correct? && open_transactions == 0
-          refind_correct_host!
+        if !in_transaction?
+          with_lost_cx_guard { hard_verify }
         end
         super
       end
 
       def execute(sql, name = nil)
-        if open_transactions == 0 && sql =~ /^(INSERT|UPDATE|DELETE|ALTER|CHANGE)/ && !cx_correct?
-          refind_correct_host!
+        if in_transaction?
+          super # no way to rescue any lost cx or wrong-host errors at this point.
         else
-          @select_counter += 1
-          if (@select_counter % CHECK_EVERY_N_SELECTS == 0) && !cx_correct?
-            # on select statements, check every 10 times to see if we need to switch masters,
-            # but don't sleep, and if existing connection isn't correct, go ahead anyway.
-            cx = find_correct_host
-            @connection = cx if cx
+          with_lost_cx_guard do
+            if has_side_effects?(sql)
+              hard_verify
+            else
+              soft_verify
+            end
+
+            super
           end
         end
-        super
       end
 
       def current_host
@@ -77,9 +78,77 @@ module ActiveRecord
 
       private
 
+      def in_transaction?
+        open_transactions > 0
+      end
+
+      # never try to carry on if inside a transaction
+      # otherwise try to detect when the master/slave has crashed and retry stuff.
+      def with_lost_cx_guard
+        retried = false
+
+        begin
+          yield
+        rescue Mysql2::Error, ActiveRecord::StatementInvalid => e
+          if retryable_error?(e) && !retried
+            retried = true
+            @connection = nil
+            retry
+          else
+            raise e
+          end
+        end
+      end
+
+      AR_MESSAGES = [ /^Mysql2::Error: MySQL server has gone away/,
+                      /^Mysql2::Error: Can't connect to MySQL server/ ]
+      def retryable_error?(e)
+        case e
+        when Mysql2::Error
+          # 2006 is gone-away, 61 is can't-connect (for reconnection: true connections)
+          [2006, 61].include?(e.errno)
+        when ActiveRecord::StatementInvalid
+          AR_MESSAGES.any? { |m| e.message.match(m) }
+        end
+      end
+
+      # when either doing BEGIN or INSERT/UPDATE/DELETE etc, ensure a correct connection
+      # and crash if wrong
+      def hard_verify
+        if !@connection || !cx_correct?
+          refind_correct_host!
+        end
+      end
+
+      # on select statements, check every 10 statements to see if we need to switch hosts,
+      # but don't crash if the cx is wrong, and don't sleep trying to find a correct one.
+      def soft_verify
+        if !@connection
+          @connection = find_correct_host(@rw)
+        else
+          @select_counter += 1
+          return unless @select_counter % CHECK_EVERY_N_SELECTS == 0
+
+          if !cx_correct?
+            cx = find_correct_host(@rw)
+            @connection = cx if cx
+          end
+        end
+        if @rw == :write && !@connection
+          # desperation mode: we've been asked for the master, but it's just not available.
+          # we'll go ahead and return a connection to the slave, understanding that it'll never work
+          # for writes.
+          @connection = find_correct_host(:read)
+        end
+      end
+
+      def has_side_effects?(sql)
+        sql =~ /^\s*(INSERT|UPDATE|DELETE|ALTER|CHANGE|REPLACE)/i
+      end
+
       def connect
-        @connection = find_correct_host
-        raise NoActiveMasterException unless @connection
+        @connection = find_correct_host(@rw)
+        raise_no_server_available! unless @connection
       end
 
       def raise_no_server_available!
@@ -106,11 +175,9 @@ module ActiveRecord
         tries = @tx_hold_timeout.to_f / sleep_interval
 
         tries.to_i.times do
-          cx = find_correct_host
-          if cx
-            @connection = cx
-            return
-          end
+          @connection = find_correct_host(@rw)
+          return if @connection
+
           sleep(sleep_interval)
         end
         raise_no_server_available!
@@ -124,7 +191,7 @@ module ActiveRecord
         end
       end
 
-      def find_correct_host
+      def find_correct_host(rw)
         cxs = hosts_and_ports.map do |host, port|
           initialize_connection(host, port)
         end.compact
@@ -132,7 +199,8 @@ module ActiveRecord
         correct_cxs = cxs.select { |cx| cx_correct?(cx) }
 
         chosen_cx = nil
-        if @is_master
+        case rw
+        when :write
           # for master connections, we make damn sure that we have just one master
           if correct_cxs.size == 1
             chosen_cx = correct_cxs.first
@@ -146,8 +214,8 @@ module ActiveRecord
 
             chosen_cx = nil
           end
-        else
-          # for slave connections, we just return a random RO candidate or the master if none are available
+        when :read
+          # for slave connections (or master-gone-away scenarios), we just return a random RO candidate or the master if none are available
           if correct_cxs.empty?
             chosen_cx = cxs.first
           else
@@ -178,7 +246,7 @@ module ActiveRecord
       def cx_correct?(cx = @connection)
         res = cx.query("SELECT @@read_only as ro").first
 
-        if @is_master
+        if @rw == :write
           res.first == 0
         else
           res.first == 1

data/test/ar_flexmaster_test.rb

@@ -7,20 +7,27 @@ require 'debugger'
 
 File.open(File.dirname(File.expand_path(__FILE__)) + "/database.yml", "w+") do |f|
       f.write <<-EOL
-test:
+common: &common
   adapter: mysql_flexmaster
   username: flex
-  hosts: ["127.0.0.1:#{$mysql_master.port}", "127.0.0.1:#{$mysql_slave.port}"]
-  password:
+  hosts: ["127.0.0.1:#{$mysql_master.port}", "127.0.0.1:#{$mysql_slave.port}", "127.0.0.1:#{$mysql_slave_2.port}"]
   database: flexmaster_test
 
+test:
+  <<: *common
+
 test_slave:
-  adapter: mysql_flexmaster
-  username: flex
+  <<: *common
+  slave: true
+
+reconnect:
+  <<: *common
+  reconnect: true
+
+reconnect_slave:
+  <<: *common
+  reconnect: true
   slave: true
-  hosts: ["127.0.0.1:#{$mysql_master.port}", "127.0.0.1:#{$mysql_slave.port}", "127.0.0.1:#{$mysql_slave_2.port}"]
-  password:
-  database: flexmaster_test
       EOL
 end
 
@@ -32,7 +39,17 @@ end
 
 class UserSlave < ActiveRecord::Base
   establish_connection(:test_slave)
-  set_table_name "users"
+  self.table_name = "users"
+end
+
+class Reconnect < ActiveRecord::Base
+  establish_connection(:reconnect)
+  self.table_name = "users"
+end
+
+class ReconnectSlave < ActiveRecord::Base
+  establish_connection(:reconnect_slave)
+  self.table_name = "users"
 end
 
 # $mysql_master and $mysql_slave are separate references to the master and slave that we
@@ -62,7 +79,7 @@ class TestArFlexmaster < Test::Unit::TestCase
   end
 
   def test_should_select_the_master_on_boot
-    assert main_connection_is_original_master?
+    assert_equal $mysql_master, master_connection
   end
 
   def test_should_hold_txs_until_timeout_then_abort
@@ -85,7 +102,7 @@ class TestArFlexmaster < Test::Unit::TestCase
       $mysql_slave.set_rw(true)
     end
     User.create(:name => "foo")
-    assert !main_connection_is_original_master?
+    assert_equal $mysql_slave, master_connection
     assert User.first(:conditions => {:name => "foo"})
   end
 
@@ -97,7 +114,9 @@ class TestArFlexmaster < Test::Unit::TestCase
       $mysql_slave.set_rw(true)
     end
     User.update_all(:name => "bar")
-    assert !main_connection_is_original_master?
+
+    assert_equal $mysql_slave, master_connection
+
     assert_equal "bar", User.first.name
   end
 
@@ -114,11 +133,11 @@ class TestArFlexmaster < Test::Unit::TestCase
     ActiveRecord::Base.connection
     $mysql_master.set_rw(false)
     $mysql_slave.set_rw(true)
-    assert main_connection_is_original_master?
+    assert_equal $mysql_master, master_connection
     100.times do
       u = User.first
     end
-    assert !main_connection_is_original_master?
+    assert_equal $mysql_slave, master_connection
   end
 
   # there's a small window in which the old master is read-only but the new slave hasn't come online yet.
@@ -127,7 +146,7 @@ class TestArFlexmaster < Test::Unit::TestCase
     ActiveRecord::Base.connection
     $mysql_master.set_rw(false)
     $mysql_slave.set_rw(false)
-    assert main_connection_is_original_master?
+    assert_equal $mysql_master, master_connection
     100.times do
       u = User.first
     end
@@ -149,16 +168,17 @@ class TestArFlexmaster < Test::Unit::TestCase
     assert_equal $mysql_master.port, cx.current_port
   end
 
-  def test_should_flip_the_slave_after_it_becomes_master
+  def test_should_move_off_the_slave_after_it_becomes_master
     UserSlave.first
     User.create!
     $mysql_master.set_rw(false)
     $mysql_slave.set_rw(true)
+
     20.times do
       UserSlave.connection.execute("select 1")
     end
-    connected_port = port_for_class(UserSlave)
-    assert [$mysql_slave_2.port, $mysql_master.port].include?(connected_port)
+
+    assert [$mysql_master, $mysql_slave_2].include?(slave_connection)
   end
 
   def test_xxx_non_responsive_master
@@ -167,33 +187,70 @@ class TestArFlexmaster < Test::Unit::TestCase
     start_time = Time.now.to_i
     User.connection.reconnect!
     assert Time.now.to_i - start_time >= 5, "only took #{Time.now.to_i - start_time} to timeout"
+  ensure
     ActiveRecord::Base.configurations["test"]["hosts"].pop
   end
 
-  def test_yyy_shooting_the_master_in_the_head
+  def test_shooting_the_master_in_the_head
     User.create!
+    UserSlave.first
+
+    $mysql_master.down!
 
-    $mysql_master.kill!
-    $mysql_master = nil
+    # protected against 'gone away' errors?
+    assert User.first
 
-    sleep 1
+    # this statement should
+    # put us into a bad state -- our @connection should be nil, as we'll fail to get a master connection
+    assert_raises(ActiveRecord::ConnectionAdapters::MysqlFlexmasterAdapter::NoServerAvailableException) do
+      User.create!
+    end
+
+    # now test that the next time through we ask for a read connection, we'll grudgingly give back the slave
+    User.first
+
+    assert [$mysql_slave, $mysql_slave_2].include?(master_connection)
+
+    # now a dba or someone comes along and flips the read-only bit on the slave
     $mysql_slave.set_rw(true)
-    User.connection.reconnect!
     User.create!
     UserSlave.first
-    assert !main_connection_is_original_master?
+
+    assert_equal $mysql_slave, master_connection
+  ensure
+    $mysql_master.up!
   end
 
-  # test that when nothing else is available we can fall back to the master in a slave role
-  # note that by the time this test runs, the 'yyy' test has already killed the master
-  def test_zzz_shooting_the_other_slave_in_the_head
+  def test_losing_the_server_with_reconnect_on
+    Reconnect.create!
+    ReconnectSlave.first
+
+    $mysql_master.down!
+
+    assert Reconnect.first
+    assert ReconnectSlave.first
+
+    assert_raises(ActiveRecord::ConnectionAdapters::MysqlFlexmasterAdapter::NoServerAvailableException) do
+      Reconnect.create!
+    end
+
     $mysql_slave.set_rw(true)
+    Reconnect.create!
+    ReconnectSlave.first
+  ensure
+    $mysql_master.up!
+  end
 
-    $mysql_slave_2.kill!
-    $mysql_slave_2 = nil
+  # test that when nothing else is available we can fall back to the master in a slave role
+  def test_master_can_act_as_slave
+    $mysql_slave.down!
+    $mysql_slave_2.down!
 
-    UserSlave.connection.reconnect!
-    assert port_for_class(UserSlave) == $mysql_slave.port
+    UserSlave.first
+    assert_equal $mysql_master, slave_connection
+  ensure
+    $mysql_slave.up!
+    $mysql_slave_2.up!
   end
 
 
@@ -207,4 +264,17 @@ class TestArFlexmaster < Test::Unit::TestCase
     port = port_for_class(ActiveRecord::Base)
     port == $original_master_port
   end
+
+  def connection_for_class(klass)
+    port = port_for_class(klass)
+    [$mysql_master, $mysql_slave, $mysql_slave_2].find { |cx| cx.port == port }
+  end
+
+  def master_connection
+    connection_for_class(User)
+  end
+
+  def slave_connection
+    connection_for_class(UserSlave)
+  end
 end

data/unplanned_failovers.md

@@ -0,0 +1,47 @@
+Here's a logic flow I've been thinking about for dealing with master outages.  In plain english, I'm generally 
+thinking:
+
+- inside a transaction, we will never attempt to recover from a master failure or attempt to ensure proper connections
+- outside a transaction, we'll try to be liberal about recovering from bad states and limp along in a read-only mode
+  as best we can.
+
+guard:
+  begin
+    yield
+  rescue 'server gone away', "can't connect to server"
+    retry-once
+  end
+
+hard_verify(INSERTs) == verify correct connection, try for 5 seconds, crash and unset connection if you can't
+
+soft_verify(SELECTs) == every N requests, try to verify that your connection is the right one.
+
+If you're running a SELECT targeted at the master, and no master is online, it's acceptable to use either
+the old master (your existing connection) or a slave connection for the purposes of reads (until the next
+verify)
+
+This will allow us to limp along in read-only mode for a short time until a new master can be promoted. 
+The downside of this approach is that in a pathological case we could be making decisions based on stale 
+or incorrect data.  I feel that the odds of this are long and are probably made up for by having a 
+halfway decent read-only mode.
+
+```
+switch incoming_sql:
+  BEGIN:
+    - in transaction? 
+     -> do not verify, do not guard.
+
+    - guard { hard-verify }
+    - execute BEGIN statement (without guard).  hard to reconnect here because of side effects
+
+  INSERT/UPDATE/DELETE:
+    - in transaction?
+      -> do not verify, do not guard 
+    -> guard { hard-verify, execute }
+
+  SELECT:
+    - in transaction? 
+      -> no verify, no guard
+    -> guard { soft-verify / execute } 
+``` 
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ar_mysql_flexmaster
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-19 00:00:00.000000000 Z
+date: 2013-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2
@@ -96,17 +96,17 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
 description: ar_mysql_flexmaster allows configuring N mysql servers in database.yml
   and auto-selects which is a master at runtime
 email:
@@ -141,6 +141,7 @@ files:
 - test/integration/there_and_back_again_test.rb
 - test/integration/with_queries_to_be_killed_test.rb
 - test/integration/wrong_setup_test.rb
+- unplanned_failovers.md
 homepage: http://github.com/osheroff/ar_mysql_flexmaster
 licenses: []
 post_install_message: 
@@ -161,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: select a master at runtime from a list