multi_db 0.2.1 → 0.2.2

data/README.rdoc

@@ -137,6 +137,35 @@ And add this to your ApplicationController:
 *NOTE*: It's not possible to toggle this mode in a running process, as the dynamically
 generated methods will have the initially defined "stickyness" built in.
 
+=== Using the weighted scheduler
+The standard scheduler roundrobins queries to evenly to all slaves. This means that if you're using servers with different capacity (slower machines, some slaves receiving traffic from other apps etc) you might run into problems. The weighted scheduler tries to address this by assigning a weight attribute to each slave and distribute queries evenly among the server pool.
+
+In your database.yml file add your weights like so:
+  test_slave_database_1:
+    <<: *creds
+    host: my.slavedb_1
+    weight: 1
+  
+  test_slave_database_2:
+    <<: *creds
+    host: my.slavedb_2
+    weight: 10
+
+The above configuration will lead to slavedb_2 to receive 9 times more queries than slavedb_1. Adding in a new slave with:
+  test_slave_database_3:
+    <<: *creds
+    host: my.slavedb_3
+    weight: 5
+    
+leads to a distribution of 1:10:5. For 100k queries the numbers could look like this:
+  Slave 1, with weight 1: 6302 queries
+  Slave 2, with weight 10: 62764 queries
+  Slave 3, with weight 5: 30934 queries
+  
+The weighted scheduler does not guarantee that the same slave will not receive two queries in a row. We feel this is not an issue, or rather, that such a guarantee doesn't help much as it's the complexity of the queries rather than the number that creates problems.
+
+If no weight param is given for a slave, a weight of 1 is assumed. A weight of 0 is caught and silently transformed into a weight of 1.
+
 === Usage outside of Rails
 
 You can use multi_db together with other framworks or in standalone scripts.
@@ -216,6 +245,7 @@ threadsafe and allows sharding of data.
 
 === Contributors
 
+* David Palm http://github.com/dvdplm
 * Matt Conway http://github.com/wr0ngway
 * Matthias Marshall http://github.com/webops
 

data/lib/multi_db/connection_proxy.rb

@@ -40,7 +40,7 @@ module MultiDb
 
       # Replaces the connection of ActiveRecord::Base with a proxy and
       # establishes the connections to the slaves.
-      def setup!
+      def setup!(scheduler = Scheduler)
         self.master_models ||= DEFAULT_MASTER_MODELS
         self.environment   ||= (defined?(RAILS_ENV) ? RAILS_ENV : 'development')
         self.sticky_slave  ||= false
@@ -50,10 +50,10 @@ module MultiDb
         raise "No slaves databases defined for environment: #{self.environment}" if slaves.empty?
         master.send :include, MultiDb::ActiveRecordExtensions
         ActiveRecord::Observer.send :include, MultiDb::ObserverExtensions
-        master.connection_proxy = new(master, slaves)
+        master.connection_proxy = new(master, slaves, scheduler)
         master.logger.info("** multi_db with master and #{slaves.length} slave#{"s" if slaves.length > 1} loaded.")
       end
-
+      
       protected
 
       # Slave entries in the database.yml must be named like this
@@ -65,12 +65,18 @@ module MultiDb
       # These would be available later as MultiDb::SlaveDatabaseSomeserver
       def init_slaves
         returning([]) do |slaves|
-          ActiveRecord::Base.configurations.keys.each do |name|
+          ActiveRecord::Base.configurations.each do |name, values|
             if name.to_s =~ /#{self.environment}_(slave_database.*)/
+              weight  = if values['weight'].blank?
+                          1
+                        else
+                          (v=values['weight'].to_i.abs).zero?? 1 : v
+                        end
               MultiDb.module_eval %Q{
                 class #{$1.camelize} < ActiveRecord::Base
                   self.abstract_class = true
                   establish_connection :#{name}
+                  WEIGHT = #{weight} unless const_defined?('WEIGHT')
                 end
               }, __FILE__, __LINE__
               slaves << "MultiDb::#{$1.camelize}".constantize
@@ -83,8 +89,8 @@ module MultiDb
       
     end
 
-    def initialize(master, slaves)
-      @slaves    = Scheduler.new(slaves)
+    def initialize(master, slaves, scheduler = Scheduler)
+      @slaves    = scheduler.new(slaves)
       @master    = master
       @reconnect = false
       self.current      = @slaves.current
@@ -94,6 +100,10 @@ module MultiDb
     def slave
       @slaves.current
     end
+    
+    def scheduler
+      @slaves
+    end
 
     def with_master
       self.current = @master

data/multi_db.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{multi_db}
-  s.version = "0.2.1"
+  s.version = "0.2.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Maximilian Sch\303\266fmann"]

data/spec/config/database.yml

@@ -21,3 +21,22 @@ test_slave_database_2:
   password:
   host: 127.0.0.1
   pool: 5
+  weight: 10
+  
+test_slave_database_3:
+  adapter: mysql
+  database: multi_db_test
+  username: root
+  password:
+  host: 127.0.0.1
+  pool: 5
+  weight: 5
+
+test_slave_database_4:
+  adapter: mysql
+  database: multi_db_test
+  username: root
+  password:
+  host: 127.0.0.1
+  pool: 5
+  weight: 10

data/spec/connection_proxy_spec.rb

@@ -20,210 +20,270 @@ describe MultiDb::ConnectionProxy do
     @sql = 'SELECT 1 + 1 FROM DUAL'
   end
   
-  before(:each) do
-    MultiDb::ConnectionProxy.master_models = ['MasterModel']
-    MultiDb::ConnectionProxy.setup!
-    @proxy = ActiveRecord::Base.connection_proxy
-    @master = @proxy.master.retrieve_connection
-    @slave1 = MultiDb::SlaveDatabase1.retrieve_connection
-    @slave2 = MultiDb::SlaveDatabase2.retrieve_connection
-  end
-
-  after(:each) do
-    ActiveRecord::Base.send :alias_method, :reload, :reload_without_master
-  end
+  describe "with standard Scheduler" do
+    before(:each) do
+      MultiDb::ConnectionProxy.master_models = ['MasterModel']
+      MultiDb::ConnectionProxy.setup!
+      @proxy = ActiveRecord::Base.connection_proxy
+      @master = @proxy.master.retrieve_connection
+      @slave1 = MultiDb::SlaveDatabase1.retrieve_connection
+      @slave2 = MultiDb::SlaveDatabase2.retrieve_connection
+      @slave3 = MultiDb::SlaveDatabase3.retrieve_connection
+      @slave4 = MultiDb::SlaveDatabase4.retrieve_connection
+    end
 
-  it 'AR::B should respond to #connection_proxy' do
-    ActiveRecord::Base.connection_proxy.should be_kind_of(MultiDb::ConnectionProxy)
-  end
+    after(:each) do
+      ActiveRecord::Base.send :alias_method, :reload, :reload_without_master
+    end
+  
+    it 'AR::B should respond to #connection_proxy' do
+      ActiveRecord::Base.connection_proxy.should be_kind_of(MultiDb::ConnectionProxy)
+    end
 
-  it 'FooModel#connection should return an instance of MultiDb::ConnectionProxy' do
-    FooModel.connection.should be_kind_of(MultiDb::ConnectionProxy)
-  end
+    it 'FooModel#connection should return an instance of MultiDb::ConnectionProxy' do
+      FooModel.connection.should be_kind_of(MultiDb::ConnectionProxy)
+    end
 
-  it 'MasterModel#connection should not return an instance of MultiDb::ConnectionProxy' do
-    MasterModel.connection.should_not be_kind_of(MultiDb::ConnectionProxy)
-  end
+    it 'MasterModel#connection should not return an instance of MultiDb::ConnectionProxy' do
+      MasterModel.connection.should_not be_kind_of(MultiDb::ConnectionProxy)
+    end
 
-  it "should generate classes for each entry in the database.yml" do
-    defined?(MultiDb::SlaveDatabase1).should_not be_nil
-    defined?(MultiDb::SlaveDatabase2).should_not be_nil
-  end
+    it "should generate classes for each entry in the database.yml" do
+      defined?(MultiDb::SlaveDatabase1).should_not be_nil
+      defined?(MultiDb::SlaveDatabase2).should_not be_nil
+    end
   
-  it 'should handle nested with_master-blocks correctly' do
-    @proxy.current.should_not == @proxy.master
-    @proxy.with_master do
-      @proxy.current.should == @proxy.master
+    it 'should handle nested with_master-blocks correctly' do
+      @proxy.current.should_not == @proxy.master
       @proxy.with_master do
         @proxy.current.should == @proxy.master
         @proxy.with_master do
           @proxy.current.should == @proxy.master
+          @proxy.with_master do
+            @proxy.current.should == @proxy.master
+          end
+          @proxy.current.should == @proxy.master
         end
         @proxy.current.should == @proxy.master
       end
-      @proxy.current.should == @proxy.master
+      @proxy.current.should_not == @proxy.master
     end
-    @proxy.current.should_not == @proxy.master
-  end
   
-  it 'should perform transactions on the master' do
-    @master.should_receive(:select_all).exactly(1) # makes sure the first one goes to a slave
-    @proxy.select_all(@sql)
-    ActiveRecord::Base.transaction do
+    it 'should perform transactions on the master' do
+      @master.should_receive(:select_all).exactly(1) # makes sure the first one goes to a slave
       @proxy.select_all(@sql)
+      ActiveRecord::Base.transaction do
+        @proxy.select_all(@sql)
+      end
     end
-  end
   
-  it 'should switch to the next reader on selects' do
-    @slave1.should_receive(:select_one).exactly(2)
-    @slave2.should_receive(:select_one).exactly(2)
-    4.times { @proxy.select_one(@sql) }
-  end
+    it 'should switch to the next reader on selects' do
+      @slave1.should_receive(:select_one).exactly(2)
+      @slave2.should_receive(:select_one).exactly(2)
+      6.times { @proxy.select_one(@sql) }
+    end
   
-  it 'should not switch to the next reader when whithin a with_master-block' do
-    @master.should_receive(:select_one).twice
-    @slave1.should_not_receive(:select_one)
-    @slave2.should_not_receive(:select_one)
-    @proxy.with_master do
-      2.times { @proxy.select_one(@sql) }
+    it 'should not switch to the next reader when whithin a with_master-block' do
+      @master.should_receive(:select_one).twice
+      @slave1.should_not_receive(:select_one)
+      @slave2.should_not_receive(:select_one)
+      @proxy.with_master do
+        2.times { @proxy.select_one(@sql) }
+      end
     end
-  end
 
-  it 'should send dangerous methods to the master' do
-    meths = [:insert, :update, :delete, :execute]
-    meths.each do |meth|
-      @slave1.stub!(meth).and_raise(RuntimeError)
-      @master.should_receive(meth).and_return(true)
-      @proxy.send(meth, @sql)
+    it 'should send dangerous methods to the master' do
+      meths = [:insert, :update, :delete, :execute]
+      meths.each do |meth|
+        @slave1.stub!(meth).and_raise(RuntimeError)
+        @master.should_receive(meth).and_return(true)
+        @proxy.send(meth, @sql)
+      end
     end
-  end
   
-  it 'should dynamically generate safe methods' do
-    @proxy.should_not respond_to(:select_value)
-    @proxy.select_value(@sql)
-    @proxy.should respond_to(:select_value)
-  end
-  
-  it 'should cache queries using select_all' do
-    ActiveRecord::Base.cache do
-      # next_reader will be called and switch to the SlaveDatabase2
-      @slave2.should_receive(:select_all).exactly(1)
-      @slave1.should_not_receive(:select_all)
-      @master.should_not_receive(:select_all)
-      3.times { @proxy.select_all(@sql) }
+    it 'should dynamically generate safe methods' do
+      @proxy.should_not respond_to(:select_value)
+      @proxy.select_value(@sql)
+      @proxy.should respond_to(:select_value)
     end
-  end
   
-  it 'should invalidate the cache on insert, delete and update' do
-    ActiveRecord::Base.cache do
-      meths = [:insert, :update, :delete]
-      meths.each do |meth|
-        @master.should_receive(meth).and_return(true)
+    it 'should cache queries using select_all' do
+      ActiveRecord::Base.cache do
+        # next_reader will be called and switch to the SlaveDatabase2
+        @slave2.should_receive(:select_all).exactly(1)
+        @slave1.should_not_receive(:select_all)
+        @master.should_not_receive(:select_all)
+        3.times { @proxy.select_all(@sql) }
       end
-      @slave2.should_receive(:select_all).twice
-      @slave1.should_receive(:select_all).once
-      3.times do |i|
-        @proxy.select_all(@sql)
-        @proxy.send(meths[i])
+    end
+  
+    it 'should invalidate the cache on insert, delete and update' do
+      ActiveRecord::Base.cache do
+        meths = [:insert, :update, :delete, :insert, :update]
+        meths.each do |meth|
+          @master.should_receive(meth).and_return(true)
+        end
+        @slave2.should_receive(:select_all).twice
+        @slave1.should_receive(:select_all).once
+        5.times do |i|
+          @proxy.select_all(@sql)
+          @proxy.send(meths[i])
+        end
       end
     end
-  end
   
-  it 'should retry the next slave when one fails and finally fall back to the master' do
-    @slave1.should_receive(:select_all).once.and_raise(RuntimeError)
-    @slave2.should_receive(:select_all).once.and_raise(RuntimeError)
-    @master.should_receive(:select_all).and_return(true)
-    @proxy.select_all(@sql)
-  end
+    it 'should retry the next slave when one fails and finally fall back to the master' do
+      @slave1.should_receive(:select_all).once.and_raise(RuntimeError)
+      @slave2.should_receive(:select_all).once.and_raise(RuntimeError)
+      @slave3.should_receive(:select_all).once.and_raise(RuntimeError)
+      @slave4.should_receive(:select_all).once.and_raise(RuntimeError)
+      @master.should_receive(:select_all).and_return(true)
+      @proxy.select_all(@sql)
+    end
   
-  it 'should try to reconnect the master connection after the master has failed' do
-    @master.should_receive(:update).and_raise(RuntimeError)
-    lambda { @proxy.update(@sql) }.should raise_error
-    @master.should_receive(:reconnect!).and_return(true)
-    @master.should_receive(:insert).and_return(1)
-    @proxy.insert(@sql)
-  end
+    it 'should try to reconnect the master connection after the master has failed' do
+      @master.should_receive(:update).and_raise(RuntimeError)
+      lambda { @proxy.update(@sql) }.should raise_error
+      @master.should_receive(:reconnect!).and_return(true)
+      @master.should_receive(:insert).and_return(1)
+      @proxy.insert(@sql)
+    end
 
-  it 'should reload models from the master' do
-    foo = FooModel.create!(:bar => 'baz')
-    foo.bar = "not_saved"
-    @slave1.should_not_receive(:select_all)
-    @slave2.should_not_receive(:select_all)
-    foo.reload
-    # we didn't stub @master#select_all here, check that we actually hit the db
-    foo.bar.should == 'baz'
-  end
+    it 'should reload models from the master' do
+      foo = FooModel.create!(:bar => 'baz')
+      foo.bar = "not_saved"
+      @slave1.should_not_receive(:select_all)
+      @slave2.should_not_receive(:select_all)
+      foo.reload
+      # we didn't stub @master#select_all here, check that we actually hit the db
+      foo.bar.should == 'baz'
+    end
   
-  describe 'with sticky_slave ' do
+    describe 'with sticky_slave ' do
 
-    before { MultiDb::ConnectionProxy.sticky_slave = true  }
-    after  { MultiDb::ConnectionProxy.sticky_slave = false }
+      before { MultiDb::ConnectionProxy.sticky_slave = true  }
+      after  { MultiDb::ConnectionProxy.sticky_slave = false }
 
-    it 'should not switch to the next reader automatically' do
-      @slave1.should_receive(:select_all).exactly(3)
-      @slave2.should_receive(:select_all).exactly(0)
-      3.times { @proxy.select_all(@sql) }
-    end
+      it 'should not switch to the next reader automatically' do
+        @slave1.should_receive(:select_all).exactly(3)
+        @slave2.should_receive(:select_all).exactly(0)
+        3.times { @proxy.select_all(@sql) }
+      end
 
-    it '#next_reader! should switch to the next slave' do
-      @slave1.should_receive(:select_one).exactly(3)
-      @slave2.should_receive(:select_one).exactly(7)
-      3.times { @proxy.select_one(@sql) }
-      @proxy.next_reader!
-      7.times { @proxy.select_one(@sql) }
-    end
+      it '#next_reader! should switch to the next slave' do
+        @slave1.should_receive(:select_one).exactly(3)
+        @slave2.should_receive(:select_one).exactly(7)
+        3.times { @proxy.select_one(@sql) }
+        @proxy.next_reader!
+        7.times { @proxy.select_one(@sql) }
+      end
 
-  end
+    end
 
-  describe '(accessed from multiple threads)' do
-    # NOTE: We cannot put expectations on the connection objects itself
-    #       for the threading specs, as connection pooling will cause
-    #       different connections being returned for different threads.
+    describe '(accessed from multiple threads)' do
+      # NOTE: We cannot put expectations on the connection objects itself
+      #       for the threading specs, as connection pooling will cause
+      #       different connections being returned for different threads.
 
-    it '#current and #next_reader! should be local to the thread' do
-      @proxy.current.should == MultiDb::SlaveDatabase1
-      @proxy.next_reader!.should == MultiDb::SlaveDatabase2
-      Thread.new do
+      it '#current and #next_reader! should be local to the thread' do
         @proxy.current.should == MultiDb::SlaveDatabase1
         @proxy.next_reader!.should == MultiDb::SlaveDatabase2
+        Thread.new do
+          @proxy.current.should == MultiDb::SlaveDatabase1
+          @proxy.next_reader!.should == MultiDb::SlaveDatabase2
+          @proxy.current.should == MultiDb::SlaveDatabase2
+          @proxy.next_reader!.should == MultiDb::SlaveDatabase1
+          @proxy.current.should == MultiDb::SlaveDatabase1
+        end
         @proxy.current.should == MultiDb::SlaveDatabase2
-        @proxy.next_reader!.should == MultiDb::SlaveDatabase1
-        @proxy.current.should == MultiDb::SlaveDatabase1
       end
-      @proxy.current.should == MultiDb::SlaveDatabase2
-    end
 
-    it '#with_master should be local to the thread' do
-      @proxy.current.should_not == @proxy.master
-      @proxy.with_master do
-        @proxy.current.should == @proxy.master
-        Thread.new do
-          @proxy.current.should_not == @proxy.master
-          @proxy.with_master do
-            @proxy.current.should == @proxy.master
+      it '#with_master should be local to the thread' do
+        @proxy.current.should_not == @proxy.master
+        @proxy.with_master do
+          @proxy.current.should == @proxy.master
+          Thread.new do
+            @proxy.current.should_not == @proxy.master
+            @proxy.with_master do
+              @proxy.current.should == @proxy.master
+            end
+            @proxy.current.should_not == @proxy.master
           end
-          @proxy.current.should_not == @proxy.master
+          @proxy.current.should == @proxy.master
         end
-        @proxy.current.should == @proxy.master
+        @proxy.current.should_not == @proxy.master
       end
-      @proxy.current.should_not == @proxy.master
-    end
 
-    it 'should switch to the next reader even whithin with_master-block in different threads' do
-      # Because of connection pooling in AR 2.2, the second thread will cause
-      # a new connection being created behind the scenes. We therefore just test
-      # that these connections are beting retrieved for the right databases here.
-      @proxy.master.should_not_receive(:retrieve_connection).and_return(@master)
-      MultiDb::SlaveDatabase1.should_receive(:retrieve_connection).twice.and_return(@slave1)
-      MultiDb::SlaveDatabase2.should_receive(:retrieve_connection).once.and_return(@slave2)
-      @proxy.with_master do
-        Thread.new do
-          3.times { @proxy.select_one(@sql) }
-        end.join
+      it 'should switch to the next reader even whithin with_master-block in different threads' do
+        # Because of connection pooling in AR 2.2, the second thread will cause
+        # a new connection being created behind the scenes. We therefore just test
+        # that these connections are beting retrieved for the right databases here.
+        @proxy.master.should_not_receive(:retrieve_connection).and_return(@master)
+        MultiDb::SlaveDatabase1.should_receive(:retrieve_connection).twice.and_return(@slave1)
+        MultiDb::SlaveDatabase2.should_receive(:retrieve_connection).once.and_return(@slave2)
+        MultiDb::SlaveDatabase3.should_receive(:retrieve_connection).once.and_return(@slave3)
+        MultiDb::SlaveDatabase4.should_receive(:retrieve_connection).once.and_return(@slave4)
+        @proxy.with_master do
+          Thread.new do
+            5.times { @proxy.select_one(@sql) }
+          end.join
+        end
       end
+
     end
 
-  end
+  end # with normal scheduler
 
+  describe "alternative scheduler" do
+    class MyScheduler
+      def initialize(slaves); :done; end
+      def current; :current; end
+      def next_reader!; :next end
+      def blacklist!; :blacklisted; end
+    end
+
+    it "has a 'scheduler' method that returns the current scheduler instance" do
+      my_scheduler = mock('My Scheduler!', :current => nil)
+      MyScheduler.should_receive(:new).and_return(my_scheduler)
+      MultiDb::ConnectionProxy.setup!(MyScheduler)
+      ActiveRecord::Base.connection_proxy.should respond_to(:scheduler)
+      ActiveRecord::Base.connection_proxy.scheduler.should be(my_scheduler)
+    end
+    
+    it "can be initialized with an optional alternative scheduling class" do
+      slaves = MultiDb::ConnectionProxy.send(:init_slaves)
+      proxy = MultiDb::ConnectionProxy.send(:new, @master, slaves, MyScheduler)
+      proxy.scheduler.should be_an_instance_of(MyScheduler)
+    end
+    
+    it "uses an alternative scheduler if setup! is called with a compatible class" do
+      MultiDb::ConnectionProxy.setup!(MyScheduler)
+      ActiveRecord::Base.connection_proxy.scheduler.should be_an_instance_of(MyScheduler)
+    end
+    
+    it "uses the default scheduler if no param is passed" do
+      MultiDb::ConnectionProxy.setup!
+      ActiveRecord::Base.connection_proxy.scheduler.should be_an_instance_of(MultiDb::Scheduler)
+    end
+
+    describe "has weights for query distribution" do
+      before do
+        MultiDb::ConnectionProxy.setup!
+      end
+      
+      it "adds a WEIGHT constant to the MultiDb::SlaveDatabaseN 'models'" do
+        MultiDb::SlaveDatabase1.const_defined?('WEIGHT').should be_true
+      end
+      
+      it "sets the WEIGHT to 1 if no weight is configured" do
+        MultiDb::SlaveDatabase1::WEIGHT.should == 1
+      end
+      
+      it "sets the WEIGHT to whatever it is configured to" do
+        MultiDb::SlaveDatabase2::WEIGHT.should == 10
+      end
+    end
+    
+  end # with alternative scheduler
 end
 

data/spec/spec_helper.rb

@@ -1,5 +1,5 @@
 require 'rubygems'
-gem 'activerecord', '2.2.2'
+# gem 'activerecord', '2.2.2'
 %w[tlattr_accessors active_record yaml erb spec].each {|lib| require lib}
 
 RAILS_ENV = ENV['RAILS_ENV'] = 'test'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: multi_db
 version: !ruby/object:Gem::Version 
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors: 
 - "Maximilian Sch\xC3\xB6fmann"