schoefmax-multi_db 0.1.4 → 0.2.0

data/README.rdoc

@@ -8,10 +8,27 @@ Within transactions, while executing ActiveRecord Observers and
 within "with_master" blocks (see below), even read queries are sent to the 
 master database.
 
+=== Important changes in 0.2.0
+
+* As of this version, <tt>ActiveRecord::Base.connection</tt> does not return the
+  connection proxy by default anymore (therefore the jump to 0.2.0). Only models
+  inheriting from AR::B return the proxy, unless they are defined as master_models
+  (see below). If you want to access the connection proxy from AR::B directly,
+  use <tt>ActiveRecord::Base.connection_proxy</tt>.
+
+* This version is the first attempt for thread-safety of this gem. <em>There might
+  still be some threading issues left!</em>. So please test your apps thoroughly
+  and report any issues you might encounter.
+
+* <tt>CGI::Session::ActiveRecordStore::Session</tt> is now automatically registered
+  as a master model.
+
 === Caveats
 
-* works with activerecord 2.1 and 2.2, but is not threadsafe (yet)
-* when using Passenger or lightspeed you will probably need to introduce a before_filter which checks if the proxy is setup (see the discussion in the masochism readme: http://github.com/technoweenie/masochism)
+* works with activerecord 2.1 and 2.2
+* when using Passenger or lightspeed you will probably need to introduce a
+  before_filter which checks if the proxy is setup (see the discussion in the
+  masochism readme: http://github.com/technoweenie/masochism)
 
 === Install
 
@@ -73,13 +90,16 @@ one on your development machine.
 
 Just add this to your controller:
 
-  around_filter(:only => :foo_action) { |c,a| MyModel.connection.with_master { a.call } }
+  around_filter(:only => :foo_action) { |c,a| ActiveRecord::Base.connection_proxy.with_master { a.call } }
 
 === Forcing the master for certain models
 
-In your environment.rb or an initializer, add:
+In your environment.rb or an initializer, add this *before* the call to <tt>setup!</tt>:
+
+  MultiDb::ConnectionProxy.master_models = ['CGI::Session::ActiveRecordStore::Session', 'PaymentTransaction', ...]
+  MultiDb::ConnectionProxy.setup!
 
-  MultiDb::ConnectionProxy.master_models = ['CGI::Session::ActiveRecordStore', 'PaymentTransaction', ...]
+*NOTE*: You cannot safely add more master_models after calling <tt>setup!</tt>.
 
 === Making one slave database sticky during a request
 
@@ -153,7 +173,8 @@ Note that the configurations hash should contain strings as keys instead of symb
   This might help with HA setups using virtual IPs (a test setup would be nice
   to verify this)
 * You specify slave databases in the configuration instead of specifying an extra
-  master database. This makes disabling or removing multi_db less dangerous.
+  master database. This makes disabling or removing multi_db less dangerous
+  (Update: Recent versions of masochism support this, too).
 * There are no <tt>set_to_master!</tt> and <tt>set_to_slave!</tt> methods, just
   <tt>with_master(&block)</tt>
 * All proxied methods are dynamically generated for better performance

data/lib/multi_db.rb

@@ -1,3 +1,4 @@
+require 'multi_db/thread_local_accessors'
 require 'multi_db/scheduler'
 require 'multi_db/active_record_extensions'
 require 'multi_db/observer_extensions'

data/lib/multi_db/active_record_extensions.rb

@@ -1,27 +1,54 @@
 module MultiDb
   module ActiveRecordExtensions
     def self.included(base)
+      base.send :include, InstanceMethods
+      base.send :extend, ClassMethods
       base.alias_method_chain :reload, :master
+      base.cattr_accessor :connection_proxy
+      # handle subclasses which were defined by the framework or plugins
+      base.send(:subclasses).each do |child|
+        child.hijack_connection
+      end
+    end
 
-      class << base
-        
-        cattr_accessor :connection_proxy
+    module InstanceMethods
+      def reload_with_master(*args, &block)
+        self.connection_proxy.with_master { reload_without_master }
+      end
+    end
 
-        # hijack the original method
-        def connection
-          if ConnectionProxy.master_models.include?(self.to_s)
-            self.retrieve_connection
-          else
-            self.connection_proxy
-          end
+    module ClassMethods
+      # Make sure transactions always switch to the master
+      def transaction(&block)
+        if self.connection.kind_of?(ConnectionProxy)
+          super
+        else
+          self.connection_proxy.with_master { super }
         end
       end
 
-    end
-    
-    def reload_with_master(*args, &block)
-      self.class.connection_proxy.with_master do
-        reload_without_master(*args, &block)
+      # make caching always use the ConnectionProxy
+      def cache(&block)
+        if ActiveRecord::Base.configurations.blank?
+          yield
+        else
+          self.connection_proxy.cache(&block)
+        end
+      end
+
+      def inherited(child)
+        super
+        child.hijack_connection
+      end
+
+      def hijack_connection
+        return if ConnectionProxy.master_models.include?(self.to_s)
+        logger.info "[MULTIDB] hijacking connection for #{self.to_s}"
+        class << self
+          def connection
+            self.connection_proxy
+          end
+        end
       end
     end
   end

data/lib/multi_db/connection_proxy.rb

@@ -1,7 +1,8 @@
 module MultiDb
   class ConnectionProxy
-    include MultiDb::QueryCacheCompat
+    include QueryCacheCompat
     include ActiveRecord::ConnectionAdapters::QueryCache
+    extend ThreadLocalAccessors
     
     # Safe methods are those that should either go to the slave ONLY or go
     # to the current active connection.
@@ -9,8 +10,11 @@ module MultiDb
       :select_rows, :select, :verify!, :raw_connection, :active?, :reconnect!,
       :disconnect!, :reset_runtime, :log, :log_info ]
 
+    DEFAULT_MASTER_MODELS = ['CGI::Session::ActiveRecordStore::Session']
+
     attr_accessor :master
-    attr_accessor :current
+    tlattr_accessor :master_depth
+    tlattr_accessor :current
     
     class << self
 
@@ -22,7 +26,7 @@ module MultiDb
       #
       # Example:
       #
-      #  MultiDb::ConnectionProxy.master_models = ['CGI::Session::ActiveRecordStore']
+      #  MultiDb::ConnectionProxy.master_models = ['MySessionStore', 'PaymentTransaction']
       attr_accessor :master_models
 
       # decides if we should switch to the next reader automatically.
@@ -34,7 +38,7 @@ module MultiDb
       # Replaces the connection of ActiveRecord::Base with a proxy and
       # establishes the connections to the slaves.
       def setup!
-        self.master_models ||= []
+        self.master_models ||= DEFAULT_MASTER_MODELS
         self.environment   ||= (defined?(RAILS_ENV) ? RAILS_ENV : 'development')
         self.sticky_slave  ||= false
         
@@ -47,6 +51,8 @@ module MultiDb
         master.logger.info("** multi_db with master and #{slaves.length} slave#{"s" if slaves.length > 1} loaded.")
       end
 
+      protected
+
       # Slave entries in the database.yml must be named like this
       #   development_slave_database:
       # or
@@ -69,32 +75,34 @@ module MultiDb
           end
         end
       end
+
+      private :new
       
     end
 
     def initialize(master, slaves)
-      @slaves      = Scheduler.new(slaves)
-      @master      = master
-      @current     = @slaves.current
-      @reconnect   = false
-      @with_master = 0
+      @slaves    = Scheduler.new(slaves)
+      @master    = master
+      @reconnect = false
+      self.current      = @slaves.current
+      self.master_depth = 0
     end
-    
+
     def slave
       @slaves.current
     end
 
     def with_master
-      @current = @master
-      @with_master += 1
+      self.current = @master
+      self.master_depth += 1
       yield
     ensure
-      @with_master -= 1
-      @current = @slaves.current if @with_master == 0
+      self.master_depth -= 1
+      self.current = slave if master_depth.zero?
     end
     
     def transaction(start_db_transaction = true, &block)
-      with_master { get_connection(@current).transaction(start_db_transaction, &block) }
+      with_master { @master.retrieve_connection.transaction(start_db_transaction, &block) }
     end
 
     # Calls the method on master/slave and dynamically creates a new
@@ -108,19 +116,15 @@ module MultiDb
     # Switches to the next slave database for read operations.
     # Fails over to the master database if all slaves are unavailable.
     def next_reader!
-      return if @with_master > 0 # don't if in with_master block
-      @current = @slaves.next
+      return unless master_depth.zero? # don't if in with_master block
+      self.current = @slaves.next
     rescue Scheduler::NoMoreItems
       logger.warn "[MULTIDB] All slaves are blacklisted. Reading from master"
-      @current = @master
+      self.current = @master
     end
 
     protected
 
-    def get_connection(db_class)
-      db_class.retrieve_connection
-    end
-
     def create_delegation_method!(method)
       self.instance_eval %Q{
         def #{method}(*args, &block)
@@ -136,27 +140,27 @@ module MultiDb
     
     def send_to_master(method, *args, &block)
       reconnect_master! if @reconnect
-      get_connection(@master).send(method, *args, &block)
+      @master.retrieve_connection.send(method, *args, &block)
     rescue => e
       raise_master_error(e)
     end
     
     def send_to_current(method, *args, &block)
       reconnect_master! if @reconnect && master?
-      get_connection(@current).send(method, *args, &block)
+      current.retrieve_connection.send(method, *args, &block)
     rescue NotImplementedError, NoMethodError
       raise
     rescue => e # TODO don't rescue everything
       raise_master_error(e) if master?
       logger.warn "[MULTIDB] Error reading from slave database"
       logger.error %(#{e.message}\n#{e.backtrace.join("\n")})
-      @slaves.blacklist!(@current)
+      @slaves.blacklist!(current)
       next_reader!
       retry
     end
     
     def reconnect_master!
-      get_connection(@master).reconnect!
+      @master.retrieve_connection.reconnect!
       @reconnect = false
     end
     
@@ -171,12 +175,12 @@ module MultiDb
     end
     
     def master?
-      @current == @master
+      current == @master
     end
         
     def logger
       ActiveRecord::Base.logger
     end
-    
+
   end
 end

data/lib/multi_db/scheduler.rb

@@ -1,16 +1,18 @@
 module MultiDb
   class Scheduler
     class NoMoreItems < Exception; end
-    
+    extend ThreadLocalAccessors
+
     attr :items
     delegate :[], :[]=, :to => :items
+    tlattr_accessor :current_index
     
     def initialize(items, blacklist_timeout = 1.minute)
       @n = items.length
       @items     = items
       @blacklist = Array.new(@n, Time.at(0))
-      @current   = 0
       @blacklist_timeout = blacklist_timeout
+      self.current_index = 0
     end
     
     def blacklist!(item)
@@ -18,22 +20,22 @@ module MultiDb
     end
     
     def current
-      @items[@current]
+      @items[current_index]
     end
     
     def next
-      previous = @current
+      previous = current_index
       until(@blacklist[next_index!] < Time.now - @blacklist_timeout) do
-        raise NoMoreItems, 'All items are blacklisted' if @current == previous
+        raise NoMoreItems, 'All items are blacklisted' if current_index == previous
       end
-      @items[@current]
+      current
     end
     
     protected
     
     def next_index!
-      @current = (@current + 1) % @n
+      self.current_index = (current_index + 1) % @n
     end
-    
+
   end
 end

data/lib/multi_db/thread_local_accessors.rb

@@ -0,0 +1,32 @@
+module MultiDb
+  module ThreadLocalAccessors
+    # Creates thread-local accessors for the given attribute name.
+    #
+    # The first invokation of the setter will make this value the default
+    # for any subsequent threads.
+    def tlattr_accessor(name)
+      ivar = "@_tlattr_#{name}"
+      class_eval %Q{
+        def #{name}
+          if #{ivar}
+            #{ivar}[Thread.current.object_id]
+          else
+            nil
+          end
+        end
+
+        def #{name}=(val)
+          if #{ivar}
+            unless #{ivar}.has_key?(Thread.current.object_id)
+              ObjectSpace.define_finalizer(Thread.current, lambda {|id| #{ivar}.delete(id) })
+            end
+            #{ivar}[Thread.current.object_id] = val
+          else
+            #{ivar} = Hash.new {|h, k| h[k] = val}
+            val
+          end
+        end
+      }, __FILE__, __LINE__
+    end
+  end
+end

data/multi_db.gemspec

@@ -2,15 +2,15 @@
 
 Gem::Specification.new do |s|
   s.name = %q{multi_db}
-  s.version = "0.1.4"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Maximilian Sch\303\266fmann"]
-  s.date = %q{2009-01-29}
+  s.date = %q{2009-03-08}
   s.description = "Connection proxy for ActiveRecord for single master / multiple slave database deployments"
   s.email = "max@pragmatic-it.de"
   s.extra_rdoc_files = ["LICENSE", "README.rdoc"]
-  s.files = ["lib/multi_db.rb", "lib/multi_db/active_record_extensions.rb", "lib/multi_db/connection_proxy.rb", "lib/multi_db/observer_extensions.rb", "lib/multi_db/query_cache_compat.rb", "lib/multi_db/scheduler.rb", "LICENSE", "README.rdoc", "spec/config/database.yml", "spec/connection_proxy_spec.rb", "spec/scheduler_spec.rb", "spec/spec_helper.rb", "multi_db.gemspec"]
+  s.files = ["lib/multi_db.rb", "lib/multi_db/thread_local_accessors.rb", "lib/multi_db/active_record_extensions.rb", "lib/multi_db/connection_proxy.rb", "lib/multi_db/observer_extensions.rb", "lib/multi_db/query_cache_compat.rb", "lib/multi_db/scheduler.rb", "LICENSE", "README.rdoc", "spec/config/database.yml", "spec/connection_proxy_spec.rb", "spec/scheduler_spec.rb", "spec/thread_local_accessors_spec.rb", "spec/spec_helper.rb", "multi_db.gemspec"]
   s.has_rdoc = true
   s.homepage = "http://github.com/schoefmax/multi_db"
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "multi_db", "--main", "README.rdoc"]

data/spec/config/database.yml

@@ -4,6 +4,7 @@ test:
   username: root
   password:
   host: 127.0.0.1
+  pool: 5
 
 test_slave_database_1:
   adapter: mysql
@@ -11,6 +12,7 @@ test_slave_database_1:
   username: root
   password:
   host: 127.0.0.1
+  pool: 5
 
 test_slave_database_2:
   adapter: mysql
@@ -18,3 +20,4 @@ test_slave_database_2:
   username: root
   password:
   host: 127.0.0.1
+  pool: 5

data/spec/connection_proxy_spec.rb

@@ -2,6 +2,7 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 require MULTI_DB_SPEC_DIR + '/../lib/multi_db/query_cache_compat'
 require MULTI_DB_SPEC_DIR + '/../lib/multi_db/active_record_extensions'
 require MULTI_DB_SPEC_DIR + '/../lib/multi_db/observer_extensions'
+require MULTI_DB_SPEC_DIR + '/../lib/multi_db/thread_local_accessors'
 require MULTI_DB_SPEC_DIR + '/../lib/multi_db/scheduler'
 require MULTI_DB_SPEC_DIR + '/../lib/multi_db/connection_proxy'
 
@@ -21,6 +22,7 @@ describe MultiDb::ConnectionProxy do
   end
   
   before(:each) do
+    MultiDb::ConnectionProxy.master_models = ['MasterModel']
     MultiDb::ConnectionProxy.setup!
     @proxy = ActiveRecord::Base.connection_proxy
     @master = @proxy.master.retrieve_connection
@@ -32,8 +34,16 @@ describe MultiDb::ConnectionProxy do
     ActiveRecord::Base.send :alias_method, :reload, :reload_without_master
   end
 
-  it 'AR::B#connection should return an instance of MultiDb::ConnectionProxy' do
-    ActiveRecord::Base.connection.should be_kind_of(MultiDb::ConnectionProxy)
+  it 'AR::B should respond to #connection_proxy' do
+    ActiveRecord::Base.connection_proxy.should be_kind_of(MultiDb::ConnectionProxy)
+  end
+
+  it 'FooModel#connection should return an instance of MultiDb::ConnectionProxy' do
+    FooModel.connection.should be_kind_of(MultiDb::ConnectionProxy)
+  end
+
+  it 'MasterModel#connection should not return an instance of MultiDb::ConnectionProxy' do
+    MasterModel.connection.should_not be_kind_of(MultiDb::ConnectionProxy)
   end
 
   it "should generate classes for each entry in the database.yml" do
@@ -135,18 +145,6 @@ describe MultiDb::ConnectionProxy do
     @proxy.insert(@sql)
   end
 
-  it 'should always use the master database for models configured as master models' do
-    @slave2.should_receive(:select_all).once.and_return([])
-    MasterModel.connection.should == @proxy
-    MasterModel.first
-    
-    MultiDb::ConnectionProxy.master_models = ['MasterModel']
-    MasterModel.connection.should == @master
-    @slave1.should_not_receive(:select_all)
-    MasterModel.retrieve_connection.should_receive(:select_all).once.and_return([])
-    MasterModel.first
-  end
-
   it 'should reload models from the master' do
     foo = FooModel.create!(:bar => 'baz')
     foo.bar = "not_saved"
@@ -178,5 +176,55 @@ describe MultiDb::ConnectionProxy do
 
   end
 
+  describe '(accessed from multiple threads)' do
+    # NOTE: We cannot put expectations on the connection objects itself
+    #       for the threading specs, as connection pooling will cause
+    #       different connections being returned for different threads.
+
+    it '#current and #next_reader! should be local to the thread' do
+      @proxy.current.should == MultiDb::SlaveDatabase1
+      @proxy.next_reader!.should == MultiDb::SlaveDatabase2
+      Thread.new do
+        @proxy.current.should == MultiDb::SlaveDatabase1
+        @proxy.next_reader!.should == MultiDb::SlaveDatabase2
+        @proxy.current.should == MultiDb::SlaveDatabase2
+        @proxy.next_reader!.should == MultiDb::SlaveDatabase1
+        @proxy.current.should == MultiDb::SlaveDatabase1
+      end
+      @proxy.current.should == MultiDb::SlaveDatabase2
+    end
+
+    it '#with_master should be local to the thread' do
+      @proxy.current.should_not == @proxy.master
+      @proxy.with_master do
+        @proxy.current.should == @proxy.master
+        Thread.new do
+          @proxy.current.should_not == @proxy.master
+          @proxy.with_master do
+            @proxy.current.should == @proxy.master
+          end
+          @proxy.current.should_not == @proxy.master
+        end
+        @proxy.current.should == @proxy.master
+      end
+      @proxy.current.should_not == @proxy.master
+    end
+
+    it 'should switch to the next reader even whithin with_master-block in different threads' do
+      # Because of connection pooling in AR 2.2, the second thread will cause
+      # a new connection being created behind the scenes. We therefore just test
+      # that these connections are beting retrieved for the right databases here.
+      @proxy.master.should_not_receive(:retrieve_connection).and_return(@master)
+      MultiDb::SlaveDatabase1.should_receive(:retrieve_connection).twice.and_return(@slave1)
+      MultiDb::SlaveDatabase2.should_receive(:retrieve_connection).once.and_return(@slave2)
+      @proxy.with_master do
+        Thread.new do
+          3.times { @proxy.select_one(@sql) }
+        end.join
+      end
+    end
+
+  end
+
 end
 

data/spec/scheduler_spec.rb

@@ -1,10 +1,11 @@
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require MULTI_DB_SPEC_DIR + '/../lib/multi_db/thread_local_accessors'
 require MULTI_DB_SPEC_DIR + '/../lib/multi_db/scheduler'
 
 describe MultiDb::Scheduler do
   
   before do
-    @items = [5, 2, 4, 8]
+    @items = [5, 7, 4, 8]
     @scheduler = MultiDb::Scheduler.new(@items.clone)
   end
   
@@ -35,9 +36,23 @@ describe MultiDb::Scheduler do
   
   it 'should unblacklist items automatically' do
     @scheduler = MultiDb::Scheduler.new(@items.clone, 1.second)
-    @scheduler.blacklist!(2)
+    @scheduler.blacklist!(7)
     sleep(1)
-    @scheduler.next.should == 2 
+    @scheduler.next.should == 7
+  end
+
+  describe '(accessed from multiple threads)' do
+
+    it '#current and #next should return the same item for the same thread' do
+      @scheduler.current.should == 5
+      @scheduler.next.should == 7
+      Thread.new do
+        @scheduler.current.should == 5
+        @scheduler.next.should == 7
+      end.join
+      @scheduler.next.should == 4
+    end
+
   end
   
 end

data/spec/spec_helper.rb

@@ -1,4 +1,6 @@
-%w[rubygems active_record yaml erb spec].each {|lib| require lib}
+require 'rubygems'
+gem 'activerecord', '2.2.2'
+%w[active_record yaml erb spec].each {|lib| require lib}
 
 RAILS_ENV = ENV['RAILS_ENV'] = 'test'
 

data/spec/thread_local_accessors_spec.rb

@@ -0,0 +1,40 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require MULTI_DB_SPEC_DIR + '/../lib/multi_db/thread_local_accessors.rb'
+
+class Foo
+  extend MultiDb::ThreadLocalAccessors
+  tlattr_accessor :bar
+end
+
+describe MultiDb::ThreadLocalAccessors do
+
+  it "should store values local to the thread" do
+    x = Foo.new
+    x.bar = 2
+    Thread.new do
+      x.bar.should == 2
+      x.bar = 3
+      Thread.new do
+        x.bar.should == 2
+        x.bar = 5
+      end.join
+      x.bar.should == 3
+    end.join
+    x.bar.should == 2
+  end
+
+  it 'should not leak memory' do
+    x = Foo.new
+    n = 6000
+    # create many thread-local values to make sure GC is invoked
+    n.times do
+      Thread.new do
+        x.bar = rand
+      end.join
+    end
+    hash = x.send :instance_variable_get, '@_tlattr_bar'
+    hash.size.should < (n / 2) # it should be a lot lower than n!
+  end
+
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: schoefmax-multi_db
 version: !ruby/object:Gem::Version 
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors: 
 - "Maximilian Sch\xC3\xB6fmann"
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-29 00:00:00 -08:00
+date: 2009-03-08 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -33,6 +33,7 @@ extra_rdoc_files:
 - README.rdoc
 files: 
 - lib/multi_db.rb
+- lib/multi_db/thread_local_accessors.rb
 - lib/multi_db/active_record_extensions.rb
 - lib/multi_db/connection_proxy.rb
 - lib/multi_db/observer_extensions.rb
@@ -43,6 +44,7 @@ files:
 - spec/config/database.yml
 - spec/connection_proxy_spec.rb
 - spec/scheduler_spec.rb
+- spec/thread_local_accessors_spec.rb
 - spec/spec_helper.rb
 - multi_db.gemspec
 has_rdoc: true