ar-octopus 0.8.4 → 0.8.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 663045a0938adf065becc68b2ed936fdadca804b
-  data.tar.gz: bd3e9ce12b5c7c33f15c76ee2f138c37953a775a
+  metadata.gz: a2e171072013526c5597b2d233d983f129a84cec
+  data.tar.gz: 0c05c12e05f5dfb64aaf6f15e0accb59c2174a53
 SHA512:
-  metadata.gz: 7da33ad1580e5b30fea1a94182f8ccb7f8ac718547bfb9338336786c146fd04a465ec0231ab93af5f19d76dfe13d5fd62fed745cfbf251ed2793f143f1e17db4
-  data.tar.gz: 3b6bb393f346d75b9ff859996c625274a16dd7a00bac901c8ff09191f1b12a3a78191d05ba7fab8492f596420bd061a625931ca0c1b120cdf9708cb3ff5fc1d1
+  metadata.gz: 19f50eb815a6273a87430354c5124096d90d561a8b27c1bfe0073a5a81f3a2242c71f099b88db0bab031018c56a8662ed587122412a0042ba543bc41c6b0d745
+  data.tar.gz: 11828df91744a59445fe6a4add2c6081971f724ed9394b96eb12b36c6a09e69db2546d7dc1eadc56c19074ed5de1d7cd28f39a38c0bd7d3ad88030238a46a98e

data/.ruby-version

@@ -1 +1 @@
-2.1.2
+2.1.5

data/.travis.yml

@@ -4,14 +4,18 @@ env:
 before_script:
   - "bundle exec rake db:prepare"
 rvm:
-  - 1.9.3
   - 2.0.0
-  - 2.1.1
+  - 2.1.5
+  - 2.2.0
 gemfile:
-  - gemfiles/rails32.gemfile
   - gemfiles/rails4.gemfile
   - gemfiles/rails41.gemfile
+  - gemfiles/rails42.gemfile
 notifications:
   recipients:
     - gabriel.sobrinho@gmail.com
     - thiago.pradi@gmail.com
+matrix:
+  include:
+    - rvm: 1.9.3
+      gemfile: gemfiles/rails32.gemfile

data/Appraisals

@@ -10,4 +10,7 @@ appraise "rails41" do
   gem "activerecord", "~> 4.1.0"
 end
 
+appraise "rails42" do
+  gem "activerecord", "~> 4.2.0"
+end
 # vim: ft=ruby

data/README.mkdn

@@ -158,6 +158,24 @@ class CustomConnection < ActiveRecord::Base
 end
 ```
 
+### allow_shard
+If you'd like to use specific shards with a model that has a Rails-managed connection, you can use `allow_shard`:
+
+```ruby
+class CustomConnectedModel
+   octopus_establish_connection(...)
+   allow_shard :my_shard
+end
+
+#This uses :my_shard
+CustomConnectedModel.using(:my_shard).first
+
+#This uses the Rails-managed connection pool (the call to 'using' is ignored)
+CustomConnectedModel.using(:some_other_shard).first
+```
+
+This can be useful if you have a model that lives in a separate database and would like to add sharding or replication to it. For other use cases, you may be better off with <a href="https://github.com/tchandy/octopus/wiki/Slave-Groups">slave groups</a>.
+
 ## Contributing with Octopus
 Contributors are welcome! To run the test suite, you need mysql, postgresql and sqlite3 installed. This is what you need to setup your Octopus development environment:
 

data/Rakefile

@@ -6,7 +6,7 @@ require 'appraisal'
 RSpec::Core::RakeTask.new
 RuboCop::RakeTask.new
 
-task :default => [:spec, :rubocop]
+task :default => [:spec]
 
 namespace :db do
   desc 'Build the databases for tests'
@@ -127,6 +127,7 @@ namespace :db do
         u.string :name
         u.string :commentable_type
         u.integer :commentable_id
+        u.boolean :open, default: false
       end
 
       BlankModel.using(shard_symbol).connection.create_table(:parts) do |u|
@@ -142,6 +143,10 @@ namespace :db do
         u.string :name
       end
 
+      BlankModel.using(shard_symbol).connection.create_table(:custom) do |u|
+        u.string :value
+      end
+
       if shard_symbol == :alone_shard
         BlankModel.using(shard_symbol).connection.create_table(:mmorpg_players) do |u|
           u.string :player_name

data/gemfiles/rails42.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 4.2.0"
+
+gemspec :path => "../"

data/lib/octopus.rb

@@ -64,6 +64,20 @@ module Octopus
     @environments ||= config['environments'] || ['production']
   end
 
+  def self.robust_environments=(environments)
+    @robust_environments = environments.map(&:to_s)
+  end
+
+  # Environments in which to swallow failures from a single shard
+  # when iterating through all.
+  def self.robust_environments
+    @robust_environments ||= config['robust_environments'] || ['production']
+  end
+
+  def self.robust_environment?
+    robust_environments.include? rails_env
+  end
+
   def self.rails3?
     ActiveRecord::VERSION::MAJOR <= 3
   end
@@ -80,6 +94,16 @@ module Octopus
     defined?(Rails)
   end
 
+  attr_writer :logger
+
+  def self.logger
+    if defined?(Rails)
+      @logger ||= Rails.logger
+    else
+      @logger ||= Logger.new($stderr)
+    end
+  end
+
   def self.shards=(shards)
     config[rails_env] = HashWithIndifferentAccess.new(shards)
     ActiveRecord::Base.connection.initialize_shards(@config)

data/lib/octopus/model.rb

@@ -44,10 +44,12 @@ module Octopus
         return unless Octopus.enabled?
 
         if new_record? || self.class.connection_proxy.block
-          self.current_shard = self.class.connection_proxy.current_shard
+          shard = self.class.connection_proxy.current_shard
         else
-          self.current_shard = self.class.connection_proxy.last_current_shard || self.class.connection_proxy.current_shard
+          shard = self.class.connection_proxy.last_current_shard || self.class.connection_proxy.current_shard
         end
+
+        self.current_shard = shard if self.class.allowed_shard?(shard)
       end
 
       def should_set_current_shard?
@@ -75,6 +77,7 @@ module Octopus
       def self.extended(base)
         base.class_attribute(:replicated)
         base.class_attribute(:sharded)
+        base.class_attribute(:allowed_shards)
         base.hijack_methods
       end
 
@@ -86,8 +89,13 @@ module Octopus
         self.sharded = true
       end
 
+      def allow_shard(*shards)
+        self.allowed_shards ||= []
+        self.allowed_shards += shards
+      end
+
       def hijack_methods
-        around_save :run_on_shard
+        around_save :run_on_shard, :unless => -> { self.class.custom_octopus_connection }
         after_initialize :set_current_shard
 
         class << self
@@ -116,7 +124,19 @@ module Octopus
       end
 
       def should_use_normal_connection?
-        !Octopus.enabled? || custom_octopus_connection
+        if !Octopus.enabled?
+          true
+        elsif custom_octopus_connection
+          !connection_proxy.block || !allowed_shard?(connection_proxy.current_shard)
+        end
+      end
+
+      def allowed_shard?(shard)
+        if custom_octopus_connection
+          allowed_shards && shard && allowed_shards.include?(shard)
+        else
+          true
+        end
       end
 
       def connection_with_octopus

data/lib/octopus/persistence.rb

@@ -25,7 +25,7 @@ module Octopus
         run_on_shard { super }
       end
 
-      def touch(name = nil)
+      def touch(*args)
         run_on_shard { super }
       end
 

data/lib/octopus/proxy.rb

@@ -17,7 +17,7 @@ module Octopus
       @slave_groups = HashWithIndifferentAccess.new
       @groups = {}
       @adapters = Set.new
-      @config = ActiveRecord::Base.connection_pool_without_octopus.connection.instance_variable_get(:@config)
+      @config = ActiveRecord::Base.connection_pool_without_octopus.spec.config
 
       unless config.nil?
         @entire_sharded = config['entire_sharded']
@@ -202,6 +202,9 @@ module Octopus
     # reconnect, but in Rails 3.1 the flag prevents this.
     def safe_connection(connection_pool)
       connection_pool.automatic_reconnect ||= true
+      if !connection_pool.connected? && @shards[:master].connection.query_cache_enabled
+        connection_pool.connection.enable_query_cache!
+      end
       connection_pool.connection
     end
 
@@ -218,7 +221,7 @@ module Octopus
     end
 
     def run_queries_on_shard(shard, &_block)
-      keeping_connection_proxy do
+      keeping_connection_proxy(shard) do
         using_shard(shard) do
           yield
         end
@@ -233,8 +236,9 @@ module Octopus
 
     def clean_connection_proxy
       self.current_shard = :master
+      self.current_model = nil
       self.current_group = nil
-      self.block = false
+      self.block = nil
     end
 
     def check_schema_migrations(shard)
@@ -244,8 +248,7 @@ module Octopus
     end
 
     def transaction(options = {}, &block)
-      replicated = @replicated && (current_model.replicated || fully_replicated?)
-      if !sharded && replicated
+      if !sharded && current_model_replicated?
         run_queries_on_shard(:master) do
           select_connection.transaction(options, &block)
         end
@@ -281,23 +284,23 @@ module Octopus
 
     def enable_query_cache!
       clear_query_cache
-      @shards.each { |_k, v| safe_connection(v).enable_query_cache! }
+      with_each_healthy_shard { |v| v.connected? && safe_connection(v).enable_query_cache! }
     end
 
     def disable_query_cache!
-      @shards.each { |_k, v| safe_connection(v).disable_query_cache! }
+      with_each_healthy_shard { |v| v.connected? && safe_connection(v).disable_query_cache! }
     end
 
     def clear_query_cache
-      @shards.each { |_k, v| safe_connection(v).clear_query_cache }
+      with_each_healthy_shard { |v| v.connected? && safe_connection(v).clear_query_cache }
     end
 
     def clear_active_connections!
-      @shards.each { |_k, v| v.release_connection }
+      with_each_healthy_shard(&:release_connection)
     end
 
     def clear_all_connections!
-      @shards.each { |_k, v| v.disconnect! }
+      with_each_healthy_shard(&:disconnect!)
     end
 
     def connected?
@@ -322,6 +325,44 @@ module Octopus
 
     protected
 
+    # Ensure that a single failing slave doesn't take down the entire application
+    def with_each_healthy_shard
+      @shards.each do |shard_name, v|
+        begin
+          yield(v)
+        rescue => e
+          if Octopus.robust_environment?
+            Octopus.logger.error "Error on shard #{shard_name}: #{e.message}"
+          else
+            raise
+          end
+        end
+      end
+
+      conn_handler = ActiveRecord::Base.connection_handler
+      if conn_handler.respond_to?(:connection_pool_list)
+        # Rails 4+
+        ar_pools = conn_handler.connection_pool_list
+      else
+        # Rails 3.2
+        ar_pools = conn_handler.connection_pools.values
+      end
+
+      ar_pools.each do |pool|
+        next if pool == @shards[:master] # Already handled this
+
+        begin
+          yield(pool)
+        rescue => e
+          if Octopus.robust_environment?
+            Octopus.logger.error "Error on pool (spec: #{pool.spec}): #{e.message}"
+          else
+            raise
+          end
+        end
+      end
+    end
+
     def connection_pool_for(adapter, config)
       if Octopus.rails4?
         arg = ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(adapter.dup, config)
@@ -356,7 +397,7 @@ module Octopus
     end
 
     def should_clean_connection_proxy?(method)
-      method.to_s =~ /insert|select|execute/ && !@replicated && !block
+      method.to_s =~ /insert|select|execute/ && !current_model_replicated? && (!block || block != current_shard)
     end
 
     # Try to use slaves if and only if `replicated: true` is specified in `shards.yml` and no slaves groups are defined
@@ -364,6 +405,10 @@ module Octopus
       @replicated && method.to_s =~ /select/ && !block && !slaves_grouped?
     end
 
+    def current_model_replicated?
+      @replicated && (current_model.try(:replicated) || fully_replicated?)
+    end
+
     def send_queries_to_selected_slave(method, *args, &block)
       if current_model.replicated || fully_replicated?
         selected_slave = @slaves_load_balancer.next
@@ -384,7 +429,7 @@ module Octopus
     # while ensuring that we revert `current_shard` from the selected slave to the (shard's) master
     # not to make queries other than SELECT leak to the slave.
     def should_use_slaves_for_method?(method)
-      @replicated && (current_model.replicated || fully_replicated?) && method.to_s =~ /select/
+      current_model_replicated? && method.to_s =~ /select/
     end
 
     def slaves_grouped?
@@ -409,14 +454,14 @@ module Octopus
     #
     # @see Octopus::Proxy#should_clean_connection?
     # @see Octopus::Proxy#clean_connection_proxy
-    def keeping_connection_proxy(&_block)
+    def keeping_connection_proxy(shard, &_block)
       last_block = block
 
       begin
-        self.block = true
+        self.block = shard
         yield
       ensure
-        self.block = last_block || false
+        self.block = last_block || nil
       end
     end
 
@@ -425,7 +470,9 @@ module Octopus
       older_shard = current_shard
 
       begin
-        self.current_shard = shard
+        unless current_model && !current_model.allowed_shard?(shard)
+          self.current_shard = shard
+        end
         yield
       ensure
         self.current_shard = older_shard

data/lib/octopus/relation_proxy.rb

@@ -17,7 +17,7 @@ module Octopus
     end
 
     def method_missing(method, *args, &block)
-      run_on_shard { @ar_relation.send(method, *args, &block) }
+      run_on_shard { @ar_relation.public_send(method, *args, &block) }
     end
 
     def ==(other)

data/lib/octopus/scope_proxy.rb

@@ -28,13 +28,19 @@ module Octopus
     end
 
     def connection
-      @klass.connection.current_shard = @current_shard
-      @klass.connection
+      @klass.connection_proxy.current_shard = @current_shard
+
+      if @klass.custom_octopus_connection && @klass.allowed_shard?(@current_shard)
+        # Force use of proxy, given we called 'using' explicitly to get here
+        @klass.connection_proxy.current_model = @klass
+        @klass.connection_proxy
+      else
+        @klass.connection
+      end
     end
 
     def method_missing(method, *args, &block)
       result = run_on_shard { @klass.send(method, *args, &block) }
-
       if result.respond_to?(:all)
         @klass = result
         return self

data/lib/octopus/version.rb

@@ -1,3 +1,3 @@
 module Octopus
-  VERSION = '0.8.4'
+  VERSION = '0.8.5'
 end

data/spec/config/shards.yml

@@ -3,6 +3,12 @@ mysql: &mysql
   username: <%= ENV['MYSQL_USER'] || 'root' %>
   host: localhost
 
+mysql_unavailable: &mysql_unavailable
+  adapter: mysql2
+  username: <%= ENV['MYSQL_USER'] || 'root' %>
+  host: 192.0.2.1
+  connect_timeout: 3
+
 octopus: &octopus
   shards:
     alone_shard:
@@ -87,6 +93,13 @@ replicated_with_one_slave:
       database: octopus_shard_2
       <<: *mysql
 
+replicated_with_one_slave_unavailable:
+  replicated: true
+  shards:
+    slave1:
+      database: octopus_shard_2
+      <<: *mysql_unavailable
+
 
 production_fully_replicated:
   replicated: true

data/spec/octopus/model_spec.rb

@@ -91,7 +91,7 @@ describe Octopus::Model do
 
     it 'should work when you have a SQLite3 shard' do
       u = User.using(:sqlite_shard).create!(:name => 'Sqlite3')
-      expect(User.using(:sqlite_shard).find_by_name('Sqlite3')).to eq(u)
+      expect(User.using(:sqlite_shard).where(name: 'Sqlite3').first).to eq(u)
     end
 
     it 'should clean #current_shard from proxy when using execute' do
@@ -471,6 +471,99 @@ describe Octopus::Model do
     end
   end
 
+  describe 'custom connection' do
+    context 'by default' do
+      it 'with plain call should use custom connection' do
+        expect(CustomConnection.connection.current_database).to eq('octopus_shard_2')
+      end
+
+      it 'should ignore using called on relation' do
+        expect(CustomConnection.using(:postgresql_shard).connection.current_database).to eq('octopus_shard_2')
+      end
+
+      it 'should ignore Octopus.using block' do
+        Octopus.using(:postgresql_shard) do
+          expect(CustomConnection.connection.current_database).to eq('octopus_shard_2')
+        end
+      end
+
+      it 'should save to correct shard' do
+        expect { CustomConnection.create(:value => 'custom value') }.to change {
+          CustomConnection
+            .connection
+            .execute("select count(*) as ct from custom where value = 'custom value'")
+            .to_a.first.first
+        }.by 1
+      end
+    end
+
+    context 'with allowed_shards configured' do
+      before do
+        CustomConnection.allow_shard :postgresql_shard
+      end
+
+      it 'with plain call should use custom connection' do
+        expect(CustomConnection.connection.current_database).to eq('octopus_shard_2')
+      end
+
+      it 'with using called on relation with allowed shard should use' do
+        expect(CustomConnection.using(:postgresql_shard).connection.current_database).to eq('octopus_shard_1')
+      end
+
+      it 'within Octopus.using block with allowed shard should use' do
+        Octopus.using(:postgresql_shard) do
+          expect(CustomConnection.connection.current_database).to eq('octopus_shard_1')
+        end
+      end
+
+      it 'with using called on relation with disallowed shard should not use' do
+        expect(CustomConnection.using(:brazil).connection.current_database).to eq('octopus_shard_2')
+      end
+
+      it 'within Octopus.using block with disallowed shard should not use' do
+        Octopus.using(:brazil) do
+          expect(CustomConnection.connection.current_database).to eq('octopus_shard_2')
+        end
+      end
+
+      it 'should save to correct shard' do
+        expect { CustomConnection.create(:value => 'custom value') }.to change {
+          CustomConnection
+            .connection
+            .execute("select count(*) as ct from custom where value = 'custom value'")
+            .to_a.first.first
+        }.by 1
+      end
+
+      it 'should clean up correctly' do
+        User.create!(:name => 'CleanUser')
+        CustomConnection.using(:postgresql_shard).first
+        expect(User.first).not_to be_nil
+      end
+
+      it 'should clean up correctly even inside block' do
+        User.create!(:name => 'CleanUser')
+
+        Octopus.using(:master) do
+          CustomConnection.using(:postgresql_shard).connection.execute('select count(*) from users')
+          expect(User.first).not_to be_nil
+        end
+      end
+    end
+
+    describe 'clear_active_connections!' do
+      it 'should not leak connection' do
+        CustomConnection.create(:value => 'custom value')
+
+        # This is what Rails, Sidekiq etc call--this normally handles all connection pools in the app
+        expect { ActiveRecord::Base.clear_active_connections! }
+          .to change { CustomConnection.connection_pool.active_connection? }
+
+        expect(CustomConnection.connection_pool.active_connection?).to be_falsey
+      end
+    end
+  end
+
   describe 'when using set_table_name' do
     it 'should work correctly' do
       Bacon.using(:brazil).create!(:name => 'YUMMMYYYY')

data/spec/octopus/relation_proxy_spec.rb

@@ -12,6 +12,13 @@ describe Octopus::RelationProxy do
       expect(@relation.current_shard).to eq(:canada)
     end
 
+    unless Octopus.rails3?
+      it 'can define collection association with the same name as ancestor private method' do
+        @client.comments << Comment.using(:canada).create!(open: true)
+        expect(@client.comments.open).to be_a_kind_of(ActiveRecord::Relation)
+      end
+    end
+
     context 'when comparing to other Relation objects' do
       before :each do
         @relation.reset

data/spec/octopus/replication_spec.rb

@@ -27,23 +27,33 @@ describe 'when the database is replicated' do
   end
 
   describe 'When enabling the query cache' do
-    include_context 'with query cache enabled'
-
-    it 'should do the queries with cache' do
-      OctopusHelper.using_environment :replicated_with_one_slave  do
-        cat1 = Cat.using(:master).create!(:name => 'Master Cat 1')
-        _ct2 = Cat.using(:master).create!(:name => 'Master Cat 2')
-        expect(Cat.using(:master).find(cat1.id)).to eq(cat1)
-        expect(Cat.using(:master).find(cat1.id)).to eq(cat1)
-        expect(Cat.using(:master).find(cat1.id)).to eq(cat1)
-
-        cat3 = Cat.using(:slave1).create!(:name => 'Slave Cat 3')
-        _ct4 = Cat.using(:slave1).create!(:name => 'Slave Cat 4')
-        expect(Cat.find(cat3.id).id).to eq(cat3.id)
-        expect(Cat.find(cat3.id).id).to eq(cat3.id)
-        expect(Cat.find(cat3.id).id).to eq(cat3.id)
-
-        expect(counter.query_count).to eq(16)
+    include_context 'with query cache enabled' do
+      it 'should do the queries with cache' do
+        OctopusHelper.using_environment :replicated_with_one_slave  do
+          cat1 = Cat.using(:master).create!(:name => 'Master Cat 1')
+          _ct2 = Cat.using(:master).create!(:name => 'Master Cat 2')
+          expect(Cat.using(:master).find(cat1.id)).to eq(cat1)
+          expect(Cat.using(:master).find(cat1.id)).to eq(cat1)
+          expect(Cat.using(:master).find(cat1.id)).to eq(cat1)
+
+          cat3 = Cat.using(:slave1).create!(:name => 'Slave Cat 3')
+          _ct4 = Cat.using(:slave1).create!(:name => 'Slave Cat 4')
+          expect(Cat.find(cat3.id).id).to eq(cat3.id)
+          expect(Cat.find(cat3.id).id).to eq(cat3.id)
+          expect(Cat.find(cat3.id).id).to eq(cat3.id)
+
+          expect(counter.query_count).to eq(14)
+        end
+      end
+    end
+  end
+
+  describe 'When enabling the query cache with slave unavailable' do
+    it "should not raise can't connect error" do
+      OctopusHelper.using_environment :replicated_with_one_slave_unavailable do
+        expect {
+          ActiveRecord::Base.connection.enable_query_cache!
+        }.to_not raise_error
       end
     end
   end

data/spec/support/database_models.rb

@@ -27,6 +27,7 @@ end
 
 # This class sets its own connection
 class CustomConnection < ActiveRecord::Base
+  self.table_name = 'custom'
   octopus_establish_connection(:adapter => 'mysql2', :database => 'octopus_shard_2', :username => 'root', :password => '')
 end
 
@@ -75,6 +76,7 @@ end
 
 class Comment < ActiveRecord::Base
   belongs_to :commentable, :polymorphic => true
+  scope :open, -> { where(open: true) }
 end
 
 class Bacon < ActiveRecord::Base

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ar-octopus
 version: !ruby/object:Gem::Version
-  version: 0.8.4
+  version: 0.8.5
 platform: ruby
 authors:
 - Thiago Pradi
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-23 00:00:00.000000000 Z
+date: 2015-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -178,6 +178,7 @@ files:
 - gemfiles/rails32.gemfile
 - gemfiles/rails4.gemfile
 - gemfiles/rails41.gemfile
+- gemfiles/rails42.gemfile
 - init.rb
 - lib/ar-octopus.rb
 - lib/octopus.rb
@@ -341,7 +342,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.4
 signing_key: 
 specification_version: 4
 summary: Easy Database Sharding for ActiveRecord