makara 0.4.1 → 0.5.1

data/lib/makara/logging/subscriber.rb

@@ -1,6 +1,5 @@
 module Makara
   module Logging
-
     module Subscriber
       IGNORE_PAYLOAD_NAMES = ["SCHEMA", "EXPLAIN"]
 
@@ -33,6 +32,5 @@ module Makara
         "[#{adapter._makara_name}]"
       end
     end
-
   end
 end

data/lib/makara/middleware.rb

@@ -3,7 +3,6 @@ require 'rack'
 # Persists the Makara::Context across requests ensuring the same master pool is used on subsequent requests.
 module Makara
   class Middleware
-
     def initialize(app, cookie_options = {})
       @app = app
       @cookie_options = cookie_options
@@ -11,6 +10,7 @@ module Makara
 
     def call(env)
       return @app.call(env) if ignore_request?(env)
+
       set_current_context(env)
 
       status, headers, body = @app.call(env)
@@ -19,7 +19,6 @@ module Makara
       [status, headers, body]
     end
 
-
     protected
 
     def set_current_context(env)

data/lib/makara/pool.rb

@@ -1,11 +1,11 @@
 require 'active_support/core_ext/hash/keys'
+require 'makara/strategies/shard_aware'
 
 # Wraps a collection of similar connections and chooses which one to use
 # Provides convenience methods for accessing underlying connections
 
 module Makara
   class Pool
-
     # there are cases when we understand the pool is busted and we essentially want to skip
     # all execution
     attr_accessor :disabled
@@ -13,6 +13,8 @@ module Makara
     attr_reader :role
     attr_reader :connections
     attr_reader :strategy
+    attr_reader :shard_strategy_class
+    attr_reader :default_shard
 
     def initialize(role, proxy)
       @role             = role
@@ -20,10 +22,15 @@ module Makara
       @connections      = []
       @blacklist_errors = []
       @disabled         = false
-      @strategy         = proxy.strategy_for(role)
+      if proxy.shard_aware_for(role)
+        @strategy = Makara::Strategies::ShardAware.new(self)
+        @shard_strategy_class = proxy.strategy_class_for(proxy.strategy_name_for(role))
+        @default_shard = proxy.default_shard_for(role)
+      else
+        @strategy = proxy.strategy_for(role)
+      end
     end
 
-
     def completely_blacklisted?
       @connections.each do |connection|
         return false unless connection._makara_blacklisted?
@@ -31,7 +38,6 @@ module Makara
       true
     end
 
-
     # Add a connection to this pool, wrapping the connection with a Makara::ConnectionWrapper
     def add(config)
       config[:name] ||= "#{@role}/#{@connections.length + 1}"
@@ -61,6 +67,7 @@ module Makara
 
       @connections.each do |con|
         next if con._makara_blacklisted?
+
         begin
           ret = @proxy.error_handler.handle(con) do
             if block
@@ -91,52 +98,63 @@ module Makara
     # Provide a connection that is not blacklisted and connected. Handle any errors
     # that may occur within the block.
     def provide
-      provided_connection = self.next
+      attempt = 0
+      begin
+        provided_connection = self.next
 
-      # nil implies that it's blacklisted
-      if provided_connection
+        # nil implies that it's blacklisted
+        if provided_connection
 
-        value = @proxy.error_handler.handle(provided_connection) do
-          yield provided_connection
-        end
+          value = @proxy.error_handler.handle(provided_connection) do
+            yield provided_connection
+          end
 
-        @blacklist_errors = []
+          @blacklist_errors = []
 
-        value
+          value
 
-      # if we've made any connections within this pool, we should report the blackout.
-      elsif connection_made?
-        err = Makara::Errors::AllConnectionsBlacklisted.new(self, @blacklist_errors)
-        @blacklist_errors = []
-        raise err
-      else
-        raise Makara::Errors::NoConnectionsAvailable.new(@role) unless @disabled
-      end
+        # if we've made any connections within this pool, we should report the blackout.
+        elsif connection_made?
+          err = Makara::Errors::AllConnectionsBlacklisted.new(self, @blacklist_errors)
+          @blacklist_errors = []
+          raise err
+        else
+          raise Makara::Errors::NoConnectionsAvailable.new(@role) unless @disabled
+        end
 
-    # when a connection causes a blacklist error within the provided block, we blacklist it then retry
-    rescue Makara::Errors::BlacklistConnection => e
-      @blacklist_errors.insert(0, e)
-      provided_connection._makara_blacklist!
-      retry
+      # when a connection causes a blacklist error within the provided block, we blacklist it then retry
+      rescue Makara::Errors::BlacklistConnection => e
+        @blacklist_errors.insert(0, e)
+        in_transaction = self.role == "master" && provided_connection._makara_in_transaction?
+        provided_connection._makara_blacklist!
+        raise Makara::Errors::BlacklistedWhileInTransaction.new(@role) if in_transaction
+
+        attempt += 1
+        if attempt < @connections.length
+          retry
+        elsif connection_made?
+          err = Makara::Errors::AllConnectionsBlacklisted.new(self, @blacklist_errors)
+          @blacklist_errors = []
+          raise err
+        else
+          raise Makara::Errors::NoConnectionsAvailable.new(@role) unless @disabled
+        end
+      end
     end
 
-
-
     protected
 
-
     # have we connected to any of the underlying connections.
     def connection_made?
       @connections.any?(&:_makara_connected?)
     end
 
-
     # Get the next non-blacklisted connection. If the proxy is setup
     # to be sticky, provide back the current connection assuming it is
     # not blacklisted.
     def next
-      if @proxy.sticky && @strategy.current
-        @strategy.current
+      if @proxy.sticky && (curr = @strategy.current)
+        curr
       else
         @strategy.next
       end

data/lib/makara/proxy.rb

@@ -11,11 +11,11 @@ require 'active_support/core_ext/string/inflections'
 
 module Makara
   class Proxy < ::SimpleDelegator
-
     METHOD_MISSING_SKIP = [ :byebug, :puts ]
 
-    class_attribute :hijack_methods
+    class_attribute :hijack_methods, :control_methods
     self.hijack_methods = []
+    self.control_methods = []
 
     class << self
       def hijack_method(*method_names)
@@ -23,27 +23,44 @@ module Makara
         self.hijack_methods |= method_names
 
         method_names.each do |method_name|
-          define_method method_name do |*args, &block|
+          define_method(method_name) do |*args, &block|
             appropriate_connection(method_name, args) do |con|
               con.send(method_name, *args, &block)
             end
           end
+
+          ruby2_keywords method_name if Module.private_method_defined?(:ruby2_keywords)
         end
       end
 
       def send_to_all(*method_names)
         method_names.each do |method_name|
-          define_method method_name do |*args|
-            send_to_all method_name, *args
+          define_method(method_name) do |*args|
+            send_to_all(method_name, *args)
           end
+
+          ruby2_keywords method_name if Module.private_method_defined?(:ruby2_keywords)
         end
       end
-    end
 
+      def control_method(*method_names)
+        self.control_methods = self.control_methods || []
+        self.control_methods |= method_names
+
+        method_names.each do |method_name|
+          define_method(method_name) do |*args, &block|
+            control&.send(method_name, *args, &block)
+          end
+
+          ruby2_keywords method_name if Module.private_method_defined?(:ruby2_keywords)
+        end
+      end
+    end
 
     attr_reader :error_handler
     attr_reader :sticky
     attr_reader :config_parser
+    attr_reader :control
 
     def initialize(config)
       @config         = config.symbolize_keys
@@ -84,6 +101,14 @@ module Makara
       @config_parser.makara_config["#{role}_strategy".to_sym]
     end
 
+    def shard_aware_for(role)
+      @config_parser.makara_config["#{role}_shard_aware".to_sym]
+    end
+
+    def default_shard_for(role)
+      @config_parser.makara_config["#{role}_default_shard".to_sym]
+    end
+
     def strategy_class_for(strategy_name)
       case strategy_name
       when 'round_robin', 'roundrobin', nil, ''
@@ -97,27 +122,25 @@ module Makara
 
     def method_missing(m, *args, &block)
       if METHOD_MISSING_SKIP.include?(m)
-        return super(m, *args, &block)
+        return super
       end
 
       any_connection do |con|
-        if con.respond_to?(m)
-          con.public_send(m, *args, &block)
-        elsif con.respond_to?(m, true)
-          con.__send__(m, *args, &block)
+        if con.respond_to?(m, true)
+          con.send(m, *args, &block)
         else
-          super(m, *args, &block)
+          super
         end
       end
     end
 
-    class_eval <<-RUBY_EVAL, __FILE__, __LINE__ + 1
-      def respond_to#{RUBY_VERSION.to_s =~ /^1.8/ ? nil : '_missing'}?(m, include_private = false)
-        any_connection do |con|
-          con._makara_connection.respond_to?(m, true)
-        end
+    ruby2_keywords :method_missing if Module.private_method_defined?(:ruby2_keywords)
+
+    def respond_to_missing?(m, include_private = false)
+      any_connection do |con|
+        con._makara_connection.respond_to?(m, true)
       end
-    RUBY_EVAL
+    end
 
     def graceful_connection_for(config)
       fake_wrapper = Makara::ConnectionWrapper.new(self, nil, config)
@@ -138,18 +161,25 @@ module Makara
 
     protected
 
-
     def send_to_all(method_name, *args)
       # slave pool must run first to allow for slave-->master failover without running operations on master twice.
       handling_an_all_execution(method_name) do
-        @slave_pool.send_to_all method_name, *args
-        @master_pool.send_to_all method_name, *args
+        @slave_pool.send_to_all(method_name, *args)
+        @master_pool.send_to_all(method_name, *args)
       end
     end
 
+    ruby2_keywords :send_to_all if Module.private_method_defined?(:ruby2_keywords)
+
     def any_connection
-      @master_pool.provide do |con|
-        yield con
+      if @master_pool.disabled
+        @slave_pool.provide do |con|
+          yield con
+        end
+      else
+        @master_pool.provide do |con|
+          yield con
+        end
       end
     rescue ::Makara::Errors::AllConnectionsBlacklisted, ::Makara::Errors::NoConnectionsAvailable
       begin
@@ -175,14 +205,11 @@ module Makara
       end
     end
 
-
     # master or slave
     def appropriate_pool(method_name, args)
-
       # for testing purposes
       pool = _appropriate_pool(method_name, args)
       yield pool
-
     rescue ::Makara::Errors::AllConnectionsBlacklisted, ::Makara::Errors::NoConnectionsAvailable => e
       if pool == @master_pool
         @master_pool.connections.each(&:_makara_whitelist!)
@@ -241,7 +268,6 @@ module Makara
       @hijacked = false
     end
 
-
     def stuck_to_master?
       sticky? && Makara::Context.stuck?(@id)
     end
@@ -286,18 +312,18 @@ module Makara
       yield
     rescue ::Makara::Errors::NoConnectionsAvailable => e
       if e.role == 'master'
-        Kernel.raise ::Makara::Errors::NoConnectionsAvailable.new('master and slave')
+        # this means slave connections are good.
+        return
       end
+
       @slave_pool.disabled = true
       yield
     ensure
       @slave_pool.disabled = false
     end
 
-
     def connection_for(config)
       Kernel.raise NotImplementedError
     end
-
   end
 end

data/lib/makara/railtie.rb

@@ -1,9 +1,7 @@
 module Makara
   class Railtie < ::Rails::Railtie
-
     initializer "makara.configure_rails_initialization" do |app|
       app.middleware.use Makara::Middleware
     end
-
   end
 end

data/lib/makara/strategies/abstract.rb

@@ -2,6 +2,7 @@ module Makara
   module Strategies
     class Abstract
       attr_reader :pool
+
       def initialize(pool)
         @pool = pool
         init

data/lib/makara/strategies/priority_failover.rb

@@ -27,6 +27,7 @@ module Makara
         @weighted_connections.each_with_index do |con, index|
           check = safe_value(index)
           next unless check
+
           @current_idx = index
           return check
         end
@@ -41,6 +42,7 @@ module Makara
         con = @weighted_connections[idx]
         return nil unless con
         return nil if con._makara_blacklisted?
+
         con
       end
     end

data/lib/makara/strategies/round_robin.rb

@@ -26,8 +26,8 @@ module Makara
         return safe_value(0, true) if single_one?
 
         idx = @current_idx
-        begin
 
+        begin
           idx = next_index(idx)
 
           # if we've looped all the way around, return our safe value
@@ -48,7 +48,6 @@ module Makara
         idx
       end
 
-
       # return the connection if it's not blacklisted
       # otherwise return nil
       # optionally, store the position and context we're returning
@@ -64,7 +63,6 @@ module Makara
         con
       end
 
-
       # stub in test mode to ensure consistency
       def should_shuffle?
         true

data/lib/makara/strategies/shard_aware.rb

@@ -0,0 +1,45 @@
+require 'makara/errors/invalid_shard'
+
+module Makara
+  module Strategies
+    class ShardAware < ::Makara::Strategies::Abstract
+      def init
+        @shards = {}
+        @default_shard = pool.default_shard
+      end
+
+      def connection_added(wrapper)
+        id = wrapper._makara_shard_id
+        shard_strategy(id).connection_added(wrapper)
+      end
+
+      def shard_strategy(shard_id)
+        id = shard_id
+        shard_strategy = @shards[id]
+        unless shard_strategy
+          shard_strategy = pool.shard_strategy_class.new(pool)
+          @shards[id] = shard_strategy
+        end
+        shard_strategy
+      end
+
+      def current
+        id = shard_id
+        raise Makara::Errors::InvalidShard.new(pool.role, id) unless id && @shards[id]
+
+        @shards[id].current
+      end
+
+      def next
+        id = shard_id
+        raise Makara::Errors::InvalidShard.new(pool.role, id) unless id && @shards[id]
+
+        @shards[id].next
+      end
+
+      def shard_id
+        Thread.current['makara_shard_id'] || pool.default_shard
+      end
+    end
+  end
+end

data/lib/makara/version.rb

@@ -1,15 +1,13 @@
 module Makara
   module VERSION
-
     MAJOR = 0
-    MINOR = 4
+    MINOR = 5
     PATCH = 1
     PRE = nil
 
     def self.to_s
       [MAJOR, MINOR, PATCH, PRE].compact.join('.')
     end
-
   end unless defined?(::Makara::VERSION)
   ::Makara::VERSION
 end

data/lib/makara.rb

@@ -2,7 +2,6 @@ require 'active_support'
 require 'makara/version'
 require 'makara/railtie' if defined?(Rails)
 module Makara
-
   autoload :Cache,              'makara/cache'
   autoload :ConfigParser,       'makara/config_parser'
   autoload :ConnectionWrapper,  'makara/connection_wrapper'
@@ -14,10 +13,12 @@ module Makara
   autoload :Proxy,              'makara/proxy'
 
   module Errors
-    autoload :MakaraError,                'makara/errors/makara_error'
-    autoload :AllConnectionsBlacklisted,  'makara/errors/all_connections_blacklisted'
-    autoload :BlacklistConnection,        'makara/errors/blacklist_connection'
-    autoload :NoConnectionsAvailable,     'makara/errors/no_connections_available'
+    autoload :MakaraError,                   'makara/errors/makara_error'
+    autoload :AllConnectionsBlacklisted,     'makara/errors/all_connections_blacklisted'
+    autoload :BlacklistConnection,           'makara/errors/blacklist_connection'
+    autoload :NoConnectionsAvailable,        'makara/errors/no_connections_available'
+    autoload :BlacklistedWhileInTransaction, 'makara/errors/blacklisted_while_in_transaction'
+    autoload :InvalidShard,                  'makara/errors/invalid_shard'
   end
 
   module Logging
@@ -29,8 +30,8 @@ module Makara
     autoload :Abstract,         'makara/strategies/abstract'
     autoload :RoundRobin,       'makara/strategies/round_robin'
     autoload :PriorityFailover, 'makara/strategies/priority_failover'
+    autoload :ShardAware,       'makara/strategies/shard_aware'
   end
-
 end
 
 ActiveSupport.on_load(:active_record) do

data/makara.gemspec

@@ -1,4 +1,3 @@
-# -*- encoding: utf-8 -*-
 require File.expand_path('../lib/makara/version', __FILE__)
 
 Gem::Specification.new do |gem|
@@ -6,7 +5,11 @@ Gem::Specification.new do |gem|
   gem.email         = ["mike@mikeonrails.com"]
   gem.description   = %q{Read-write split your DB yo}
   gem.summary       = %q{Read-write split your DB yo}
-  gem.homepage      = ""
+  gem.homepage      = "https://github.com/instacart/makara"
+  gem.licenses      = ['MIT']
+  gem.metadata      = {
+    "source_code_uri" => 'https://github.com/instacart/makara'
+  }
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -15,5 +18,25 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Makara::VERSION
 
-  gem.add_dependency 'activerecord', '>= 3.0.0'
+  gem.required_ruby_version = ">= 2.5.0"
+
+  gem.add_dependency "activerecord", ">= 5.2.0"
+
+  gem.add_development_dependency "rack"
+  gem.add_development_dependency "rake", "~> 13.0"
+  gem.add_development_dependency "rspec", "~> 3.9"
+  gem.add_development_dependency "timecop"
+  gem.add_development_dependency "rubocop", "~> 1.9.1"
+
+  if RUBY_ENGINE == "jruby"
+    gem.add_development_dependency "activerecord-jdbcmysql-adapter"
+    gem.add_development_dependency "activerecord-jdbcpostgresql-adapter"
+    gem.add_development_dependency "ruby-debug"
+  else
+    gem.add_development_dependency "activerecord-postgis-adapter"
+    gem.add_development_dependency "pry-byebug"
+    gem.add_development_dependency "mysql2"
+    gem.add_development_dependency "pg"
+    gem.add_development_dependency "rgeo"
+  end
 end

data/spec/active_record/connection_adapters/makara_abstract_adapter_error_handling_spec.rb

@@ -2,7 +2,6 @@ require 'spec_helper'
 require 'active_record/connection_adapters/makara_abstract_adapter'
 
 describe ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter::ErrorHandler do
-
   let(:handler){ described_class.new }
   let(:proxy) { FakeAdapter.new(config(1,1)) }
   let(:connection){ proxy.master_pool.connections.first }
@@ -60,9 +59,8 @@ describe ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter::ErrorHandler d
   end
 
   describe 'custom errors' do
-
     let(:config_path) { File.join(File.expand_path('../../../', __FILE__), 'support', 'mysql2_database_with_custom_errors.yml') }
-    let(:config) { YAML.load_file(config_path)['test'] }
+    let(:config) { YAML.load(ERB.new(File.read(config_path)).result)['test'] }
     let(:handler){ described_class.new }
     let(:proxy) { FakeAdapter.new(config) }
     let(:connection){ proxy.master_pool.connections.first }
@@ -85,8 +83,5 @@ describe ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter::ErrorHandler d
         end
       }.to raise_error(Makara::Errors::BlacklistConnection)
     end
-
   end
-
-
 end

data/spec/active_record/connection_adapters/makara_abstract_adapter_spec.rb

@@ -2,7 +2,6 @@ require 'spec_helper'
 require 'active_record/connection_adapters/makara_abstract_adapter'
 
 describe ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter do
-
   let(:klass){ FakeAdapter }
 
   {
@@ -38,15 +37,12 @@ describe ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter do
     'select pg_advisory_lock(12345)' => true,
     'select pg_advisory_unlock(12345)' => true
   }.each do |sql, should_go_to_master|
-
     it "determines that \"#{sql}\" #{should_go_to_master ? 'requires' : 'does not require'} master" do
       proxy = klass.new(config(1,1))
       expect(proxy.master_for?(sql)).to eq(should_go_to_master)
     end
-
   end
 
-
   {
     "SET @@things" => true,
     "INSERT INTO wisdom ('The truth will set you free.')" => false,
@@ -61,7 +57,6 @@ describe ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter do
         max_treats IS NULL
     } => false
   }.each do |sql, should_send_to_all_connections|
-
     it "determines that \"#{sql}\" #{should_send_to_all_connections ? 'should' : 'should not'} be sent to all underlying connections" do
       proxy = klass.new(config(1,1))
       proxy.master_pool.connections.each{|con| expect(con).to receive(:execute).with(sql).once}
@@ -75,7 +70,6 @@ describe ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter do
 
       proxy.execute(sql)
     end
-
   end
 
   {
@@ -108,12 +102,9 @@ describe ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter do
         max_treats IS NULL
     } => true
   }.each do |sql,should_stick|
-
     it "should #{should_stick ? 'stick' : 'not stick'} to master if handling sql like \"#{sql}\"" do
       proxy = klass.new(config(0,0))
       expect(proxy.would_stick?(sql)).to eq(should_stick)
     end
-
   end
-
 end