activerecord-turntable 1.0.0

data/activerecord-turntable.gemspec

@@ -0,0 +1,47 @@
+$:.push File.expand_path("../lib", __FILE__)
+require "active_record/turntable/version"
+
+Gem::Specification.new do |s|
+  s.name = "activerecord-turntable"
+  s.version = ActiveRecord::Turntable::VERSION
+  s.authors     = ["gussan"]
+  s.homepage    = "https://github.com/drecom/activerecord-turntable"
+  s.summary = %q{ActiveRecord Sharding plugin}
+  s.description = %q{AcviveRecord Sharding plugin}
+
+  s.rubyforge_project = "activerecord-turntable"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+
+  s.files         = `git ls-files | grep -v "^spec"`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.licenses = ["MIT"]
+  s.rubygems_version = "1.8.16"
+
+  # runtime dependencies
+  s.add_runtime_dependency(%q<activerecord>, [">= 3.0.0"])
+  s.add_runtime_dependency(%q<activesupport>, [">=3.0.0"])
+  s.add_runtime_dependency(%q<sql_tree>, ["= 0.2.0"])
+  s.add_runtime_dependency(%q<bsearch>, ["~> 1.5"])
+  s.add_runtime_dependency(%q<httpclient>, [">= 0"])
+
+  # development dependencies
+  s.add_development_dependency(%q<rake>, ["~> 0.9.2"])
+  s.add_development_dependency(%q<rspec>, [">= 0"])
+  s.add_development_dependency(%q<rr>, [">= 0"])
+  s.add_development_dependency(%q<mysql2>, [">= 0"])
+  s.add_development_dependency(%q<fabrication>, [">= 0"])
+  s.add_development_dependency(%q<faker>, [">= 0"])
+  s.add_development_dependency(%q<activerecord-import>, [">= 0"])
+  s.add_development_dependency(%q<pry>, [">= 0"])
+  s.add_development_dependency(%q<guard-rspec>, [">= 0"])
+
+  if RUBY_PLATFORM =~ /darwin/
+    s.add_development_dependency(%q<growl>, [">= 0"])
+  end
+end
+

data/lib/active_record/turntable.rb

@@ -0,0 +1,58 @@
+#
+#= ActiveRecord::Turntable
+#
+# ActiveRecord Sharding Plugin
+#
+require 'active_record/turntable/version'
+require 'active_record'
+require 'active_record/fixtures'
+require 'active_support/concern'
+require 'active_record/turntable/error'
+require 'logger'
+require 'singleton'
+
+module ActiveRecord::Turntable
+  extend ActiveSupport::Concern
+
+  autoload :ActiveRecordExt, 'active_record/turntable/active_record_ext'
+  autoload :Algorithm, 'active_record/turntable/algorithm'
+  autoload :Base, 'active_record/turntable/base'
+  autoload :Cluster, 'active_record/turntable/cluster'
+  autoload :Config, 'active_record/turntable/config'
+  autoload :Compatible, "active_record/turntable/compatible"
+  autoload :ConnectionProxy, 'active_record/turntable/connection_proxy'
+  autoload :Helpers, 'active_record/turntable/helpers'
+  autoload :MasterShard, 'active_record/turntable/master_shard'
+  autoload :Migration, 'active_record/turntable/migration'
+  autoload :Mixer, 'active_record/turntable/mixer'
+  autoload :PoolProxy, 'active_record/turntable/pool_proxy'
+  autoload :Rack, 'active_record/turntable/rack'
+  autoload :SeqShard, 'active_record/turntable/seq_shard'
+  autoload :Sequencer, 'active_record/turntable/sequencer'
+  autoload :Shard, 'active_record/turntable/shard'
+
+  included do
+    include ActiveRecordExt
+    include Base
+  end
+
+  module ClassMethods
+    DEFAULT_PATH = File.dirname(File.dirname(__FILE__))
+
+    def turntable_config_file
+      @@turntable_config_file ||=
+        File.join(defined?(::Rails) ?
+                   ::Rails.root.to_s : DEFAULT_PATH, 'config/turntable.yml')
+    end
+
+    def turntable_config_file=(filename)
+      @@turntable_config_file = filename
+    end
+
+    def turntable_config
+      ActiveRecord::Turntable::Config.instance
+    end
+  end
+
+  require "active_record/turntable/railtie" if defined?(Rails)
+end

data/lib/active_record/turntable/active_record_ext.rb

@@ -0,0 +1,26 @@
+module ActiveRecord::Turntable
+  module ActiveRecordExt
+    extend ActiveSupport::Concern
+
+    autoload :AbstractAdapter, 'active_record/turntable/active_record_ext/abstract_adapter'
+    autoload :CleverLoad, 'active_record/turntable/active_record_ext/clever_load'
+    autoload :LogSubscriber, 'active_record/turntable/active_record_ext/log_subscriber'
+    autoload :Persistence, 'active_record/turntable/active_record_ext/persistence'
+    autoload :SchemaDumper, 'active_record/turntable/active_record_ext/schema_dumper'
+    autoload :Sequencer, 'active_record/turntable/active_record_ext/sequencer'
+    autoload :Transactions, 'active_record/turntable/active_record_ext/transactions'
+
+    included do
+      include Transactions
+      ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:include, Sequencer)
+      ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:include, AbstractAdapter)
+      ActiveRecord::LogSubscriber.send(:include, LogSubscriber)
+      ActiveRecord::Persistence.send(:include, Persistence)
+      ActiveRecord::Relation.send(:include, CleverLoad)
+      ActiveRecord::VERSION::STRING < '3.1' ?
+        ActiveRecord::Migration.extend(ActiveRecord::Turntable::Migration) :
+        ActiveRecord::Migration.send(:include, ActiveRecord::Turntable::Migration)
+      require 'active_record/turntable/active_record_ext/fixtures'
+    end
+  end
+end

data/lib/active_record/turntable/active_record_ext/abstract_adapter.rb

@@ -0,0 +1,50 @@
+module ActiveRecord::Turntable
+  module ActiveRecordExt
+    module AbstractAdapter
+      extend ActiveSupport::Concern
+
+      included do
+        protected
+
+        if ActiveRecord::VERSION::STRING < '3.1'
+          def log(sql, name)
+            name ||= "SQL"
+            @instrumenter.instrument("sql.active_record",
+                                     :sql => sql, :name => name, :connection_id => object_id,
+                                     :turntable_shard_name => turntable_shard_name) do
+              yield
+            end
+          rescue Exception => e
+            message = "#{e.class.name}: #{e.message}: #{sql} : #{turntable_shard_name}"
+            @logger.debug message if @logger
+            raise translate_exception(e, message)
+          end
+        else
+          def log(sql, name = "SQL", binds = [])
+            @instrumenter.instrument(
+                                     "sql.active_record",
+                                     :sql           => sql,
+                                     :name          => name,
+                                     :connection_id => object_id,
+                                     :binds         => binds,
+                                     :turntable_shard_name => turntable_shard_name) { yield }
+          rescue Exception => e
+            message = "#{e.class.name}: #{e.message}: #{sql} : #{turntable_shard_name}"
+            @logger.debug message if @logger
+            exception = translate_exception(e, message)
+            exception.set_backtrace e.backtrace
+            raise exception
+          end
+        end
+      end
+
+      def turntable_shard_name=(name)
+        @turntable_shard_name = name.to_s
+      end
+
+      def turntable_shard_name
+        @turntable_shard_name
+      end
+    end
+  end
+end

data/lib/active_record/turntable/active_record_ext/clever_load.rb

@@ -0,0 +1,90 @@
+module ActiveRecord::Turntable::ActiveRecordExt
+  module CleverLoad
+    extend ActiveSupport::Concern
+
+    included do
+      ActiveRecord::VERSION::STRING < '3.1' ?
+        include(AR30):
+        include(AR31)
+
+      class << ActiveRecord::Base
+        delegate :clever_load!, :to => :scoped
+      end
+    end
+
+    module AR30
+      def clever_load!(association_name)
+        # load records
+        records = self.to_a
+        klass = records.first.class
+        reflection = klass.reflections[association_name]
+
+        if reflection
+          foreign_class = reflection.klass
+          foreign_objects = case reflection.macro
+                            when :has_one
+                              foreign_class.where(reflection.primary_key_name => records.map(&reflection.association_primary_key.to_sym).uniq)
+                            when :belongs_to
+                              foreign_class.where(reflection.association_primary_key => records.map(&reflection.primary_key_name.to_sym).uniq)
+                            else
+                              []
+                            end
+
+          self.each do |obj|
+            matched_object = case reflection.macro
+                             when :has_one
+                               foreign_objects.find {|fo|
+                                 obj.send(reflection.association_primary_key) == fo.send(reflection.primary_key_name)
+                               }
+                             when :belongs_to
+                               foreign_objects.find {|fo|
+                                 obj.send(reflection.primary_key_name) == fo.send(reflection.association_primary_key)
+                               }
+                             end
+            association_proxy = obj.send("set_#{reflection.name}_target", matched_object)
+            # TODO: set reverse_instance
+            # association_proxy.send(:set_inverse_instance, matched_object, obj)
+          end
+        end
+        records
+      end
+    end
+
+    module AR31
+      def clever_load!(association_name)
+        # load records
+        records = self.to_a
+        klass = records.first.class
+        reflection = klass.reflections[association_name]
+
+        if reflection
+          foreign_class = reflection.klass
+          foreign_objects = case reflection.macro
+                            when :has_one
+                              foreign_class.where(reflection.foreign_key => records.map(&reflection.association_primary_key.to_sym).uniq)
+                            when :belongs_to
+                              foreign_class.where(reflection.association_primary_key => records.map(&reflection.foreign_key.to_sym).uniq)
+                            else
+                              []
+                            end
+
+          self.each do |obj|
+            matched_object = case reflection.macro
+                             when :has_one
+                               foreign_objects.find {|fo|
+                                 obj.send(reflection.association_primary_key) == fo.send(reflection.foreign_key)
+                               }
+                             when :belongs_to
+                               foreign_objects.find {|fo|
+                                 obj.send(reflection.foreign_key) == fo.send(reflection.association_primary_key)
+                               }
+                             end
+            obj.association(association_name).target = matched_object
+            obj.association(association_name).send(:set_inverse_instance, matched_object)
+          end
+        end
+        records
+      end
+    end
+  end
+end

data/lib/active_record/turntable/active_record_ext/fixtures.rb

@@ -0,0 +1,131 @@
+#
+# force TestFixtures to begin transaction with all shards.
+#
+require 'active_record/fixtures'
+module ActiveRecord
+  class Fixtures
+    def self.create_fixtures(fixtures_directory, table_names, class_names = {})
+      table_names = [table_names].flatten.map { |n| n.to_s }
+      table_names.each { |n|
+        class_names[n.tr('/', '_').to_sym] = n.classify if n.include?('/')
+      }
+
+      # FIXME: Apparently JK uses this.
+      connection = block_given? ? yield : ActiveRecord::Base.connection
+
+      files_to_read = table_names.reject { |table_name|
+        fixture_is_cached?(connection, table_name)
+      }
+
+      unless files_to_read.empty?
+        connection.disable_referential_integrity do
+          fixtures_map = {}
+
+          fixture_files = files_to_read.map do |path|
+            table_name = path.tr '/', '_'
+
+            fixtures_map[path] = ActiveRecord::Fixtures.new(
+              connection,
+              table_name,
+              class_names[table_name.to_sym] || table_name.classify,
+              ::File.join(fixtures_directory, path))
+          end
+
+          all_loaded_fixtures.update(fixtures_map)
+
+          ActiveRecord::Turntable::Base.force_transaction_all_shards!(:requires_new => true) do
+            fixture_files.each do |ff|
+              conn = ff.model_class.respond_to?(:connection) ? ff.model_class.connection : connection
+              table_rows = ff.table_rows
+
+              table_rows.keys.each do |table|
+                conn.delete "DELETE FROM #{conn.quote_table_name(table)}", 'Fixture Delete'
+              end
+
+              table_rows.each do |table_name,rows|
+                rows.each do |row|
+                  conn.insert_fixture(row, table_name)
+                end
+              end
+            end
+
+            # Cap primary key sequences to max(pk).
+            if connection.respond_to?(:reset_pk_sequence!)
+              table_names.each do |table_name|
+                connection.reset_pk_sequence!(table_name.tr('/', '_'))
+              end
+            end
+          end
+
+          cache_fixtures(connection, fixtures_map)
+        end
+      end
+      cached_fixtures(connection, table_names)
+    end
+
+  end
+
+  module TestFixtures
+    def setup_fixtures
+      return unless !ActiveRecord::Base.configurations.blank?
+
+      if pre_loaded_fixtures && !use_transactional_fixtures
+        raise RuntimeError, 'pre_loaded_fixtures requires use_transactional_fixtures'
+      end
+
+      @fixture_cache = {}
+      @fixture_connections = []
+      @@already_loaded_fixtures ||= {}
+
+      # Load fixtures once and begin transaction.
+      if run_in_transaction?
+        if @@already_loaded_fixtures[self.class]
+          @loaded_fixtures = @@already_loaded_fixtures[self.class]
+        else
+          @loaded_fixtures = load_fixtures
+          @@already_loaded_fixtures[self.class] = @loaded_fixtures
+        end
+        ActiveRecord::Base.force_connect_all_shards!
+        @fixture_connections = enlist_fixture_connections
+        @fixture_connections.each do |connection|
+          connection.increment_open_transactions
+          connection.transaction_joinable = false
+          connection.begin_db_transaction
+        end
+        # Load fixtures for every test.
+      else
+        ActiveRecord::Fixtures.reset_cache
+        @@already_loaded_fixtures[self.class] = nil
+        @loaded_fixtures = load_fixtures
+      end
+
+      # Instantiate fixtures for every test if requested.
+      instantiate_fixtures if use_instantiated_fixtures
+    end
+
+    def enlist_fixture_connections
+      ActiveRecord::Base.connection_handler.connection_pools.values.map(&:connection) +
+        ActiveRecord::Base.turntable_connections.values.map(&:connection)
+    end
+
+    def teardown_fixtures
+      return unless defined?(ActiveRecord) && !ActiveRecord::Base.configurations.blank?
+
+      unless run_in_transaction?
+        ActiveRecord::Fixtures.reset_cache
+      end
+
+      # Rollback changes if a transaction is active.
+      if run_in_transaction?
+        @fixture_connections.each do |connection|
+          if connection.open_transactions != 0
+            connection.rollback_db_transaction
+            connection.decrement_open_transactions
+          end
+        end
+        @fixture_connections.clear
+      end
+      ActiveRecord::Base.clear_active_connections!
+    end
+  end
+end

data/lib/active_record/turntable/active_record_ext/log_subscriber.rb

@@ -0,0 +1,64 @@
+module ActiveRecord::Turntable
+  module ActiveRecordExt
+    module LogSubscriber
+      extend ActiveSupport::Concern
+
+      included do
+        if ActiveRecord::VERSION::STRING < '3.1'
+          def sql(event)
+            self.class.runtime += event.duration
+            return unless logger.debug?
+
+            name = '%s (%.1fms)' % [event.payload[:name], event.duration]
+            shard = '[Shard: %s]' % (event.payload[:turntable_shard_name] ? event.payload[:turntable_shard_name] : nil)
+            sql  = event.payload[:sql].squeeze(' ')
+
+            if odd?
+              name = color(name, ActiveRecord::LogSubscriber::CYAN, true)
+              shard = color(shard, ActiveRecord::LogSubscriber::CYAN, true)
+              sql  = color(sql, nil, true)
+            else
+              name = color(name, ActiveRecord::LogSubscriber::MAGENTA, true)
+              shard = color(shard, ActiveRecord::LogSubscriber::MAGENTA, true)
+            end
+
+            debug "  #{name} #{shard} #{sql}"
+          end
+
+        else
+          def sql(event)
+            self.class.runtime += event.duration
+            return unless logger.debug?
+
+            payload = event.payload
+
+            return if 'SCHEMA' == payload[:name]
+
+            name    = '%s (%.1fms)' % [payload[:name], event.duration]
+            shard = '[Shard: %s]' % (event.payload[:turntable_shard_name] ? event.payload[:turntable_shard_name] : nil)
+            sql     = payload[:sql].squeeze(' ')
+            binds   = nil
+
+            unless (payload[:binds] || []).empty?
+              binds = "  " + payload[:binds].map { |col,v|
+                [col.name, v]
+              }.inspect
+            end
+
+            if odd?
+              name = color(name, ActiveRecord::LogSubscriber::CYAN, true)
+              shard = color(shard, ActiveRecord::LogSubscriber::CYAN, true)
+              sql  = color(sql, nil, true)
+            else
+              name = color(name, ActiveRecord::LogSubscriber::MAGENTA, true)
+              shard = color(shard, ActiveRecord::LogSubscriber::MAGENTA, true)
+            end
+
+            debug "  #{name} #{shard} #{sql}#{binds}"
+          end
+        end
+      end
+    end
+  end
+end
+