yam-db-charmer 1.7.01

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/.gitignore +4 -0
  2. data/CHANGES +184 -0
  3. data/LICENSE +21 -0
  4. data/Makefile +2 -0
  5. data/README.rdoc +612 -0
  6. data/Rakefile +4 -0
  7. data/db-charmer.gemspec +29 -0
  8. data/init.rb +1 -0
  9. data/lib/db_charmer/action_controller/force_slave_reads.rb +69 -0
  10. data/lib/db_charmer/active_record/association_preload.rb +23 -0
  11. data/lib/db_charmer/active_record/class_attributes.rb +101 -0
  12. data/lib/db_charmer/active_record/connection_switching.rb +81 -0
  13. data/lib/db_charmer/active_record/db_magic.rb +85 -0
  14. data/lib/db_charmer/active_record/migration/multi_db_migrations.rb +71 -0
  15. data/lib/db_charmer/active_record/multi_db_proxy.rb +77 -0
  16. data/lib/db_charmer/active_record/sharding.rb +40 -0
  17. data/lib/db_charmer/connection_factory.rb +76 -0
  18. data/lib/db_charmer/connection_proxy.rb +27 -0
  19. data/lib/db_charmer/core_extensions.rb +23 -0
  20. data/lib/db_charmer/force_slave_reads.rb +36 -0
  21. data/lib/db_charmer/rails2/abstract_adapter/log_formatting.rb +24 -0
  22. data/lib/db_charmer/rails2/active_record/master_slave_routing.rb +49 -0
  23. data/lib/db_charmer/rails2/active_record/named_scope/scope_proxy.rb +26 -0
  24. data/lib/db_charmer/rails3/abstract_adapter/connection_name.rb +38 -0
  25. data/lib/db_charmer/rails3/active_record/log_subscriber.rb +23 -0
  26. data/lib/db_charmer/rails3/active_record/master_slave_routing.rb +46 -0
  27. data/lib/db_charmer/rails3/active_record/relation/connection_routing.rb +147 -0
  28. data/lib/db_charmer/rails3/active_record/relation_method.rb +28 -0
  29. data/lib/db_charmer/sharding/connection.rb +31 -0
  30. data/lib/db_charmer/sharding/method/db_block_group_map.rb +257 -0
  31. data/lib/db_charmer/sharding/method/db_block_map.rb +211 -0
  32. data/lib/db_charmer/sharding/method/hash_map.rb +23 -0
  33. data/lib/db_charmer/sharding/method/range.rb +33 -0
  34. data/lib/db_charmer/sharding/method.rb +10 -0
  35. data/lib/db_charmer/sharding/stub_connection.rb +60 -0
  36. data/lib/db_charmer/sharding.rb +18 -0
  37. data/lib/db_charmer/version.rb +10 -0
  38. data/lib/db_charmer.rb +192 -0
  39. data/lib/tasks/databases.rake +82 -0
  40. metadata +178 -0
@@ -0,0 +1,77 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module MultiDbProxy
4
+ # Simple proxy class that switches connections and then proxies all the calls
5
+ # This class is used to implement chained on_db calls
6
+ class OnDbProxy < ActiveSupport::BasicObject
7
+ # We need to do this because in Rails 2.3 BasicObject does not remove object_id method, which is stupid
8
+ undef_method(:object_id) if instance_methods.member?('object_id')
9
+
10
+ def initialize(proxy_target, slave)
11
+ @proxy_target = proxy_target
12
+ @slave = slave
13
+ end
14
+
15
+ private
16
+
17
+ def method_missing(meth, *args, &block)
18
+ # Switch connection and proxy the method call
19
+ @proxy_target.on_db(@slave) do |proxy_target|
20
+ res = proxy_target.__send__(meth, *args, &block)
21
+
22
+ # If result is a scope/association, return a new proxy for it, otherwise return the result itself
23
+ (res.proxy?) ? OnDbProxy.new(res, @slave) : res
24
+ end
25
+ end
26
+ end
27
+
28
+ module ClassMethods
29
+ def on_db(con, proxy_target = nil)
30
+ proxy_target ||= self
31
+
32
+ # Chain call
33
+ return OnDbProxy.new(proxy_target, con) unless block_given?
34
+
35
+ # Block call
36
+ begin
37
+ self.db_charmer_connection_level += 1
38
+ old_proxy = db_charmer_connection_proxy
39
+ switch_connection_to(con, DbCharmer.connections_should_exist?)
40
+ yield(proxy_target)
41
+ ensure
42
+ switch_connection_to(old_proxy)
43
+ self.db_charmer_connection_level -= 1
44
+ end
45
+ end
46
+ end
47
+
48
+ module InstanceMethods
49
+ def on_db(con, proxy_target = nil, &block)
50
+ proxy_target ||= self
51
+ self.class.on_db(con, proxy_target, &block)
52
+ end
53
+ end
54
+
55
+ module MasterSlaveClassMethods
56
+ def on_slave(con = nil, proxy_target = nil, &block)
57
+ con ||= db_charmer_random_slave
58
+ raise ArgumentError, "No slaves found in the class and no slave connection given" unless con
59
+ on_db(con, proxy_target, &block)
60
+ end
61
+
62
+ def on_master(proxy_target = nil, &block)
63
+ on_db(db_charmer_default_connection, proxy_target, &block)
64
+ end
65
+
66
+ def first_level_on_slave
67
+ first_level = db_charmer_top_level_connection? && on_master.connection.open_transactions.zero?
68
+ if first_level && db_charmer_force_slave_reads? && db_charmer_slaves.any?
69
+ on_slave { yield }
70
+ else
71
+ yield
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,40 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module Sharding
4
+
5
+ def self.extended(model)
6
+ model.cattr_accessor(:sharded_connection)
7
+ end
8
+
9
+ def shard_for(key, proxy_target = nil, &block)
10
+ raise ArgumentError, "No sharded connection configured!" unless sharded_connection
11
+ conn = sharded_connection.sharder.shard_for_key(key)
12
+ on_db(conn, proxy_target, &block)
13
+ end
14
+
15
+ # Run on default shard (if supported by the sharding method)
16
+ def on_default_shard(proxy_target = nil, &block)
17
+ raise ArgumentError, "No sharded connection configured!" unless sharded_connection
18
+
19
+ if sharded_connection.support_default_shard?
20
+ shard_for(:default, proxy_target, &block)
21
+ else
22
+ raise ArgumentError, "This model's sharding method does not support default shard"
23
+ end
24
+ end
25
+
26
+ # Enumerate shards
27
+ def on_each_shard(proxy_target = nil, &block)
28
+ raise ArgumentError, "No sharded connection configured!" unless sharded_connection
29
+
30
+ conns = sharded_connection.shard_connections
31
+ raise ArgumentError, "This model's sharding method does not support shards enumeration" unless conns
32
+
33
+ conns.each do |conn|
34
+ on_db(conn, proxy_target, &block)
35
+ end
36
+ end
37
+
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,76 @@
1
+ #
2
+ # This class is used to automatically generate small abstract ActiveRecord classes
3
+ # that would then be used as a source of database connections for DbCharmer magic.
4
+ # This way we do not need to re-implement all the connection establishing code
5
+ # that ActiveRecord already has and we make our code less dependant on Rails versions.
6
+ #
7
+ module DbCharmer
8
+ module ConnectionFactory
9
+ @@connection_classes = {}
10
+
11
+ def self.reset!
12
+ @@connection_classes = {}
13
+ end
14
+
15
+ # Establishes connection or return an existing one from cache
16
+ def self.connect(connection_name, should_exist = true)
17
+ connection_name = connection_name.to_s
18
+ @@connection_classes[connection_name] ||= establish_connection(connection_name, should_exist)
19
+ end
20
+
21
+ # Establishes connection or return an existing one from cache (not using AR database configs)
22
+ def self.connect_to_db(connection_name, config)
23
+ connection_name = connection_name.to_s
24
+ @@connection_classes[connection_name] ||= establish_connection_to_db(connection_name, config)
25
+ end
26
+
27
+ # Establish connection with a specified name
28
+ def self.establish_connection(connection_name, should_exist = true)
29
+ abstract_class = generate_abstract_class(connection_name, should_exist)
30
+ DbCharmer::ConnectionProxy.new(abstract_class, connection_name)
31
+ end
32
+
33
+ # Establish connection with a specified name (not using AR database configs)
34
+ def self.establish_connection_to_db(connection_name, config)
35
+ abstract_class = generate_abstract_class_for_db(connection_name, config)
36
+ DbCharmer::ConnectionProxy.new(abstract_class, connection_name)
37
+ end
38
+
39
+ # Generate an abstract AR class with specified connection established
40
+ def self.generate_abstract_class(connection_name, should_exist = true)
41
+ # Generate class
42
+ klass = generate_empty_abstract_ar_class(abstract_connection_class_name(connection_name))
43
+
44
+ # Establish connection
45
+ klass.establish_real_connection_if_exists(connection_name.to_sym, !!should_exist)
46
+
47
+ # Return the class
48
+ return klass
49
+ end
50
+
51
+ # Generate an abstract AR class with specified connection established (not using AR database configs)
52
+ def self.generate_abstract_class_for_db(connection_name, config)
53
+ # Generate class
54
+ klass = generate_empty_abstract_ar_class(abstract_connection_class_name(connection_name))
55
+
56
+ # Establish connection
57
+ klass.establish_connection(config)
58
+
59
+ # Return the class
60
+ return klass
61
+ end
62
+
63
+ def self.generate_empty_abstract_ar_class(klass)
64
+ # Define class
65
+ module_eval "class #{klass} < ::ActiveRecord::Base; self.abstract_class = true; end"
66
+
67
+ # Return class
68
+ klass.constantize
69
+ end
70
+
71
+ # Generates unique names for our abstract AR classes
72
+ def self.abstract_connection_class_name(connection_name)
73
+ "::AutoGeneratedAbstractConnectionClass#{connection_name.to_s.gsub(/\W+/, '_').camelize}"
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,27 @@
1
+ # Simple proxy that sends all method calls to a real database connection
2
+ module DbCharmer
3
+ class ConnectionProxy < ActiveSupport::BasicObject
4
+ # We need to do this because in Rails 2.3 BasicObject does not remove object_id method, which is stupid
5
+ undef_method(:object_id) if instance_methods.member?('object_id')
6
+
7
+ # We use this to get a connection class from the proxy
8
+ attr_accessor :abstract_connection_class
9
+
10
+ def initialize(abstract_class, db_name)
11
+ @abstract_connection_class = abstract_class
12
+ @db_name = db_name
13
+ end
14
+
15
+ def db_charmer_connection_name
16
+ @db_name
17
+ end
18
+
19
+ def db_charmer_connection_proxy
20
+ self
21
+ end
22
+
23
+ def method_missing(meth, *args, &block)
24
+ @abstract_connection_class.retrieve_connection.send(meth, *args, &block)
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,23 @@
1
+ class Object
2
+ unless defined?(try)
3
+ def try(method, *options, &block)
4
+ send(method, *options, &block)
5
+ end
6
+ end
7
+
8
+ # These methods are added to all objects so we could call proxy? on anything
9
+ # and figure if an object is a proxy w/o hitting method_missing or respond_to?
10
+ def self.proxy?
11
+ false
12
+ end
13
+
14
+ def proxy?
15
+ false
16
+ end
17
+ end
18
+
19
+ class NilClass
20
+ def try(method, *options, &block)
21
+ nil
22
+ end
23
+ end
@@ -0,0 +1,36 @@
1
+ module DbCharmer
2
+ @@current_controller = nil
3
+ mattr_accessor :current_controller
4
+
5
+ @@forced_slave_reads = false
6
+
7
+ def self.force_slave_reads?
8
+ # If global force slave reads is requested, do it
9
+ return @@forced_slave_reads if @@forced_slave_reads
10
+
11
+ # If not, try to use current controller to decide on this
12
+ return false unless current_controller.respond_to?(:force_slave_reads?)
13
+
14
+ slave_reads = current_controller.force_slave_reads?
15
+ logger.debug("Using controller to figure out if slave reads should be forced: #{slave_reads}")
16
+ return slave_reads
17
+ end
18
+
19
+ def self.with_controller(controller)
20
+ raise ArgumentError, "No block given" unless block_given?
21
+ logger.debug("Setting current controller for db_charmer: #{controller.class.name}")
22
+ self.current_controller = controller
23
+ yield
24
+ ensure
25
+ logger.debug('Clearing current controller for db_charmer')
26
+ self.current_controller = nil
27
+ end
28
+
29
+ def self.force_slave_reads
30
+ raise ArgumentError, "No block given" unless block_given?
31
+ @@forced_slave_reads = true
32
+ yield
33
+ ensure
34
+ @@forced_slave_reads = false
35
+ end
36
+ end
@@ -0,0 +1,24 @@
1
+ module DbCharmer
2
+ module AbstractAdapter
3
+ module LogFormatting
4
+ # TODO: this causes stack overflows
5
+ =begin
6
+ def self.included(base)
7
+ base.alias_method_chain :format_log_entry, :connection_name
8
+ end
9
+
10
+ def connection_name
11
+ raise "Can't find connection configuration!" unless @config
12
+ @config[:connection_name]
13
+ end
14
+
15
+ # Rails 2.X specific logging method
16
+ def format_log_entry_with_connection_name(message, dump = nil)
17
+ msg = connection_name ? "[#{connection_name}] " : ''
18
+ msg = " \e[0;34;1m#{msg}\e[0m" if connection_name && ::ActiveRecord::Base.colorize_logging
19
+ msg << format_log_entry_without_connection_name(message, dump)
20
+ end
21
+ =end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,49 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module MasterSlaveRouting
4
+
5
+ module ClassMethods
6
+ SLAVE_METHODS = [ :find_by_sql, :count_by_sql, :calculate ]
7
+ MASTER_METHODS = [ :update, :create, :delete, :destroy, :delete_all, :destroy_all, :update_all, :update_counters ]
8
+
9
+ SLAVE_METHODS.each do |slave_method|
10
+ class_eval <<-EOF, __FILE__, __LINE__ + 1
11
+ def #{slave_method}(*args, &block)
12
+ first_level_on_slave do
13
+ super(*args, &block)
14
+ end
15
+ end
16
+ EOF
17
+ end
18
+
19
+ MASTER_METHODS.each do |master_method|
20
+ class_eval <<-EOF, __FILE__, __LINE__ + 1
21
+ def #{master_method}(*args, &block)
22
+ on_master do
23
+ super(*args, &block)
24
+ end
25
+ end
26
+ EOF
27
+ end
28
+
29
+ def find(*args, &block)
30
+ options = args.last
31
+ if options.is_a?(Hash) && options[:lock]
32
+ on_master { super(*args, &block) }
33
+ else
34
+ super(*args, &block)
35
+ end
36
+ end
37
+ end
38
+
39
+ module InstanceMethods
40
+ def reload(*args, &block)
41
+ self.class.on_master do
42
+ super(*args, &block)
43
+ end
44
+ end
45
+ end
46
+
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,26 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module NamedScope
4
+ module ScopeProxy
5
+
6
+ def proxy?
7
+ true
8
+ end
9
+
10
+ def on_db(con, proxy_target = nil, &block)
11
+ proxy_target ||= self
12
+ proxy_scope.on_db(con, proxy_target, &block)
13
+ end
14
+
15
+ def on_slave(con = nil, &block)
16
+ proxy_scope.on_slave(con, self, &block)
17
+ end
18
+
19
+ def on_master(&block)
20
+ proxy_scope.on_master(self, &block)
21
+ end
22
+
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,38 @@
1
+ module DbCharmer
2
+ module AbstractAdapter
3
+ module ConnectionName
4
+
5
+ # We use this proxy to push connection name down to instrumenters w/o monkey-patching the log method itself
6
+ class InstrumenterDecorator < ActiveSupport::BasicObject
7
+ def initialize(adapter, instrumenter)
8
+ @adapter = adapter
9
+ @instrumenter = instrumenter
10
+ end
11
+
12
+ def instrument(name, payload = {}, &block)
13
+ payload[:connection_name] ||= @adapter.connection_name
14
+ @instrumenter.instrument(name, payload, &block)
15
+ end
16
+
17
+ def method_missing(meth, *args, &block)
18
+ @instrumenter.send(meth, *args, &block)
19
+ end
20
+ end
21
+
22
+ def self.included(base)
23
+ base.alias_method_chain :initialize, :connection_name
24
+ end
25
+
26
+ def connection_name
27
+ raise "Can't find connection configuration!" unless @config
28
+ @config[:connection_name]
29
+ end
30
+
31
+ def initialize_with_connection_name(*args)
32
+ initialize_without_connection_name(*args)
33
+ @instrumenter = InstrumenterDecorator.new(self, @instrumenter)
34
+ end
35
+
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,23 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module LogSubscriber
4
+
5
+ def self.included(base)
6
+ base.send(:attr_accessor, :connection_name)
7
+ base.alias_method_chain :sql, :connection_name
8
+ base.alias_method_chain :debug, :connection_name
9
+ end
10
+
11
+ def sql_with_connection_name(event)
12
+ self.connection_name = event.payload[:connection_name]
13
+ sql_without_connection_name(event)
14
+ end
15
+
16
+ def debug_with_connection_name(msg)
17
+ conn = connection_name ? color(" [#{connection_name}]", ActiveSupport::LogSubscriber::BLUE, true) : ''
18
+ debug_without_connection_name(conn + msg)
19
+ end
20
+
21
+ end
22
+ end
23
+ end