yam-db-charmer 1.7.01

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/.gitignore +4 -0
  2. data/CHANGES +184 -0
  3. data/LICENSE +21 -0
  4. data/Makefile +2 -0
  5. data/README.rdoc +612 -0
  6. data/Rakefile +4 -0
  7. data/db-charmer.gemspec +29 -0
  8. data/init.rb +1 -0
  9. data/lib/db_charmer/action_controller/force_slave_reads.rb +69 -0
  10. data/lib/db_charmer/active_record/association_preload.rb +23 -0
  11. data/lib/db_charmer/active_record/class_attributes.rb +101 -0
  12. data/lib/db_charmer/active_record/connection_switching.rb +81 -0
  13. data/lib/db_charmer/active_record/db_magic.rb +85 -0
  14. data/lib/db_charmer/active_record/migration/multi_db_migrations.rb +71 -0
  15. data/lib/db_charmer/active_record/multi_db_proxy.rb +77 -0
  16. data/lib/db_charmer/active_record/sharding.rb +40 -0
  17. data/lib/db_charmer/connection_factory.rb +76 -0
  18. data/lib/db_charmer/connection_proxy.rb +27 -0
  19. data/lib/db_charmer/core_extensions.rb +23 -0
  20. data/lib/db_charmer/force_slave_reads.rb +36 -0
  21. data/lib/db_charmer/rails2/abstract_adapter/log_formatting.rb +24 -0
  22. data/lib/db_charmer/rails2/active_record/master_slave_routing.rb +49 -0
  23. data/lib/db_charmer/rails2/active_record/named_scope/scope_proxy.rb +26 -0
  24. data/lib/db_charmer/rails3/abstract_adapter/connection_name.rb +38 -0
  25. data/lib/db_charmer/rails3/active_record/log_subscriber.rb +23 -0
  26. data/lib/db_charmer/rails3/active_record/master_slave_routing.rb +46 -0
  27. data/lib/db_charmer/rails3/active_record/relation/connection_routing.rb +147 -0
  28. data/lib/db_charmer/rails3/active_record/relation_method.rb +28 -0
  29. data/lib/db_charmer/sharding/connection.rb +31 -0
  30. data/lib/db_charmer/sharding/method/db_block_group_map.rb +257 -0
  31. data/lib/db_charmer/sharding/method/db_block_map.rb +211 -0
  32. data/lib/db_charmer/sharding/method/hash_map.rb +23 -0
  33. data/lib/db_charmer/sharding/method/range.rb +33 -0
  34. data/lib/db_charmer/sharding/method.rb +10 -0
  35. data/lib/db_charmer/sharding/stub_connection.rb +60 -0
  36. data/lib/db_charmer/sharding.rb +18 -0
  37. data/lib/db_charmer/version.rb +10 -0
  38. data/lib/db_charmer.rb +192 -0
  39. data/lib/tasks/databases.rake +82 -0
  40. metadata +178 -0
data/Rakefile ADDED
@@ -0,0 +1,4 @@
1
+ require 'rake'
2
+ require 'bundler'
3
+
4
+ Bundler::GemHelper.install_tasks
@@ -0,0 +1,29 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+ require 'db_charmer/version'
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = 'yam-db-charmer'
7
+ s.version = DbCharmer::Version::STRING
8
+ s.platform = Gem::Platform::RUBY
9
+
10
+ s.authors = [ 'Oleksiy Kovyrin', 'Mike Ihbe']
11
+ s.email = 'mihbe@yammer-inc.com'
12
+ s.homepage = 'http://kovyrin.github.com/db-charmer'
13
+ s.summary = 'ActiveRecord Connections Magic (slaves, multiple connections, etc)'
14
+ s.description = 'DbCharmer is a Rails plugin (and gem) that could be used to manage AR model connections, implement master/slave query schemes, sharding and other magic features many high-scale applications need.'
15
+
16
+ s.rdoc_options = [ '--charset=UTF-8' ]
17
+
18
+ s.files = `git ls-files`.split("\n")
19
+ s.require_paths = [ 'lib' ]
20
+ s.extra_rdoc_files = [ 'LICENSE', 'README.rdoc' ]
21
+
22
+ # Dependencies
23
+ s.add_dependency 'activesupport', '< 3.1'
24
+ s.add_dependency 'activerecord', '< 3.1'
25
+
26
+ s.add_development_dependency 'rspec'
27
+ s.add_development_dependency 'yard'
28
+ s.add_development_dependency 'actionpack'
29
+ end
data/init.rb ADDED
@@ -0,0 +1 @@
1
+ require 'db_charmer'
@@ -0,0 +1,69 @@
1
+ module DbCharmer
2
+ module ActionController
3
+ module ForceSlaveReads
4
+
5
+ module ClassMethods
6
+ @@db_charmer_force_slave_reads_actions = {}
7
+ def force_slave_reads(params)
8
+ @@db_charmer_force_slave_reads_actions[self.name] = {
9
+ :except => params[:except] ? [*params[:except]].map(&:to_s) : [],
10
+ :only => params[:only] ? [*params[:only]].map(&:to_s) : []
11
+ }
12
+ end
13
+
14
+ def force_slave_reads_options
15
+ @@db_charmer_force_slave_reads_actions[self.name]
16
+ end
17
+
18
+ def force_slave_reads_action?(name = nil)
19
+ name = name.to_s
20
+
21
+ options = force_slave_reads_options
22
+ # If no options were defined for this controller, all actions are not forced to use slaves
23
+ return false unless options
24
+
25
+ # Actions where force_slave_reads mode was turned off
26
+ return false if options[:except].include?(name)
27
+
28
+ # Only for these actions force_slave_reads was turned on
29
+ return options[:only].include?(name) if options[:only].any?
30
+
31
+ # If :except is not empty, we're done with the checks and rest of the actions are should force slave reads
32
+ # Otherwise, all the actions are not in force_slave_reads mode
33
+ options[:except].any?
34
+ end
35
+ end
36
+
37
+ module InstanceMethods
38
+ DISPATCH_METHOD = (DbCharmer.rails3?) ? :process_action : :perform_action
39
+
40
+ def self.included(base)
41
+ base.alias_method_chain DISPATCH_METHOD, :forced_slave_reads
42
+ end
43
+
44
+ def force_slave_reads!
45
+ @db_charmer_force_slave_reads = true
46
+ end
47
+
48
+ def dont_force_slave_reads!
49
+ @db_charmer_force_slave_reads = false
50
+ end
51
+
52
+ def force_slave_reads?
53
+ @db_charmer_force_slave_reads || self.class.force_slave_reads_action?(params[:action])
54
+ end
55
+
56
+ protected
57
+
58
+ class_eval <<-EOF
59
+ def #{DISPATCH_METHOD}_with_forced_slave_reads(*args, &block)
60
+ DbCharmer.with_controller(self) do
61
+ #{DISPATCH_METHOD}_without_forced_slave_reads(*args, &block)
62
+ end
63
+ end
64
+ EOF
65
+ end
66
+
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,23 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module AssociationPreload
4
+ ASSOCIATION_TYPES = [ :has_one, :has_many, :belongs_to, :has_and_belongs_to_many ]
5
+
6
+ def self.extended(base)
7
+ ASSOCIATION_TYPES.each do |association_type|
8
+ base.class_eval <<-EOF, __FILE__, __LINE__ + 1
9
+ def self.preload_#{association_type}_association(records, reflection, preload_options = {})
10
+ if self.db_charmer_top_level_connection? || reflection.options[:polymorphic] ||
11
+ self.db_charmer_default_connection != reflection.klass.db_charmer_default_connection
12
+ return super(records, reflection, preload_options)
13
+ end
14
+ reflection.klass.on_db(self) do
15
+ super(records, reflection, preload_options)
16
+ end
17
+ end
18
+ EOF
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,101 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module ClassAttributes
4
+ @@db_charmer_opts = {}
5
+ def db_charmer_opts=(opts)
6
+ @@db_charmer_opts[self.name] = opts
7
+ end
8
+
9
+ def db_charmer_opts
10
+ @@db_charmer_opts[self.name] || {}
11
+ end
12
+
13
+ #-----------------------------------------------------------------------------
14
+ @@db_charmer_connection_proxies = {}
15
+ def db_charmer_connection_proxy=(proxy)
16
+ @@db_charmer_connection_proxies[self.name] = proxy
17
+ end
18
+
19
+ def db_charmer_connection_proxy
20
+ @@db_charmer_connection_proxies[self.name]
21
+ end
22
+
23
+ #-----------------------------------------------------------------------------
24
+ @@db_charmer_default_connections = {}
25
+ def db_charmer_default_connection=(conn)
26
+ @@db_charmer_default_connections[self.name] = conn
27
+ end
28
+
29
+ def db_charmer_default_connection
30
+ @@db_charmer_default_connections[self.name]
31
+ end
32
+
33
+ #-----------------------------------------------------------------------------
34
+ @@db_charmer_slaves = {}
35
+ def db_charmer_slaves=(slaves)
36
+ @@db_charmer_slaves[self.name] = slaves
37
+ end
38
+
39
+ def db_charmer_slaves
40
+ @@db_charmer_slaves[self.name] || []
41
+ end
42
+
43
+ def db_charmer_random_slave
44
+ return nil unless db_charmer_slaves.any?
45
+ db_charmer_slaves[rand(db_charmer_slaves.size)]
46
+ end
47
+
48
+ #-----------------------------------------------------------------------------
49
+ @@db_charmer_force_slave_reads = {}
50
+ def db_charmer_force_slave_reads=(force)
51
+ @@db_charmer_force_slave_reads[self.name] = force
52
+ end
53
+
54
+ def db_charmer_force_slave_reads
55
+ @@db_charmer_force_slave_reads[self.name]
56
+ end
57
+
58
+ # Slave reads are used in two cases:
59
+ # - per-model slave reads are enabled (see db_magic method for more details)
60
+ # - global slave reads enforcing is enabled (in a controller action)
61
+ def db_charmer_force_slave_reads?
62
+ db_charmer_force_slave_reads || DbCharmer.force_slave_reads?
63
+ end
64
+
65
+ #-----------------------------------------------------------------------------
66
+ @@db_charmer_connection_levels = Hash.new(0)
67
+ def db_charmer_connection_level=(level)
68
+ @@db_charmer_connection_levels[self.name] = level
69
+ end
70
+
71
+ def db_charmer_connection_level
72
+ @@db_charmer_connection_levels[self.name] || 0
73
+ end
74
+
75
+ def db_charmer_top_level_connection?
76
+ db_charmer_connection_level.zero?
77
+ end
78
+
79
+ #-----------------------------------------------------------------------------
80
+ @@db_charmer_database_remappings = Hash.new
81
+ def db_charmer_remapped_connection
82
+ return nil if (db_charmer_connection_level || 0) > 0
83
+ name = :master
84
+ proxy = db_charmer_connection_proxy
85
+ name = proxy.db_charmer_connection_name.to_sym if proxy
86
+
87
+ remapped = @@db_charmer_database_remappings[name]
88
+ remapped ? DbCharmer::ConnectionFactory.connect(remapped, true) : nil
89
+ end
90
+
91
+ def db_charmer_database_remappings
92
+ @@db_charmer_database_remappings
93
+ end
94
+
95
+ def db_charmer_database_remappings=(mappings)
96
+ raise "Mappings must be nil or respond to []" if mappings && (! mappings.respond_to?(:[]))
97
+ @@db_charmer_database_remappings = mappings || { }
98
+ end
99
+ end
100
+ end
101
+ end
@@ -0,0 +1,81 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module ConnectionSwitching
4
+ def establish_real_connection_if_exists(name, should_exist = false)
5
+ name = name.to_s
6
+
7
+ # Check environment name
8
+ config = configurations[DbCharmer.env]
9
+ unless config
10
+ error = "Invalid environment name (does not exist in database.yml): #{DbCharmer.env}. Please set correct Rails.env or DbCharmer.env."
11
+ raise ArgumentError, error
12
+ end
13
+
14
+ # Check connection name
15
+ config = config[name]
16
+ unless config
17
+ if should_exist
18
+ raise ArgumentError, "Invalid connection name (does not exist in database.yml): #{DbCharmer.env}/#{name}"
19
+ end
20
+ return # No need to establish connection - they do not want us to
21
+ end
22
+
23
+ # Pass connection name with config
24
+ config[:connection_name] = name
25
+ establish_connection(config)
26
+ end
27
+
28
+ #-----------------------------------------------------------------------------------------------------------------
29
+ def hijack_connection!
30
+ return if self.respond_to?(:connection_with_magic)
31
+ class << self
32
+ # Make sure we check our accessors before going to the default connection retrieval method
33
+ def connection_with_magic
34
+ db_charmer_remapped_connection || db_charmer_connection_proxy || connection_without_magic
35
+ end
36
+ alias_method_chain :connection, :magic
37
+ end
38
+ end
39
+
40
+ #-----------------------------------------------------------------------------------------------------------------
41
+ def coerce_to_connection_proxy(conn, should_exist = true)
42
+ return nil if conn.nil?
43
+
44
+ if conn.kind_of?(Symbol) || conn.kind_of?(String)
45
+ return DbCharmer::ConnectionFactory.connect(conn, should_exist)
46
+ end
47
+
48
+ if conn.kind_of?(Hash)
49
+ conn = conn.symbolize_keys
50
+ raise ArgumentError, "Missing required :connection_name parameter" unless conn[:connection_name]
51
+ return DbCharmer::ConnectionFactory.connect_to_db(conn[:connection_name], conn)
52
+ end
53
+
54
+ if conn.respond_to?(:db_charmer_connection_proxy)
55
+ return conn.db_charmer_connection_proxy
56
+ end
57
+
58
+ if conn.kind_of?(::ActiveRecord::ConnectionAdapters::AbstractAdapter) || conn.kind_of?(DbCharmer::Sharding::StubConnection)
59
+ return conn
60
+ end
61
+
62
+ raise "Unsupported connection type: #{conn.class}"
63
+ end
64
+
65
+ #-----------------------------------------------------------------------------------------------------------------
66
+ def switch_connection_to(conn, should_exist = true)
67
+ new_conn = coerce_to_connection_proxy(conn, should_exist)
68
+
69
+ if db_charmer_connection_proxy.is_a?(DbCharmer::Sharding::StubConnection)
70
+ db_charmer_connection_proxy.set_real_connection(new_conn)
71
+ end
72
+
73
+ self.db_charmer_connection_proxy = new_conn
74
+ self.hijack_connection!
75
+
76
+ # self.reset_column_information
77
+ end
78
+
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,85 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module DbMagic
4
+
5
+ def db_magic(opt = {})
6
+ # Make sure we could use our connections management here
7
+ hijack_connection!
8
+
9
+ # Should requested connections exist in the config?
10
+ should_exist = opt.has_key?(:should_exist) ? opt[:should_exist] : DbCharmer.connections_should_exist?
11
+
12
+ # Main connection management
13
+ setup_connection_magic(opt[:connection], should_exist)
14
+
15
+ # Set up slaves pool
16
+ opt[:slaves] ||= []
17
+ opt[:slaves] = [ opt[:slaves] ].flatten
18
+ opt[:slaves] << opt[:slave] if opt[:slave]
19
+
20
+ # Forced reads are enabled for all models by default, could be disabled by the user
21
+ forced_slave_reads = opt.has_key?(:force_slave_reads) ? opt[:force_slave_reads] : true
22
+
23
+ # Setup all the slaves related magic if needed
24
+ setup_slaves_magic(opt[:slaves], forced_slave_reads, should_exist)
25
+
26
+ # Setup inheritance magic
27
+ setup_children_magic(opt)
28
+
29
+ # Setup sharding if needed
30
+ if opt[:sharded]
31
+ raise ArgumentError, "Can't use sharding on a model with slaves!" if opt[:slaves].any?
32
+ setup_sharding_magic(opt[:sharded])
33
+ end
34
+ end
35
+
36
+ private
37
+
38
+ def setup_children_magic(opt)
39
+ self.db_charmer_opts = opt.clone
40
+
41
+ def self.inherited(child)
42
+ child.db_magic(self.db_charmer_opts)
43
+ super
44
+ end
45
+ end
46
+
47
+ def setup_sharding_magic(config)
48
+ # Add sharding-specific methods
49
+ self.extend(DbCharmer::ActiveRecord::Sharding)
50
+
51
+ # Get configuration
52
+ name = config[:sharded_connection] or raise ArgumentError, "No :sharded_connection!"
53
+ # Assign sharded connection
54
+ self.sharded_connection = DbCharmer::Sharding.sharded_connection(name)
55
+
56
+ # Setup model default connection
57
+ setup_connection_magic(sharded_connection.default_connection)
58
+ end
59
+
60
+ def setup_connection_magic(conn, should_exist = true)
61
+ switch_connection_to(conn, should_exist)
62
+ self.db_charmer_default_connection = conn
63
+ end
64
+
65
+ def setup_slaves_magic(slaves, force_slave_reads, should_exist = true)
66
+ self.db_charmer_force_slave_reads = force_slave_reads
67
+
68
+ # Initialize the slave connections list
69
+ self.db_charmer_slaves = slaves.collect do |slave|
70
+ coerce_to_connection_proxy(slave, should_exist)
71
+ end
72
+ return if db_charmer_slaves.empty?
73
+
74
+ # Enable on_slave/on_master methods
75
+ self.extend(DbCharmer::ActiveRecord::MultiDbProxy::MasterSlaveClassMethods)
76
+
77
+ # Enable automatic master/slave queries routing (we have specialized versions on those modules for rails2/3)
78
+ self.extend(DbCharmer::ActiveRecord::MasterSlaveRouting::ClassMethods)
79
+ self.send(:include, DbCharmer::ActiveRecord::MasterSlaveRouting::InstanceMethods)
80
+ end
81
+
82
+ end
83
+ end
84
+ end
85
+
@@ -0,0 +1,71 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module Migration
4
+ module MultiDbMigrations
5
+
6
+ def self.extended(base)
7
+ class << base
8
+ alias_method_chain :migrate, :db_wrapper
9
+ end
10
+ end
11
+
12
+ @@multi_db_names = {}
13
+ def multi_db_names
14
+ @@multi_db_names[self.name] || @@multi_db_names['ActiveRecord::Migration']
15
+ end
16
+
17
+ def multi_db_names=(names)
18
+ @@multi_db_names[self.name] = names
19
+ end
20
+
21
+ def migrate_with_db_wrapper(direction)
22
+ if names = multi_db_names
23
+ names.each do |multi_db_name|
24
+ on_db(multi_db_name) do
25
+ migrate_without_db_wrapper(direction)
26
+ end
27
+ end
28
+ else
29
+ migrate_without_db_wrapper(direction)
30
+ end
31
+ end
32
+
33
+ def on_db(db_name)
34
+ name = db_name.is_a?(Hash) ? db_name[:connection_name] : db_name.inspect
35
+ announce "Switching connection to #{name}"
36
+ # Switch connection
37
+ old_proxy = ::ActiveRecord::Base.db_charmer_connection_proxy
38
+ db_name = nil if db_name == :default
39
+ ::ActiveRecord::Base.switch_connection_to(db_name, DbCharmer.connections_should_exist?)
40
+ # Yield the block
41
+ yield
42
+ ensure
43
+ # Switch it back
44
+ ::ActiveRecord::Base.verify_active_connections!
45
+ announce "Switching connection back"
46
+ ::ActiveRecord::Base.switch_connection_to(old_proxy)
47
+ end
48
+
49
+ def db_magic(opts = {})
50
+ # Collect connections from all possible options
51
+ conns = [ opts[:connection], opts[:connections] ]
52
+ conns << shard_connections(opts[:sharded_connection]) if opts[:sharded_connection]
53
+
54
+ # Get a unique set of connections
55
+ conns = conns.flatten.compact.uniq
56
+ raise ArgumentError, "No connection name - no magic!" unless conns.any?
57
+
58
+ # Save connections
59
+ self.multi_db_names = conns
60
+ end
61
+
62
+ # Return a list of connections to shards in a sharded connection
63
+ def shard_connections(conn_name)
64
+ conn = DbCharmer::Sharding.sharded_connection(conn_name)
65
+ conn.shard_connections
66
+ end
67
+
68
+ end
69
+ end
70
+ end
71
+ end