yam-db-charmer 1.7.01

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/.gitignore +4 -0
  2. data/CHANGES +184 -0
  3. data/LICENSE +21 -0
  4. data/Makefile +2 -0
  5. data/README.rdoc +612 -0
  6. data/Rakefile +4 -0
  7. data/db-charmer.gemspec +29 -0
  8. data/init.rb +1 -0
  9. data/lib/db_charmer/action_controller/force_slave_reads.rb +69 -0
  10. data/lib/db_charmer/active_record/association_preload.rb +23 -0
  11. data/lib/db_charmer/active_record/class_attributes.rb +101 -0
  12. data/lib/db_charmer/active_record/connection_switching.rb +81 -0
  13. data/lib/db_charmer/active_record/db_magic.rb +85 -0
  14. data/lib/db_charmer/active_record/migration/multi_db_migrations.rb +71 -0
  15. data/lib/db_charmer/active_record/multi_db_proxy.rb +77 -0
  16. data/lib/db_charmer/active_record/sharding.rb +40 -0
  17. data/lib/db_charmer/connection_factory.rb +76 -0
  18. data/lib/db_charmer/connection_proxy.rb +27 -0
  19. data/lib/db_charmer/core_extensions.rb +23 -0
  20. data/lib/db_charmer/force_slave_reads.rb +36 -0
  21. data/lib/db_charmer/rails2/abstract_adapter/log_formatting.rb +24 -0
  22. data/lib/db_charmer/rails2/active_record/master_slave_routing.rb +49 -0
  23. data/lib/db_charmer/rails2/active_record/named_scope/scope_proxy.rb +26 -0
  24. data/lib/db_charmer/rails3/abstract_adapter/connection_name.rb +38 -0
  25. data/lib/db_charmer/rails3/active_record/log_subscriber.rb +23 -0
  26. data/lib/db_charmer/rails3/active_record/master_slave_routing.rb +46 -0
  27. data/lib/db_charmer/rails3/active_record/relation/connection_routing.rb +147 -0
  28. data/lib/db_charmer/rails3/active_record/relation_method.rb +28 -0
  29. data/lib/db_charmer/sharding/connection.rb +31 -0
  30. data/lib/db_charmer/sharding/method/db_block_group_map.rb +257 -0
  31. data/lib/db_charmer/sharding/method/db_block_map.rb +211 -0
  32. data/lib/db_charmer/sharding/method/hash_map.rb +23 -0
  33. data/lib/db_charmer/sharding/method/range.rb +33 -0
  34. data/lib/db_charmer/sharding/method.rb +10 -0
  35. data/lib/db_charmer/sharding/stub_connection.rb +60 -0
  36. data/lib/db_charmer/sharding.rb +18 -0
  37. data/lib/db_charmer/version.rb +10 -0
  38. data/lib/db_charmer.rb +192 -0
  39. data/lib/tasks/databases.rake +82 -0
  40. metadata +178 -0
@@ -0,0 +1,211 @@
1
+ # This is a more sophisticated sharding method based on a database-backed
2
+ # blocks map that holds block-shard associations. It automatically
3
+ # creates new blocks for new keys and assigns them to shards.
4
+ #
5
+ module DbCharmer
6
+ module Sharding
7
+ module Method
8
+ class DbBlockMap
9
+ # Sharder name
10
+ attr_accessor :name
11
+
12
+ # Mapping db connection
13
+ attr_accessor :connection, :connection_name
14
+
15
+ # Mapping table name
16
+ attr_accessor :map_table
17
+
18
+ # Shards table name
19
+ attr_accessor :shards_table
20
+
21
+ # Sharding keys block size
22
+ attr_accessor :block_size
23
+
24
+ def initialize(config)
25
+ @name = config[:name] or raise(ArgumentError, "Missing required :name parameter!")
26
+ @connection = DbCharmer::ConnectionFactory.connect(config[:connection], true)
27
+ @block_size = (config[:block_size] || 10000).to_i
28
+
29
+ @map_table = config[:map_table] or raise(ArgumentError, "Missing required :map_table parameter!")
30
+ @shards_table = config[:shards_table] or raise(ArgumentError, "Missing required :shards_table parameter!")
31
+
32
+ # Local caches
33
+ @shard_info_cache = {}
34
+
35
+ @blocks_cache = Rails.cache
36
+ @blocks_cache_prefix = config[:blocks_cache_prefix] || "#{@name}_block:"
37
+ end
38
+
39
+ def shard_for_key(key)
40
+ block = block_for_key(key)
41
+
42
+ begin
43
+ # Auto-allocate new blocks
44
+ block ||= allocate_new_block_for_key(key)
45
+ rescue ::ActiveRecord::StatementInvalid => e
46
+ raise unless e.message.include?('Duplicate entry')
47
+ block = block_for_key(key)
48
+ end
49
+
50
+ raise ArgumentError, "Invalid key value, no shards found for this key and could not create a new block!" unless block
51
+
52
+ # Bail if no shard found
53
+ shard_id = block['shard_id'].to_i
54
+ shard_info = shard_info_by_id(shard_id)
55
+ raise ArgumentError, "Invalid shard_id: #{shard_id}" unless shard_info
56
+
57
+ # Get config
58
+ shard_connection_config(shard_info)
59
+ end
60
+
61
+ class ShardInfo < ::ActiveRecord::Base
62
+ validates_presence_of :db_host
63
+ validates_presence_of :db_port
64
+ validates_presence_of :db_user
65
+ validates_presence_of :db_pass
66
+ validates_presence_of :db_name
67
+ end
68
+
69
+ # Returns a block for a key
70
+ def block_for_key(key, cache = true)
71
+ # Cleanup the cache if asked to
72
+ key_range = [ block_start_for_key(key), block_end_for_key(key) ]
73
+ block_cache_key = "%d-%d" % key_range
74
+
75
+ if cache
76
+ cached_block = get_cached_block(block_cache_key)
77
+ return cached_block if cached_block
78
+ end
79
+
80
+ # Fetch cached value or load from db
81
+ block = begin
82
+ sql = "SELECT * FROM #{map_table} WHERE start_id = #{key_range.first} AND end_id = #{key_range.last} LIMIT 1"
83
+ connection.select_one(sql, 'Find a shard block')
84
+ end
85
+
86
+ set_cached_block(block_cache_key, block)
87
+
88
+ return block
89
+ end
90
+
91
+ def get_cached_block(block_cache_key)
92
+ @blocks_cache.read("#{@blocks_cache_prefix}#{block_cache_key}")
93
+ end
94
+
95
+ def set_cached_block(block_cache_key, block)
96
+ @blocks_cache.write("#{@blocks_cache_prefix}#{block_cache_key}", block)
97
+ end
98
+
99
+ # Load shard info
100
+ def shard_info_by_id(shard_id, cache = true)
101
+ # Cleanup the cache if asked to
102
+ @shard_info_cache[shard_id] = nil unless cache
103
+
104
+ # Either load from cache or from db
105
+ @shard_info_cache[shard_id] ||= begin
106
+ prepare_shard_model
107
+ ShardInfo.find_by_id(shard_id)
108
+ end
109
+ end
110
+
111
+ def allocate_new_block_for_key(key)
112
+ # Can't find any shards to use for blocks allocation!
113
+ return nil unless shard = least_loaded_shard
114
+
115
+ # Figure out block limits
116
+ start_id = block_start_for_key(key)
117
+ end_id = block_end_for_key(key)
118
+
119
+ # Try to insert a new mapping (ignore duplicate key errors)
120
+ sql = <<-SQL
121
+ INSERT INTO #{map_table}
122
+ SET start_id = #{start_id},
123
+ end_id = #{end_id},
124
+ shard_id = #{shard.id},
125
+ block_size = #{block_size},
126
+ created_at = NOW(),
127
+ updated_at = NOW()
128
+ SQL
129
+ connection.execute(sql, "Allocate new block")
130
+
131
+ # Increment the blocks counter on the shard
132
+ ShardInfo.update_counters(shard.id, :blocks_count => +1)
133
+
134
+ # Retry block search after creation
135
+ block_for_key(key)
136
+ end
137
+
138
+ def least_loaded_shard
139
+ prepare_shard_model
140
+
141
+ # Select shard
142
+ shard = ShardInfo.all(:conditions => { :enabled => true, :open => true }, :order => 'blocks_count ASC', :limit => 1).first
143
+ raise "Can't find any shards to use for blocks allocation!" unless shard
144
+ return shard
145
+ end
146
+
147
+ def block_start_for_key(key)
148
+ block_size.to_i * (key.to_i / block_size.to_i)
149
+ end
150
+
151
+ def block_end_for_key(key)
152
+ block_size.to_i + block_start_for_key(key)
153
+ end
154
+
155
+ # Create configuration (use mapping connection as a template)
156
+ def shard_connection_config(shard)
157
+ # Format connection name
158
+ shard_name = "db_charmer_db_block_map_#{name}_shard_%05d" % shard.id
159
+
160
+ # Here we get the mapping connection's configuration
161
+ # They do not expose configs so we hack in and get the instance var
162
+ # FIXME: Find a better way, maybe move config method to our ar extenstions
163
+ connection.instance_variable_get(:@config).clone.merge(
164
+ # Name for the connection factory
165
+ :connection_name => shard_name,
166
+ # Connection params
167
+ :host => shard.db_host,
168
+ :port => shard.db_port,
169
+ :username => shard.db_user,
170
+ :password => shard.db_pass,
171
+ :database => shard.db_name
172
+ )
173
+ end
174
+
175
+ def create_shard(params)
176
+ params = params.symbolize_keys
177
+ [ :db_host, :db_port, :db_user, :db_pass, :db_name ].each do |arg|
178
+ raise ArgumentError, "Missing required parameter: #{arg}" unless params[arg]
179
+ end
180
+
181
+ # Prepare model
182
+ prepare_shard_model
183
+
184
+ # Create the record
185
+ ShardInfo.create! do |shard|
186
+ shard.db_host = params[:db_host]
187
+ shard.db_port = params[:db_port]
188
+ shard.db_user = params[:db_user]
189
+ shard.db_pass = params[:db_pass]
190
+ shard.db_name = params[:db_name]
191
+ end
192
+ end
193
+
194
+ def shard_connections
195
+ # Find all shards
196
+ prepare_shard_model
197
+ shards = ShardInfo.all(:conditions => { :enabled => true })
198
+ # Map them to connections
199
+ shards.map { |shard| shard_connection_config(shard) }
200
+ end
201
+
202
+ # Prepare model for working with our shards table
203
+ def prepare_shard_model
204
+ ShardInfo.set_table_name(shards_table)
205
+ ShardInfo.switch_connection_to(connection)
206
+ end
207
+
208
+ end
209
+ end
210
+ end
211
+ end
@@ -0,0 +1,23 @@
1
+ module DbCharmer
2
+ module Sharding
3
+ module Method
4
+ class HashMap
5
+ attr_accessor :map
6
+
7
+ def initialize(config)
8
+ @map = config[:map].clone or raise ArgumentError, "No :map defined!"
9
+ end
10
+
11
+ def shard_for_key(key)
12
+ res = map[key] || map[:default]
13
+ raise ArgumentError, "Invalid key value, no shards found for this key!" unless res
14
+ return res
15
+ end
16
+
17
+ def support_default_shard?
18
+ map.has_key?(:default)
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,33 @@
1
+ module DbCharmer
2
+ module Sharding
3
+ module Method
4
+ class Range
5
+ attr_accessor :ranges
6
+
7
+ def initialize(config)
8
+ @ranges = config[:ranges] ? config[:ranges].clone : raise(ArgumentError, "No :ranges defined!")
9
+ end
10
+
11
+ def shard_for_key(key)
12
+ return ranges[:default] if key == :default
13
+
14
+ ranges.each do |range, shard|
15
+ next if range == :default
16
+ return shard if range.member?(key.to_i)
17
+ end
18
+
19
+ return ranges[:default] if ranges[:default]
20
+ raise ArgumentError, "Invalid key value, no shards found for this key!"
21
+ end
22
+
23
+ def support_default_shard?
24
+ ranges.has_key?(:default)
25
+ end
26
+
27
+ def shard_connections
28
+ ranges.values.uniq
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,10 @@
1
+ module DbCharmer
2
+ module Sharding
3
+ module Method
4
+ autoload :Range, 'db_charmer/sharding/method/range'
5
+ autoload :HashMap, 'db_charmer/sharding/method/hash_map'
6
+ autoload :DbBlockMap, 'db_charmer/sharding/method/db_block_map'
7
+ autoload :DbBlockGroupMap, 'db_charmer/sharding/method/db_block_group_map'
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,60 @@
1
+ # This is a simple proxy class used as a default connection on sharded models
2
+ #
3
+ # The idea is to proxy all utility method calls to a real connection (set by
4
+ # the +set_real_connection+ method when we switch shards) and fail on real
5
+ # database querying calls forcing users to switch shard connections.
6
+ #
7
+ module DbCharmer
8
+ module Sharding
9
+ class StubConnection
10
+ attr_accessor :sharded_connection
11
+
12
+ def initialize(sharded_connection)
13
+ @sharded_connection = sharded_connection
14
+ @real_conn = nil
15
+ end
16
+
17
+ def set_real_connection(real_conn)
18
+ @real_conn = real_conn
19
+ end
20
+
21
+ def db_charmer_connection_name
22
+ "StubConnection"
23
+ end
24
+
25
+ def real_connection
26
+ # Return memoized real connection
27
+ return @real_conn if @real_conn
28
+
29
+ # If sharded connection supports shards enumeration, get the first shard
30
+ conn = sharded_connection.shard_connections.try(:first)
31
+
32
+ # If we do not have real connection yet, try to use the default one (if it is supported by the sharder)
33
+ conn ||= sharded_connection.sharder.shard_for_key(:default) if sharded_connection.support_default_shard?
34
+
35
+ # Get connection proxy for our real connection
36
+ return nil unless conn
37
+ @real_conn = ::ActiveRecord::Base.coerce_to_connection_proxy(conn, DbCharmer.connections_should_exist?)
38
+ end
39
+
40
+ def method_missing(meth, *args, &block)
41
+ # Fail on database statements
42
+ if ::ActiveRecord::ConnectionAdapters::DatabaseStatements.instance_methods.member?(meth.to_s)
43
+ raise ::ActiveRecord::ConnectionNotEstablished, "You have to switch connection on your model before using it!"
44
+ end
45
+
46
+ # Fail if no connection has been established yet
47
+ unless real_connection
48
+ raise ::ActiveRecord::ConnectionNotEstablished, "No real connection to proxy this method to!"
49
+ end
50
+
51
+ if real_connection.kind_of?(DbCharmer::Sharding::StubConnection)
52
+ raise ::ActiveRecord::ConnectionNotEstablished, "You have to switch connection on your model before using it!"
53
+ end
54
+
55
+ # Proxy the call to our real connection target
56
+ real_connection.__send__(meth, *args, &block)
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,18 @@
1
+ module DbCharmer
2
+ module Sharding
3
+ autoload :Connection, 'db_charmer/sharding/connection'
4
+ autoload :StubConnection, 'db_charmer/sharding/stub_connection'
5
+ autoload :Method, 'db_charmer/sharding/method'
6
+
7
+ @@sharded_connections = {}
8
+
9
+ def self.register_connection(config)
10
+ name = config[:name] or raise ArgumentError, "No :name in connection!"
11
+ @@sharded_connections[name] = DbCharmer::Sharding::Connection.new(config)
12
+ end
13
+
14
+ def self.sharded_connection(name)
15
+ @@sharded_connections[name] or raise ArgumentError, "Invalid sharded connection name!"
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,10 @@
1
+ module DbCharmer
2
+ module Version
3
+ MAJOR = 1
4
+ MINOR = 7
5
+ PATCH = "01"
6
+ BUILD = nil
7
+
8
+ STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
9
+ end
10
+ end
data/lib/db_charmer.rb ADDED
@@ -0,0 +1,192 @@
1
+ # In Rails 2.2 they did not add it to the autoload so it won't work w/o this require
2
+ require 'active_record/version' unless defined?(::ActiveRecord::VERSION::MAJOR)
3
+
4
+ module DbCharmer
5
+ # Configure autoload
6
+ autoload :Sharding, 'db_charmer/sharding'
7
+ autoload :Version, 'db_charmer/version'
8
+ module ActionController
9
+ autoload :ForceSlaveReads, 'db_charmer/action_controller/force_slave_reads'
10
+ end
11
+
12
+ # Used in all Rails3-specific places
13
+ def self.rails3?
14
+ ::ActiveRecord::VERSION::MAJOR > 2
15
+ end
16
+
17
+ # Used in all Rails2-specific places
18
+ def self.rails2?
19
+ ::ActiveRecord::VERSION::MAJOR == 2
20
+ end
21
+
22
+ # Accessors
23
+ @@connections_should_exist = true
24
+ mattr_accessor :connections_should_exist
25
+
26
+ # Try to detect current environment or use development by default
27
+ if defined?(Rails)
28
+ @@env = Rails.env
29
+ elsif ENV['RAILS_ENV']
30
+ @@env = ENV['RAILS_ENV']
31
+ elsif ENV['RACK_ENV']
32
+ @@env = ENV['RACK_ENV']
33
+ else
34
+ @@env = 'development'
35
+ end
36
+ mattr_accessor :env
37
+
38
+ def self.connections_should_exist?
39
+ !! connections_should_exist
40
+ end
41
+
42
+ # Extend ActionController to support forcing slave reads
43
+ def self.enable_controller_magic!
44
+ ::ActionController::Base.extend(DbCharmer::ActionController::ForceSlaveReads::ClassMethods)
45
+ ::ActionController::Base.send(:include, DbCharmer::ActionController::ForceSlaveReads::InstanceMethods)
46
+ end
47
+
48
+ def self.logger
49
+ return Rails.logger if defined?(Rails)
50
+ @logger ||= Logger.new(STDERR)
51
+ end
52
+
53
+ def self.with_remapped_databases(mappings, &proc)
54
+ old_mappings = ::ActiveRecord::Base.db_charmer_database_remappings
55
+ begin
56
+ ::ActiveRecord::Base.db_charmer_database_remappings = mappings
57
+ if mappings[:master] || mappings['master']
58
+ with_all_hijacked(&proc)
59
+ else
60
+ proc.call
61
+ end
62
+ ensure
63
+ ::ActiveRecord::Base.db_charmer_database_remappings = old_mappings
64
+ end
65
+ end
66
+
67
+ def self.hijack_new_classes?
68
+ @@hijack_new_classes
69
+ end
70
+
71
+ private
72
+
73
+ @@hijack_new_classes = false
74
+ def self.with_all_hijacked
75
+ old_hijack_new_classes = @@hijack_new_classes
76
+ begin
77
+ @@hijack_new_classes = true
78
+ subclasses_method = DbCharmer.rails3? ? :descendants : :subclasses
79
+ ::ActiveRecord::Base.send(subclasses_method).each do |subclass|
80
+ subclass.hijack_connection!
81
+ end
82
+ yield
83
+ ensure
84
+ @@hijack_new_classes = old_hijack_new_classes
85
+ end
86
+ end
87
+ end
88
+
89
+ # Add useful methods to global object
90
+ require 'db_charmer/core_extensions'
91
+
92
+ require 'db_charmer/connection_factory'
93
+ require 'db_charmer/connection_proxy'
94
+ require 'db_charmer/force_slave_reads'
95
+
96
+ # Add our custom class-level attributes to AR models
97
+ require 'db_charmer/active_record/class_attributes'
98
+ ActiveRecord::Base.extend(DbCharmer::ActiveRecord::ClassAttributes)
99
+
100
+ # Enable connections switching in AR
101
+ require 'db_charmer/active_record/connection_switching'
102
+ ActiveRecord::Base.extend(DbCharmer::ActiveRecord::ConnectionSwitching)
103
+
104
+ # Enable AR logging extensions
105
+ if DbCharmer.rails3?
106
+ require 'db_charmer/rails3/abstract_adapter/connection_name'
107
+ require 'db_charmer/rails3/active_record/log_subscriber'
108
+ ActiveRecord::LogSubscriber.send(:include, DbCharmer::ActiveRecord::LogSubscriber)
109
+ ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:include, DbCharmer::AbstractAdapter::ConnectionName)
110
+ else
111
+ require 'db_charmer/rails2/abstract_adapter/log_formatting'
112
+ ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:include, DbCharmer::AbstractAdapter::LogFormatting)
113
+ end
114
+
115
+ # Enable connection proxy in AR
116
+ require 'db_charmer/active_record/multi_db_proxy'
117
+ ActiveRecord::Base.extend(DbCharmer::ActiveRecord::MultiDbProxy::ClassMethods)
118
+ ActiveRecord::Base.extend(DbCharmer::ActiveRecord::MultiDbProxy::MasterSlaveClassMethods)
119
+ ActiveRecord::Base.send(:include, DbCharmer::ActiveRecord::MultiDbProxy::InstanceMethods)
120
+
121
+ # Enable connection proxy for relations
122
+ if DbCharmer.rails3?
123
+ require 'db_charmer/rails3/active_record/relation_method'
124
+ require 'db_charmer/rails3/active_record/relation/connection_routing'
125
+ ActiveRecord::Base.extend(DbCharmer::ActiveRecord::RelationMethod)
126
+ ActiveRecord::Relation.send(:include, DbCharmer::ActiveRecord::Relation::ConnectionRouting)
127
+ end
128
+
129
+ # Enable connection proxy for scopes (rails 2.x only)
130
+ if DbCharmer.rails2?
131
+ require 'db_charmer/rails2/active_record/named_scope/scope_proxy'
132
+ ActiveRecord::NamedScope::Scope.send(:include, DbCharmer::ActiveRecord::NamedScope::ScopeProxy)
133
+ end
134
+
135
+ # Enable connection proxy for associations
136
+ # WARNING: Inject methods to association class right here (they proxy include calls somewhere else, so include does not work)
137
+ module ActiveRecord
138
+ module Associations
139
+ class AssociationProxy
140
+ def proxy?
141
+ true
142
+ end
143
+
144
+ def on_db(con, proxy_target = nil, &block)
145
+ proxy_target ||= self
146
+ @reflection.klass.on_db(con, proxy_target, &block)
147
+ end
148
+
149
+ def on_slave(con = nil, &block)
150
+ @reflection.klass.on_slave(con, self, &block)
151
+ end
152
+
153
+ def on_master(&block)
154
+ @reflection.klass.on_master(self, &block)
155
+ end
156
+ end
157
+ end
158
+ end
159
+
160
+ # Enable multi-db migrations
161
+ require 'db_charmer/active_record/migration/multi_db_migrations'
162
+ ActiveRecord::Migration.extend(DbCharmer::ActiveRecord::Migration::MultiDbMigrations)
163
+
164
+ # Enable the magic
165
+ if DbCharmer.rails3?
166
+ require 'db_charmer/rails3/active_record/master_slave_routing'
167
+ else
168
+ require 'db_charmer/rails2/active_record/master_slave_routing'
169
+ end
170
+
171
+ require 'db_charmer/active_record/sharding'
172
+ require 'db_charmer/active_record/db_magic'
173
+ ActiveRecord::Base.extend(DbCharmer::ActiveRecord::DbMagic)
174
+
175
+ # Setup association preload magic
176
+ require 'db_charmer/active_record/association_preload'
177
+ ActiveRecord::Base.extend(DbCharmer::ActiveRecord::AssociationPreload)
178
+
179
+ # Open up really useful API method
180
+ ActiveRecord::AssociationPreload::ClassMethods.send(:public, :preload_associations)
181
+
182
+ class ::ActiveRecord::Base
183
+ class << self
184
+ def inherited_with_hijacking(subclass)
185
+ out = inherited_without_hijacking(subclass)
186
+ hijack_connection! if DbCharmer.hijack_new_classes?
187
+ out
188
+ end
189
+
190
+ alias_method_chain :inherited, :hijacking
191
+ end
192
+ end