yam-db-charmer 1.7.01

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/.gitignore +4 -0
  2. data/CHANGES +184 -0
  3. data/LICENSE +21 -0
  4. data/Makefile +2 -0
  5. data/README.rdoc +612 -0
  6. data/Rakefile +4 -0
  7. data/db-charmer.gemspec +29 -0
  8. data/init.rb +1 -0
  9. data/lib/db_charmer/action_controller/force_slave_reads.rb +69 -0
  10. data/lib/db_charmer/active_record/association_preload.rb +23 -0
  11. data/lib/db_charmer/active_record/class_attributes.rb +101 -0
  12. data/lib/db_charmer/active_record/connection_switching.rb +81 -0
  13. data/lib/db_charmer/active_record/db_magic.rb +85 -0
  14. data/lib/db_charmer/active_record/migration/multi_db_migrations.rb +71 -0
  15. data/lib/db_charmer/active_record/multi_db_proxy.rb +77 -0
  16. data/lib/db_charmer/active_record/sharding.rb +40 -0
  17. data/lib/db_charmer/connection_factory.rb +76 -0
  18. data/lib/db_charmer/connection_proxy.rb +27 -0
  19. data/lib/db_charmer/core_extensions.rb +23 -0
  20. data/lib/db_charmer/force_slave_reads.rb +36 -0
  21. data/lib/db_charmer/rails2/abstract_adapter/log_formatting.rb +24 -0
  22. data/lib/db_charmer/rails2/active_record/master_slave_routing.rb +49 -0
  23. data/lib/db_charmer/rails2/active_record/named_scope/scope_proxy.rb +26 -0
  24. data/lib/db_charmer/rails3/abstract_adapter/connection_name.rb +38 -0
  25. data/lib/db_charmer/rails3/active_record/log_subscriber.rb +23 -0
  26. data/lib/db_charmer/rails3/active_record/master_slave_routing.rb +46 -0
  27. data/lib/db_charmer/rails3/active_record/relation/connection_routing.rb +147 -0
  28. data/lib/db_charmer/rails3/active_record/relation_method.rb +28 -0
  29. data/lib/db_charmer/sharding/connection.rb +31 -0
  30. data/lib/db_charmer/sharding/method/db_block_group_map.rb +257 -0
  31. data/lib/db_charmer/sharding/method/db_block_map.rb +211 -0
  32. data/lib/db_charmer/sharding/method/hash_map.rb +23 -0
  33. data/lib/db_charmer/sharding/method/range.rb +33 -0
  34. data/lib/db_charmer/sharding/method.rb +10 -0
  35. data/lib/db_charmer/sharding/stub_connection.rb +60 -0
  36. data/lib/db_charmer/sharding.rb +18 -0
  37. data/lib/db_charmer/version.rb +10 -0
  38. data/lib/db_charmer.rb +192 -0
  39. data/lib/tasks/databases.rake +82 -0
  40. metadata +178 -0
@@ -0,0 +1,46 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module MasterSlaveRouting
4
+
5
+ module ClassMethods
6
+ SLAVE_METHODS = [ :find_by_sql, :count_by_sql ]
7
+ MASTER_METHODS = [ ] # I don't know any methods in AR::Base that change data directly w/o going to the relation object
8
+
9
+ SLAVE_METHODS.each do |slave_method|
10
+ class_eval <<-EOF, __FILE__, __LINE__ + 1
11
+ def #{slave_method}(*args, &block)
12
+ first_level_on_slave do
13
+ super(*args, &block)
14
+ end
15
+ end
16
+ EOF
17
+ end
18
+
19
+ MASTER_METHODS.each do |master_method|
20
+ class_eval <<-EOF, __FILE__, __LINE__ + 1
21
+ def #{master_method}(*args, &block)
22
+ on_master do
23
+ super(*args, &block)
24
+ end
25
+ end
26
+ EOF
27
+ end
28
+ end
29
+
30
+ module InstanceMethods
31
+ MASTER_METHODS = [ :reload ]
32
+
33
+ MASTER_METHODS.each do |master_method|
34
+ class_eval <<-EOF, __FILE__, __LINE__ + 1
35
+ def #{master_method}(*args, &block)
36
+ self.class.on_master do
37
+ super(*args, &block)
38
+ end
39
+ end
40
+ EOF
41
+ end
42
+ end
43
+
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,147 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module Relation
4
+ module ConnectionRouting
5
+
6
+ # All the methods that could be querying the database
7
+ SLAVE_METHODS = [ :calculate, :exists? ]
8
+ MASTER_METHODS = [ :delete, :delete_all, :destroy, :destroy_all, :reload, :update, :update_all ]
9
+ ALL_METHODS = SLAVE_METHODS + MASTER_METHODS
10
+
11
+ DB_CHARMER_ATTRIBUTES = [ :db_charmer_connection, :db_charmer_connection_is_forced, :db_charmer_enable_slaves ]
12
+
13
+ # Define the default relation connection + override all the query methods here
14
+ def self.included(base)
15
+ init_attributes(base)
16
+ init_routing(base)
17
+ end
18
+
19
+ # Define our attributes + spawn methods shit needs to be changed to make sure our accessors are copied over to the new instances
20
+ def self.init_attributes(base)
21
+ DB_CHARMER_ATTRIBUTES.each do |attr|
22
+ base.send(:attr_accessor, attr)
23
+ end
24
+
25
+ # Override spawn methods
26
+ base.alias_method_chain :except, :db_charmer
27
+ base.alias_method_chain :only, :db_charmer
28
+ end
29
+
30
+ # Override all query methods
31
+ def self.init_routing(base)
32
+ ALL_METHODS.each do |meth|
33
+ base.alias_method_chain meth, :db_charmer
34
+ end
35
+
36
+ # Special case: for normal selects we go to the slave, but for selects with a lock we should use master
37
+ base.alias_method_chain :to_a, :db_charmer
38
+ end
39
+
40
+ # Copy db_charmer attributes in addition to what they're copying
41
+ def except_with_db_charmer(*args)
42
+ except_without_db_charmer(*args).tap do |result|
43
+ copy_db_charmer_options(self, result)
44
+ end
45
+ end
46
+
47
+ # Copy db_charmer attributes in addition to what they're copying
48
+ def only_with_db_charmer(*args)
49
+ only_without_db_charmer(*args).tap do |result|
50
+ copy_db_charmer_options(self, result)
51
+ end
52
+ end
53
+
54
+ # Copy our accessors from one instance to another
55
+ def copy_db_charmer_options(src, dst)
56
+ DB_CHARMER_ATTRIBUTES.each do |attr|
57
+ dst.send("#{attr}=".to_sym, src.send(attr))
58
+ end
59
+ end
60
+
61
+ # Connection switching (changes the default relation connection)
62
+ def on_db(con, &block)
63
+ if block_given?
64
+ @klass.on_db(con, &block)
65
+ else
66
+ clone.tap do |result|
67
+ result.db_charmer_connection = con
68
+ result.db_charmer_connection_is_forced = true
69
+ end
70
+ end
71
+ end
72
+
73
+ # Make sure we get the right connection here
74
+ def connection
75
+ @klass.on_db(db_charmer_connection).connection
76
+ end
77
+
78
+ # Selects preferred destination (master/slave/default) for a query
79
+ def select_destination(method, recommendation = :default)
80
+ # If this relation was created within a forced connection block (e.g Model.on_db(:foo).relation)
81
+ # Then we should use that connection everywhere except cases when a model is slave-enabled
82
+ # in those cases DML queries go to the master
83
+ if db_charmer_connection_is_forced
84
+ return :master if db_charmer_enable_slaves && MASTER_METHODS.member?(method)
85
+ return :default
86
+ end
87
+
88
+ # If this relation is created from a slave-enabled model, let's do the routing if possible
89
+ if db_charmer_enable_slaves
90
+ return :slave if SLAVE_METHODS.member?(method)
91
+ return :master if MASTER_METHODS.member?(method)
92
+ else
93
+ # Make sure we do not use recommended destination
94
+ recommendation = :default
95
+ end
96
+
97
+ # If nothing else came up, let's use the default or recommended connection
98
+ return recommendation
99
+ end
100
+
101
+ # Switch the model to default relation connection
102
+ def switch_connection_for_method(method, recommendation = nil)
103
+ # Choose where to send the query
104
+ destination ||= select_destination(method, recommendation)
105
+
106
+ # What method to use
107
+ on_db_method = [ :on_db, db_charmer_connection ]
108
+ on_db_method = :on_master if destination == :master
109
+ on_db_method = :first_level_on_slave if destination == :slave
110
+
111
+ # Perform the query
112
+ @klass.send(*on_db_method) do
113
+ yield
114
+ end
115
+ end
116
+
117
+ # For normal selects we go to the slave, but for selects with a lock we should use master
118
+ def to_a_with_db_charmer(*args, &block)
119
+ preferred_destination = :slave
120
+ preferred_destination = :master if lock_value
121
+
122
+ switch_connection_for_method(:to_a, preferred_destination) do
123
+ to_a_without_db_charmer(*args, &block)
124
+ end
125
+ end
126
+
127
+ # Need this to mimick alias_method_chain name generation (exists? => exists_with_db_charmer?)
128
+ def self.aliased_method_name(target, with)
129
+ aliased_target, punctuation = target.to_s.sub(/([?!=])$/, ''), $1
130
+ "#{aliased_target}_#{with}_db_charmer#{punctuation}"
131
+ end
132
+
133
+ # Override all the query methods here
134
+ ALL_METHODS.each do |method|
135
+ class_eval <<-EOF, __FILE__, __LINE__ + 1
136
+ def #{aliased_method_name method, :with}(*args, &block)
137
+ switch_connection_for_method(:#{method.to_s}) do
138
+ #{aliased_method_name method, :without}(*args, &block)
139
+ end
140
+ end
141
+ EOF
142
+ end
143
+
144
+ end
145
+ end
146
+ end
147
+ end
@@ -0,0 +1,28 @@
1
+ module DbCharmer
2
+ module ActiveRecord
3
+ module RelationMethod
4
+
5
+ def self.extended(base)
6
+ class << base
7
+ alias_method_chain :relation, :db_charmer
8
+ alias_method_chain :arel_engine, :db_charmer
9
+ end
10
+ end
11
+
12
+ # Create a relation object and initialize its default connection
13
+ def relation_with_db_charmer(*args, &block)
14
+ relation_without_db_charmer(*args, &block).tap do |rel|
15
+ rel.db_charmer_connection = self.connection
16
+ rel.db_charmer_enable_slaves = self.db_charmer_slaves.any?
17
+ rel.db_charmer_connection_is_forced = !db_charmer_top_level_connection?
18
+ end
19
+ end
20
+
21
+ # Use the model itself an engine for Arel, do not fall back to AR::Base
22
+ def arel_engine_with_db_charmer(*)
23
+ self
24
+ end
25
+
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,31 @@
1
+ module DbCharmer
2
+ module Sharding
3
+ class Connection
4
+ attr_accessor :config, :sharder
5
+
6
+ def initialize(config)
7
+ @config = config
8
+ @sharder = self.instantiate_sharder
9
+ end
10
+
11
+ def instantiate_sharder
12
+ raise ArgumentError, "No :method passed!" unless config[:method]
13
+ sharder_class_name = "DbCharmer::Sharding::Method::#{config[:method].to_s.classify}"
14
+ sharder_class = sharder_class_name.constantize
15
+ sharder_class.new(config)
16
+ end
17
+
18
+ def shard_connections
19
+ sharder.respond_to?(:shard_connections) ? sharder.shard_connections : nil
20
+ end
21
+
22
+ def support_default_shard?
23
+ sharder.respond_to?(:support_default_shard?) && sharder.support_default_shard?
24
+ end
25
+
26
+ def default_connection
27
+ @default_connection ||= DbCharmer::Sharding::StubConnection.new(self)
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,257 @@
1
+ # This is a more sophisticated sharding method based on a two layer database-backed
2
+ # blocks map that holds block-shard associations. Record blocks are mapped to tablegroups
3
+ # and groups are mapped to shards.
4
+ #
5
+ # It automatically creates new blocks for new keys and assigns them to existing groups.
6
+ # Warning: make sure to create at least one shard and one group before inserting any records.
7
+ #
8
+ module DbCharmer
9
+ module Sharding
10
+ module Method
11
+ class DbBlockGroupMap
12
+ # Shard connection info model
13
+ class Shard < ::ActiveRecord::Base
14
+ validates_presence_of :db_host
15
+ validates_presence_of :db_port
16
+ validates_presence_of :db_user
17
+ validates_presence_of :db_pass
18
+ validates_presence_of :db_name_prefix
19
+
20
+ has_many :groups, :class_name => 'DbCharmer::Sharding::Method::DbBlockGroupMap::Group'
21
+ end
22
+
23
+ # Table group info model
24
+ class Group < ::ActiveRecord::Base
25
+ validates_presence_of :shard_id
26
+ belongs_to :shard, :class_name => 'DbCharmer::Sharding::Method::DbBlockGroupMap::Shard'
27
+ end
28
+
29
+ #---------------------------------------------------------------------------------------------------------------
30
+ # Sharder name
31
+ attr_accessor :name
32
+
33
+ # Mapping db connection
34
+ attr_accessor :connection, :connection_name
35
+
36
+ # Mapping table name
37
+ attr_accessor :map_table
38
+
39
+ # Tablegroups table name
40
+ attr_accessor :groups_table
41
+
42
+ # Shards table name
43
+ attr_accessor :shards_table
44
+
45
+ # Sharding keys block size
46
+ attr_accessor :block_size
47
+
48
+ def initialize(config)
49
+ @name = config[:name] or raise(ArgumentError, "Missing required :name parameter!")
50
+ @connection = DbCharmer::ConnectionFactory.connect(config[:connection], true)
51
+ @block_size = (config[:block_size] || 10000).to_i
52
+
53
+ @map_table = config[:map_table] or raise(ArgumentError, "Missing required :map_table parameter!")
54
+ @groups_table = config[:groups_table] or raise(ArgumentError, "Missing required :groups_table parameter!")
55
+ @shards_table = config[:shards_table] or raise(ArgumentError, "Missing required :shards_table parameter!")
56
+
57
+ # Local caches
58
+ @shard_info_cache = {}
59
+ @group_info_cache = {}
60
+
61
+ @blocks_cache = Rails.cache
62
+ @blocks_cache_prefix = config[:blocks_cache_prefix] || "#{@name}_block:"
63
+ end
64
+
65
+ #---------------------------------------------------------------------------------------------------------------
66
+ def shard_for_key(key)
67
+ block = block_for_key(key)
68
+
69
+ # Auto-allocate new blocks
70
+ block ||= allocate_new_block_for_key(key)
71
+ raise ArgumentError, "Invalid key value, no shards found for this key and could not create a new block!" unless block
72
+
73
+ # Load shard
74
+ group_id = block['group_id'].to_i
75
+ shard_info = shard_info_by_group_id(group_id)
76
+
77
+ # Get config
78
+ shard_connection_config(shard_info, group_id)
79
+ end
80
+
81
+ #---------------------------------------------------------------------------------------------------------------
82
+ # Returns a block for a key
83
+ def block_for_key(key, cache = true)
84
+ # Cleanup the cache if asked to
85
+ key_range = [ block_start_for_key(key), block_end_for_key(key) ]
86
+ block_cache_key = "%d-%d" % key_range
87
+
88
+ if cache
89
+ cached_block = get_cached_block(block_cache_key)
90
+ return cached_block if cached_block
91
+ end
92
+
93
+ # Fetch cached value or load from db
94
+ block = begin
95
+ sql = "SELECT * FROM #{map_table} WHERE start_id = #{key_range.first} AND end_id = #{key_range.last} LIMIT 1"
96
+ connection.select_one(sql, 'Find a shard block')
97
+ end
98
+
99
+ set_cached_block(block_cache_key, block)
100
+
101
+ return block
102
+ end
103
+
104
+ #---------------------------------------------------------------------------------------------------------------
105
+ def get_cached_block(block_cache_key)
106
+ @blocks_cache.read("#{@blocks_cache_prefix}#{block_cache_key}")
107
+ end
108
+
109
+ def set_cached_block(block_cache_key, block)
110
+ @blocks_cache.write("#{@blocks_cache_prefix}#{block_cache_key}", block)
111
+ end
112
+
113
+ #---------------------------------------------------------------------------------------------------------------
114
+ # Load group info
115
+ def group_info_by_id(group_id, cache = true)
116
+ # Cleanup the cache if asked to
117
+ @group_info_cache[group_id] = nil unless cache
118
+
119
+ # Either load from cache or from db
120
+ @group_info_cache[group_id] ||= begin
121
+ prepare_shard_models
122
+ Group.find_by_id(group_id)
123
+ end
124
+ end
125
+
126
+ # Load shard info
127
+ def shard_info_by_id(shard_id, cache = true)
128
+ # Cleanup the cache if asked to
129
+ @shard_info_cache[shard_id] = nil unless cache
130
+
131
+ # Either load from cache or from db
132
+ @shard_info_cache[shard_id] ||= begin
133
+ prepare_shard_models
134
+ Shard.find_by_id(shard_id)
135
+ end
136
+ end
137
+
138
+ # Load shard info using mapping info for a group
139
+ def shard_info_by_group_id(group_id)
140
+ # Load group
141
+ group_info = group_info_by_id(group_id)
142
+ raise ArgumentError, "Invalid group_id: #{group_id}" unless group_info
143
+
144
+ shard_info = shard_info_by_id(group_info.shard_id)
145
+ raise ArgumentError, "Invalid shard_id: #{group_info.shard_id}" unless shard_info
146
+
147
+ return shard_info
148
+ end
149
+
150
+ #---------------------------------------------------------------------------------------------------------------
151
+ def allocate_new_block_for_key(key)
152
+ # Can't find any groups to use for blocks allocation!
153
+ return nil unless group = least_loaded_group
154
+
155
+ # Figure out block limits
156
+ start_id = block_start_for_key(key)
157
+ end_id = block_end_for_key(key)
158
+
159
+ # Try to insert a new mapping (ignore duplicate key errors)
160
+ sql = <<-SQL
161
+ INSERT INTO #{map_table}
162
+ (start_id, end_id, group_id, block_size, created_at, updated_at) VALUES
163
+ (#{start_id}, #{end_id}, #{group.id}, #{block_size}, NOW(), NOW())
164
+ SQL
165
+ connection.execute(sql, "Allocate new block")
166
+
167
+ # Increment the blocks counter on the shard
168
+ Group.update_counters(group.id, :blocks_count => +1)
169
+
170
+ # Retry block search after creation
171
+ block_for_key(key)
172
+ end
173
+
174
+ def least_loaded_group
175
+ prepare_shard_models
176
+
177
+ # Select group
178
+ group = Group.first(:conditions => { :enabled => true, :open => true }, :order => 'blocks_count ASC')
179
+ raise "Can't find any tablegroups to use for blocks allocation!" unless group
180
+ return group
181
+ end
182
+
183
+ #---------------------------------------------------------------------------------------------------------------
184
+ def block_start_for_key(key)
185
+ block_size.to_i * (key.to_i / block_size.to_i)
186
+ end
187
+
188
+ def block_end_for_key(key)
189
+ block_size.to_i + block_start_for_key(key)
190
+ end
191
+
192
+ #---------------------------------------------------------------------------------------------------------------
193
+ # Create configuration (use mapping connection as a template)
194
+ def shard_connection_config(shard, group_id)
195
+ # Format connection name
196
+ shard_name = "db_charmer_db_block_group_map_#{name}_s%d_g%d" % [ shard.id, group_id]
197
+
198
+ # Here we get the mapping connection's configuration
199
+ # They do not expose configs so we hack in and get the instance var
200
+ # FIXME: Find a better way, maybe move config method to our ar extenstions
201
+ connection.instance_variable_get(:@config).clone.merge(
202
+ # Name for the connection factory
203
+ :connection_name => shard_name,
204
+ # Connection params
205
+ :host => shard.db_host,
206
+ :port => shard.db_port,
207
+ :username => shard.db_user,
208
+ :password => shard.db_pass,
209
+ :database => group_database_name(shard, group_id)
210
+ )
211
+ end
212
+
213
+ def group_database_name(shard, group_id)
214
+ "%s_%05d" % [ shard.db_name_prefix, group_id ]
215
+ end
216
+
217
+ #---------------------------------------------------------------------------------------------------------------
218
+ def create_shard(params)
219
+ params = params.symbolize_keys
220
+ [ :db_host, :db_port, :db_user, :db_pass, :db_name_prefix ].each do |arg|
221
+ raise ArgumentError, "Missing required parameter: #{arg}" unless params[arg]
222
+ end
223
+
224
+ # Prepare model
225
+ prepare_shard_models
226
+
227
+ # Create the record
228
+ Shard.create! do |shard|
229
+ shard.db_host = params[:db_host]
230
+ shard.db_port = params[:db_port]
231
+ shard.db_user = params[:db_user]
232
+ shard.db_pass = params[:db_pass]
233
+ shard.db_name_prefix = params[:db_name_prefix]
234
+ end
235
+ end
236
+
237
+ def shard_connections
238
+ # Find all groups
239
+ prepare_shard_models
240
+ groups = Group.all(:conditions => { :enabled => true }, :include => :shard)
241
+ # Map them to shards
242
+ groups.map { |group| shard_connection_config(group.shard, group.id) }
243
+ end
244
+
245
+ # Prepare model for working with our shards table
246
+ def prepare_shard_models
247
+ Shard.set_table_name(shards_table)
248
+ Shard.switch_connection_to(connection)
249
+
250
+ Group.set_table_name(groups_table)
251
+ Group.switch_connection_to(connection)
252
+ end
253
+
254
+ end
255
+ end
256
+ end
257
+ end