yam-db-charmer 1.7.01 → 1.7.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +2 -0
- data/CHANGES +8 -0
- data/README.rdoc +10 -6
- data/db-charmer.gemspec +2 -2
- data/lib/db_charmer/action_controller/force_slave_reads.rb +2 -2
- data/lib/db_charmer/active_record/connection_switching.rb +17 -0
- data/lib/db_charmer/active_record/db_magic.rb +9 -4
- data/lib/db_charmer/active_record/migration/multi_db_migrations.rb +113 -35
- data/lib/db_charmer/active_record/multi_db_proxy.rb +9 -0
- data/lib/db_charmer/connection_factory.rb +8 -0
- data/lib/db_charmer/rails3/active_record/relation_method.rb +6 -1
- data/lib/db_charmer/rails31/active_record/migration/command_recorder.rb +11 -0
- data/lib/db_charmer/rails31/active_record/preloader/association.rb +21 -0
- data/lib/db_charmer/rails31/active_record/preloader/has_and_belongs_to_many.rb +23 -0
- data/lib/db_charmer/sharding/connection.rb +4 -0
- data/lib/db_charmer/sharding/method/db_block_group_map.rb +12 -177
- data/lib/db_charmer/sharding/method/db_block_group_map_base.rb +270 -0
- data/lib/db_charmer/sharding/method/db_block_map.rb +4 -9
- data/lib/db_charmer/sharding/method/db_block_schema_map.rb +179 -0
- data/lib/db_charmer/sharding/method.rb +2 -0
- data/lib/db_charmer/sharding.rb +28 -0
- data/lib/db_charmer/version.rb +2 -2
- data/lib/db_charmer.rb +50 -22
- data/lib/tasks/databases.rake +30 -4
- data/lib/tasks/test.rake +115 -0
- metadata +19 -12
@@ -43,7 +43,7 @@ module DbCharmer
|
|
43
43
|
# Auto-allocate new blocks
|
44
44
|
block ||= allocate_new_block_for_key(key)
|
45
45
|
rescue ::ActiveRecord::StatementInvalid => e
|
46
|
-
raise unless e.message.include?('Duplicate entry')
|
46
|
+
raise unless e.message.include?('Duplicate entry') || e.message.include?('duplicate key')
|
47
47
|
block = block_for_key(key)
|
48
48
|
end
|
49
49
|
|
@@ -118,13 +118,8 @@ module DbCharmer
|
|
118
118
|
|
119
119
|
# Try to insert a new mapping (ignore duplicate key errors)
|
120
120
|
sql = <<-SQL
|
121
|
-
INSERT INTO #{map_table}
|
122
|
-
|
123
|
-
end_id = #{end_id},
|
124
|
-
shard_id = #{shard.id},
|
125
|
-
block_size = #{block_size},
|
126
|
-
created_at = NOW(),
|
127
|
-
updated_at = NOW()
|
121
|
+
INSERT INTO #{map_table} (start_id, end_id, shard_id, block_size, created_at, updated_at)
|
122
|
+
VALUES (#{start_id}, #{end_id}, #{shard.id}, #{block_size}, NOW(), NOW())
|
128
123
|
SQL
|
129
124
|
connection.execute(sql, "Allocate new block")
|
130
125
|
|
@@ -201,7 +196,7 @@ module DbCharmer
|
|
201
196
|
|
202
197
|
# Prepare model for working with our shards table
|
203
198
|
def prepare_shard_model
|
204
|
-
ShardInfo.
|
199
|
+
ShardInfo.table_name = shards_table
|
205
200
|
ShardInfo.switch_connection_to(connection)
|
206
201
|
end
|
207
202
|
|
@@ -0,0 +1,179 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), '/db_block_group_map_base')
|
2
|
+
|
3
|
+
# This is a more sophisticated sharding method based on a two layer database-backed
|
4
|
+
# blocks map that holds block-shard associations. Record blocks are mapped to groups
|
5
|
+
# (= database schemas) and groups are mapped to shards (= databases).
|
6
|
+
#
|
7
|
+
# It automatically creates new blocks for new keys and assigns them to existing groups.
|
8
|
+
# Warning: make sure to create at least one shard and one group before inserting any records.
|
9
|
+
#
|
10
|
+
module DbCharmer
|
11
|
+
module Sharding
|
12
|
+
module Method
|
13
|
+
class DbBlockSchemaMap
|
14
|
+
include DbBlockGroupMapBase
|
15
|
+
|
16
|
+
# Shard connection info model
|
17
|
+
class Shard < ::ActiveRecord::Base
|
18
|
+
validates_presence_of :db_host
|
19
|
+
validates_presence_of :db_port
|
20
|
+
validates_presence_of :db_user
|
21
|
+
validates_presence_of :db_pass, :unless => Proc.new { |shard| shard.db_pass=='' }
|
22
|
+
validates_presence_of :schema_name_prefix
|
23
|
+
|
24
|
+
has_many :groups, :class_name => 'DbCharmer::Sharding::Method::DbBlockSchemaMap::Group'
|
25
|
+
end
|
26
|
+
|
27
|
+
# Table group info model
|
28
|
+
class Group < ::ActiveRecord::Base
|
29
|
+
validates_presence_of :shard_id
|
30
|
+
belongs_to :shard, :class_name => 'DbCharmer::Sharding::Method::DbBlockSchemaMap::Shard'
|
31
|
+
end
|
32
|
+
|
33
|
+
def connection_name(shard_id)
|
34
|
+
"db_charmer_db_block_schema_map_#{name}_s%d" % shard_id
|
35
|
+
end
|
36
|
+
|
37
|
+
def schema_name(schema_name_prefix, group_id)
|
38
|
+
"%s_%05d" % [ schema_name_prefix, group_id ]
|
39
|
+
end
|
40
|
+
|
41
|
+
#---------------------------------------------------------------------------------------------------------------
|
42
|
+
# Create configuration (use mapping connection as a template)
|
43
|
+
def shard_connection_config(shard, group_id)
|
44
|
+
# Format connection name
|
45
|
+
connection_name = connection_name(shard.id)
|
46
|
+
schema_name = schema_name(shard.schema_name_prefix, group_id)
|
47
|
+
|
48
|
+
# Here we get the mapping connection's configuration
|
49
|
+
# They do not expose configs so we hack in and get the instance var
|
50
|
+
# FIXME: Find a better way, maybe move config method to our ar extenstions
|
51
|
+
connection.config.clone.merge(
|
52
|
+
# Name for the connection factory
|
53
|
+
:connection_name => connection_name,
|
54
|
+
# Connection params
|
55
|
+
:host => shard.db_host,
|
56
|
+
:slave_host => shard.db_slave_host,
|
57
|
+
:port => shard.db_port,
|
58
|
+
:username => shard.db_user,
|
59
|
+
:password => shard.db_pass,
|
60
|
+
:database => shard.db_name,
|
61
|
+
:schema_name => schema_name,
|
62
|
+
:shard_id => shard.id,
|
63
|
+
:sharder_name => self.name
|
64
|
+
)
|
65
|
+
end
|
66
|
+
|
67
|
+
#---------------------------------------------------------------------------------------------------------------
|
68
|
+
def create_shard(params)
|
69
|
+
params = params.symbolize_keys
|
70
|
+
[ :db_host, :db_port, :db_user, :db_name, :schema_name_prefix ].each do |arg|
|
71
|
+
raise ArgumentError, "Missing required parameter: #{arg}" unless params[arg]
|
72
|
+
end
|
73
|
+
|
74
|
+
# Prepare model
|
75
|
+
prepare_shard_models
|
76
|
+
|
77
|
+
# Create the record
|
78
|
+
Shard.create! do |shard|
|
79
|
+
shard.db_host = params[:db_host]
|
80
|
+
shard.db_slave_host = (params[:db_slave_host] || '') if shard.respond_to? :db_slave_host
|
81
|
+
shard.db_port = params[:db_port]
|
82
|
+
shard.db_user = params[:db_user]
|
83
|
+
shard.db_pass = params[:db_pass] || ''
|
84
|
+
shard.db_name = params[:db_name]
|
85
|
+
shard.schema_name_prefix = params[:schema_name_prefix]
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def create_group(shard, open, enabled)
|
90
|
+
# Prepare model
|
91
|
+
prepare_shard_models
|
92
|
+
|
93
|
+
# Create the record
|
94
|
+
group = Group.create! do |group|
|
95
|
+
group.shard_id = shard.id
|
96
|
+
group.open = open
|
97
|
+
group.enabled = enabled
|
98
|
+
end
|
99
|
+
|
100
|
+
old_connection = Group.connection
|
101
|
+
conn_config = shard_connection_config(shard, group.id)
|
102
|
+
Group.switch_connection_to(conn_config)
|
103
|
+
|
104
|
+
schema_name = schema_name(shard.schema_name_prefix, group.id)
|
105
|
+
|
106
|
+
# create schema only if it doesn't already exist
|
107
|
+
sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name='#{schema_name}'"
|
108
|
+
existing_schema = Group.connection.execute(sql)
|
109
|
+
|
110
|
+
unless existing_schema.first
|
111
|
+
sql = "CREATE SCHEMA #{schema_name}"
|
112
|
+
Group.connection.execute(sql)
|
113
|
+
end
|
114
|
+
|
115
|
+
Group.switch_connection_to(old_connection)
|
116
|
+
end
|
117
|
+
|
118
|
+
def shard_connections
|
119
|
+
# Find all groups
|
120
|
+
prepare_shard_models
|
121
|
+
groups = Group.all(:conditions => { :enabled => true }, :include => :shard)
|
122
|
+
# Map them to shards
|
123
|
+
groups.map { |group| shard_connection_config(group.shard, group.id) }
|
124
|
+
end
|
125
|
+
|
126
|
+
# Prepare model for working with our shards table
|
127
|
+
def prepare_shard_models
|
128
|
+
Shard.switch_connection_to(connection)
|
129
|
+
Shard.table_name = shards_table
|
130
|
+
|
131
|
+
Group.switch_connection_to(connection)
|
132
|
+
Group.table_name = groups_table
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
# To be mixed in AR#base.
|
137
|
+
# Allows a schema name to be set at the same time a db connection is selected.
|
138
|
+
module SchemaTableNamePrefix
|
139
|
+
def set_schema_table_name_prefix(con)
|
140
|
+
|
141
|
+
#Rails.logger.debug "set_schema_table_name_prefix: self=#{self} @dbcharmer_table_name_prefix=#{@dbcharmer_table_name_prefix}"
|
142
|
+
if con.is_a?(Hash) && con[:schema_name]
|
143
|
+
new_prefix = con[:schema_name] + '.'
|
144
|
+
else
|
145
|
+
new_prefix = '' if self.to_s=='ActiveRecord::Base' # this is for migrations
|
146
|
+
end
|
147
|
+
|
148
|
+
# remove the old dbcharmer prefix first
|
149
|
+
if !@dbcharmer_table_name_prefix.blank? && @dbcharmer_table_name_prefix != new_prefix
|
150
|
+
self.table_name_prefix = self.table_name_prefix.gsub(/#{Regexp.escape(@dbcharmer_table_name_prefix)}/, '')
|
151
|
+
end
|
152
|
+
|
153
|
+
# Set the new table_name_prefix
|
154
|
+
if new_prefix && @dbcharmer_table_name_prefix != new_prefix
|
155
|
+
@dbcharmer_table_name_prefix = new_prefix
|
156
|
+
self.table_name_prefix = "#{new_prefix}#{self.table_name_prefix}"
|
157
|
+
|
158
|
+
# Reset all forms of table_name that were memoized in Rails.
|
159
|
+
# Don't do it in the context of migrations where the current class
|
160
|
+
# is AR::Base and there is no actual table name.
|
161
|
+
unless self.to_s=='ActiveRecord::Base'
|
162
|
+
#Rails.logger.debug "set_schema_table_name_prefix: resetting"
|
163
|
+
reset_cached_table_name
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
# Rails memoizes table_name and table_name_prefix at the time the models are loaded.
|
169
|
+
# This method forces refreshing those names
|
170
|
+
def reset_cached_table_name
|
171
|
+
self.reset_table_name
|
172
|
+
@arel_table = nil
|
173
|
+
@relation = nil
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
@@ -4,7 +4,9 @@ module DbCharmer
|
|
4
4
|
autoload :Range, 'db_charmer/sharding/method/range'
|
5
5
|
autoload :HashMap, 'db_charmer/sharding/method/hash_map'
|
6
6
|
autoload :DbBlockMap, 'db_charmer/sharding/method/db_block_map'
|
7
|
+
autoload :DbBlockGroupMapBase, 'db_charmer/sharding/method/db_block_group_map_base'
|
7
8
|
autoload :DbBlockGroupMap, 'db_charmer/sharding/method/db_block_group_map'
|
9
|
+
autoload :DbBlockSchemaMap, 'db_charmer/sharding/method/db_block_schema_map'
|
8
10
|
end
|
9
11
|
end
|
10
12
|
end
|
data/lib/db_charmer/sharding.rb
CHANGED
@@ -9,10 +9,38 @@ module DbCharmer
|
|
9
9
|
def self.register_connection(config)
|
10
10
|
name = config[:name] or raise ArgumentError, "No :name in connection!"
|
11
11
|
@@sharded_connections[name] = DbCharmer::Sharding::Connection.new(config)
|
12
|
+
|
13
|
+
# Enable multi-db migrations
|
14
|
+
if config[:method] == :db_block_schema_map
|
15
|
+
::ActiveRecord::Base.extend(DbCharmer::Sharding::Method::SchemaTableNamePrefix)
|
16
|
+
end
|
12
17
|
end
|
13
18
|
|
19
|
+
# name is the is the sharded_connection name passed to db_magic in the model
|
14
20
|
def self.sharded_connection(name)
|
15
21
|
@@sharded_connections[name] or raise ArgumentError, "Invalid sharded connection name!"
|
16
22
|
end
|
23
|
+
|
24
|
+
# Return the DbCharmer::Sharding::Connection that matches name
|
25
|
+
# name is the the connection name calculated in the sharder for a specific database
|
26
|
+
def self.sharded_connection_by_connection_name(name)
|
27
|
+
connection = @@sharded_connections.detect do |c|
|
28
|
+
c[1].config[:connection] == name.to_sym
|
29
|
+
end
|
30
|
+
connection[1] if connection
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.sharder_for_connection_name(sharder_name)
|
34
|
+
connection = sharded_connection(sharder_name)
|
35
|
+
return connection.sharder if connection
|
36
|
+
end
|
37
|
+
|
38
|
+
# Return shard record for the given config hash.
|
39
|
+
# connection_config is the config hash taken from a PostgreSQLAdapter instance.
|
40
|
+
def self.shard_for_connection_name(connection_config)
|
41
|
+
sharder = sharder_for_connection_name(connection_config[:sharder_name])
|
42
|
+
return unless sharder
|
43
|
+
sharder.shard_class.where(:id => connection_config[:shard_id]).first
|
44
|
+
end
|
17
45
|
end
|
18
46
|
end
|
data/lib/db_charmer/version.rb
CHANGED
data/lib/db_charmer.rb
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
# In Rails 2.2 they did not add it to the autoload so it won't work w/o this require
|
2
2
|
require 'active_record/version' unless defined?(::ActiveRecord::VERSION::MAJOR)
|
3
3
|
|
4
|
+
# Load the rake tasks so that they become available in the application
|
5
|
+
path = File.join(File.dirname(__FILE__),'tasks/*.rake')
|
6
|
+
Dir[path].each { |ext| load ext } if defined?(Rake)
|
7
|
+
|
4
8
|
module DbCharmer
|
5
9
|
# Configure autoload
|
6
10
|
autoload :Sharding, 'db_charmer/sharding'
|
@@ -14,6 +18,11 @@ module DbCharmer
|
|
14
18
|
::ActiveRecord::VERSION::MAJOR > 2
|
15
19
|
end
|
16
20
|
|
21
|
+
# Used in all Rails3.1-specific places
|
22
|
+
def self.rails31?
|
23
|
+
rails3? && ::ActiveRecord::VERSION::MINOR >= 1
|
24
|
+
end
|
25
|
+
|
17
26
|
# Used in all Rails2-specific places
|
18
27
|
def self.rails2?
|
19
28
|
::ActiveRecord::VERSION::MAJOR == 2
|
@@ -134,32 +143,43 @@ end
|
|
134
143
|
|
135
144
|
# Enable connection proxy for associations
|
136
145
|
# WARNING: Inject methods to association class right here (they proxy include calls somewhere else, so include does not work)
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
end
|
146
|
+
association_proxy_class = DbCharmer.rails31? ? ActiveRecord::Associations::CollectionProxy : ActiveRecord::Associations::AssociationProxy
|
147
|
+
association_proxy_class.class_eval do
|
148
|
+
def proxy?
|
149
|
+
true
|
150
|
+
end
|
143
151
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
152
|
+
if DbCharmer.rails31?
|
153
|
+
def on_db(con, proxy_target = nil, &block)
|
154
|
+
proxy_target ||= self
|
155
|
+
@association.klass.on_db(con, proxy_target, &block)
|
156
|
+
end
|
148
157
|
|
149
|
-
|
150
|
-
|
151
|
-
|
158
|
+
def on_slave(con = nil, &block)
|
159
|
+
@association.klass.on_slave(con, self, &block)
|
160
|
+
end
|
152
161
|
|
153
|
-
|
154
|
-
|
155
|
-
|
162
|
+
def on_master(&block)
|
163
|
+
@association.klass.on_master(self, &block)
|
164
|
+
end
|
165
|
+
else
|
166
|
+
def on_db(con, proxy_target = nil, &block)
|
167
|
+
proxy_target ||= self
|
168
|
+
@reflection.klass.on_db(con, proxy_target, &block)
|
169
|
+
end
|
170
|
+
|
171
|
+
def on_slave(con = nil, &block)
|
172
|
+
@reflection.klass.on_slave(con, self, &block)
|
173
|
+
end
|
174
|
+
|
175
|
+
def on_master(&block)
|
176
|
+
@reflection.klass.on_master(self, &block)
|
156
177
|
end
|
157
178
|
end
|
158
179
|
end
|
159
180
|
|
160
|
-
# Enable multi-db migrations
|
161
181
|
require 'db_charmer/active_record/migration/multi_db_migrations'
|
162
|
-
ActiveRecord::Migration.
|
182
|
+
ActiveRecord::Migration.send(:include, DbCharmer::ActiveRecord::Migration::MultiDbMigrations)
|
163
183
|
|
164
184
|
# Enable the magic
|
165
185
|
if DbCharmer.rails3?
|
@@ -173,11 +193,19 @@ require 'db_charmer/active_record/db_magic'
|
|
173
193
|
ActiveRecord::Base.extend(DbCharmer::ActiveRecord::DbMagic)
|
174
194
|
|
175
195
|
# Setup association preload magic
|
176
|
-
|
177
|
-
|
196
|
+
if DbCharmer.rails31?
|
197
|
+
require 'db_charmer/rails31/active_record/preloader/association'
|
198
|
+
ActiveRecord::Associations::Preloader::Association.send(:include, DbCharmer::ActiveRecord::Preloader::Association)
|
199
|
+
require 'db_charmer/rails31/active_record/preloader/has_and_belongs_to_many'
|
200
|
+
ActiveRecord::Associations::Preloader::HasAndBelongsToMany.send(:include, DbCharmer::ActiveRecord::Preloader::HasAndBelongsToMany)
|
201
|
+
else
|
202
|
+
require 'db_charmer/active_record/association_preload'
|
203
|
+
ActiveRecord::Base.extend(DbCharmer::ActiveRecord::AssociationPreload)
|
204
|
+
|
205
|
+
# Open up really useful API method
|
206
|
+
ActiveRecord::AssociationPreload::ClassMethods.send(:public, :preload_associations)
|
207
|
+
end
|
178
208
|
|
179
|
-
# Open up really useful API method
|
180
|
-
ActiveRecord::AssociationPreload::ClassMethods.send(:public, :preload_associations)
|
181
209
|
|
182
210
|
class ::ActiveRecord::Base
|
183
211
|
class << self
|
data/lib/tasks/databases.rake
CHANGED
@@ -23,7 +23,7 @@ namespace :db_charmer do
|
|
23
23
|
|
24
24
|
desc 'Create the databases defined in config/database.yml for the current RAILS_ENV'
|
25
25
|
task :create => "db:load_config" do
|
26
|
-
create_core_and_sub_database(ActiveRecord::Base.configurations[
|
26
|
+
create_core_and_sub_database(ActiveRecord::Base.configurations[Rails.env])
|
27
27
|
end
|
28
28
|
|
29
29
|
def create_core_and_sub_database(config)
|
@@ -45,11 +45,16 @@ namespace :db_charmer do
|
|
45
45
|
local_database?(config) { drop_core_and_sub_database(config) }
|
46
46
|
end
|
47
47
|
end
|
48
|
+
|
49
|
+
task :schema_shards => :environment do
|
50
|
+
config = ::ActiveRecord::Base.configurations[Rails.env || 'development']
|
51
|
+
drop_schema_shard_databases(config)
|
52
|
+
end
|
48
53
|
end
|
49
54
|
|
50
55
|
desc 'Drops the database for the current RAILS_ENV'
|
51
56
|
task :drop => "db:load_config" do
|
52
|
-
config = ::ActiveRecord::Base.configurations[
|
57
|
+
config = ::ActiveRecord::Base.configurations[Rails.env || 'development']
|
53
58
|
begin
|
54
59
|
drop_core_and_sub_database(config)
|
55
60
|
rescue Exception => e
|
@@ -68,15 +73,36 @@ namespace :db_charmer do
|
|
68
73
|
end
|
69
74
|
|
70
75
|
def drop_core_and_sub_database(config)
|
76
|
+
exit unless Rails.env=='test'
|
71
77
|
drop_database(config)
|
72
78
|
config.each_value do | sub_config |
|
73
79
|
next unless sub_config.is_a?(Hash)
|
74
80
|
next unless sub_config['database']
|
75
81
|
begin
|
76
82
|
drop_database(sub_config)
|
77
|
-
rescue
|
78
|
-
$stderr.puts "#{config['database']} not exists"
|
83
|
+
rescue => e
|
84
|
+
$stderr.puts "#{e.to_s}, #{config['database']} not exists"
|
79
85
|
end
|
80
86
|
end
|
81
87
|
end
|
82
88
|
|
89
|
+
# find all schema sharded databases and drop them
|
90
|
+
def drop_schema_shard_databases(config)
|
91
|
+
exit unless Rails.env=='test'
|
92
|
+
config.each do |name, sub_config|
|
93
|
+
next unless sub_config.is_a?(Hash)
|
94
|
+
next unless sub_config['database']
|
95
|
+
|
96
|
+
# find the database connection for the schema admin db
|
97
|
+
next unless sub_config['shard_db_name_prefix']
|
98
|
+
connection = DbCharmer::Sharding.sharded_connection_by_connection_name(name)
|
99
|
+
next unless connection
|
100
|
+
|
101
|
+
# itereate through entries in the shards_info table to find the
|
102
|
+
# databases that will be dropped
|
103
|
+
dbgm = DbCharmer::Sharding::Method::DbBlockSchemaMap.new(connection.config)
|
104
|
+
dbgm.drop_all_shard_databases
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
|