yam-db-charmer 1.7.01 → 1.7.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -43,7 +43,7 @@ module DbCharmer
43
43
  # Auto-allocate new blocks
44
44
  block ||= allocate_new_block_for_key(key)
45
45
  rescue ::ActiveRecord::StatementInvalid => e
46
- raise unless e.message.include?('Duplicate entry')
46
+ raise unless e.message.include?('Duplicate entry') || e.message.include?('duplicate key')
47
47
  block = block_for_key(key)
48
48
  end
49
49
 
@@ -118,13 +118,8 @@ module DbCharmer
118
118
 
119
119
  # Try to insert a new mapping (ignore duplicate key errors)
120
120
  sql = <<-SQL
121
- INSERT INTO #{map_table}
122
- SET start_id = #{start_id},
123
- end_id = #{end_id},
124
- shard_id = #{shard.id},
125
- block_size = #{block_size},
126
- created_at = NOW(),
127
- updated_at = NOW()
121
+ INSERT INTO #{map_table} (start_id, end_id, shard_id, block_size, created_at, updated_at)
122
+ VALUES (#{start_id}, #{end_id}, #{shard.id}, #{block_size}, NOW(), NOW())
128
123
  SQL
129
124
  connection.execute(sql, "Allocate new block")
130
125
 
@@ -201,7 +196,7 @@ module DbCharmer
201
196
 
202
197
  # Prepare model for working with our shards table
203
198
  def prepare_shard_model
204
- ShardInfo.set_table_name(shards_table)
199
+ ShardInfo.table_name = shards_table
205
200
  ShardInfo.switch_connection_to(connection)
206
201
  end
207
202
 
@@ -0,0 +1,179 @@
1
+ require File.join(File.dirname(__FILE__), '/db_block_group_map_base')
2
+
3
+ # This is a more sophisticated sharding method based on a two layer database-backed
4
+ # blocks map that holds block-shard associations. Record blocks are mapped to groups
5
+ # (= database schemas) and groups are mapped to shards (= databases).
6
+ #
7
+ # It automatically creates new blocks for new keys and assigns them to existing groups.
8
+ # Warning: make sure to create at least one shard and one group before inserting any records.
9
+ #
10
+ module DbCharmer
11
+ module Sharding
12
+ module Method
13
+ class DbBlockSchemaMap
14
+ include DbBlockGroupMapBase
15
+
16
+ # Shard connection info model
17
+ class Shard < ::ActiveRecord::Base
18
+ validates_presence_of :db_host
19
+ validates_presence_of :db_port
20
+ validates_presence_of :db_user
21
+ validates_presence_of :db_pass, :unless => Proc.new { |shard| shard.db_pass=='' }
22
+ validates_presence_of :schema_name_prefix
23
+
24
+ has_many :groups, :class_name => 'DbCharmer::Sharding::Method::DbBlockSchemaMap::Group'
25
+ end
26
+
27
+ # Table group info model
28
+ class Group < ::ActiveRecord::Base
29
+ validates_presence_of :shard_id
30
+ belongs_to :shard, :class_name => 'DbCharmer::Sharding::Method::DbBlockSchemaMap::Shard'
31
+ end
32
+
33
+ def connection_name(shard_id)
34
+ "db_charmer_db_block_schema_map_#{name}_s%d" % shard_id
35
+ end
36
+
37
+ def schema_name(schema_name_prefix, group_id)
38
+ "%s_%05d" % [ schema_name_prefix, group_id ]
39
+ end
40
+
41
+ #---------------------------------------------------------------------------------------------------------------
42
+ # Create configuration (use mapping connection as a template)
43
+ def shard_connection_config(shard, group_id)
44
+ # Format connection name
45
+ connection_name = connection_name(shard.id)
46
+ schema_name = schema_name(shard.schema_name_prefix, group_id)
47
+
48
+ # Here we get the mapping connection's configuration
49
+ # They do not expose configs so we hack in and get the instance var
50
+ # FIXME: Find a better way, maybe move config method to our ar extenstions
51
+ connection.config.clone.merge(
52
+ # Name for the connection factory
53
+ :connection_name => connection_name,
54
+ # Connection params
55
+ :host => shard.db_host,
56
+ :slave_host => shard.db_slave_host,
57
+ :port => shard.db_port,
58
+ :username => shard.db_user,
59
+ :password => shard.db_pass,
60
+ :database => shard.db_name,
61
+ :schema_name => schema_name,
62
+ :shard_id => shard.id,
63
+ :sharder_name => self.name
64
+ )
65
+ end
66
+
67
+ #---------------------------------------------------------------------------------------------------------------
68
+ def create_shard(params)
69
+ params = params.symbolize_keys
70
+ [ :db_host, :db_port, :db_user, :db_name, :schema_name_prefix ].each do |arg|
71
+ raise ArgumentError, "Missing required parameter: #{arg}" unless params[arg]
72
+ end
73
+
74
+ # Prepare model
75
+ prepare_shard_models
76
+
77
+ # Create the record
78
+ Shard.create! do |shard|
79
+ shard.db_host = params[:db_host]
80
+ shard.db_slave_host = (params[:db_slave_host] || '') if shard.respond_to? :db_slave_host
81
+ shard.db_port = params[:db_port]
82
+ shard.db_user = params[:db_user]
83
+ shard.db_pass = params[:db_pass] || ''
84
+ shard.db_name = params[:db_name]
85
+ shard.schema_name_prefix = params[:schema_name_prefix]
86
+ end
87
+ end
88
+
89
+ def create_group(shard, open, enabled)
90
+ # Prepare model
91
+ prepare_shard_models
92
+
93
+ # Create the record
94
+ group = Group.create! do |group|
95
+ group.shard_id = shard.id
96
+ group.open = open
97
+ group.enabled = enabled
98
+ end
99
+
100
+ old_connection = Group.connection
101
+ conn_config = shard_connection_config(shard, group.id)
102
+ Group.switch_connection_to(conn_config)
103
+
104
+ schema_name = schema_name(shard.schema_name_prefix, group.id)
105
+
106
+ # create schema only if it doesn't already exist
107
+ sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name='#{schema_name}'"
108
+ existing_schema = Group.connection.execute(sql)
109
+
110
+ unless existing_schema.first
111
+ sql = "CREATE SCHEMA #{schema_name}"
112
+ Group.connection.execute(sql)
113
+ end
114
+
115
+ Group.switch_connection_to(old_connection)
116
+ end
117
+
118
+ def shard_connections
119
+ # Find all groups
120
+ prepare_shard_models
121
+ groups = Group.all(:conditions => { :enabled => true }, :include => :shard)
122
+ # Map them to shards
123
+ groups.map { |group| shard_connection_config(group.shard, group.id) }
124
+ end
125
+
126
+ # Prepare model for working with our shards table
127
+ def prepare_shard_models
128
+ Shard.switch_connection_to(connection)
129
+ Shard.table_name = shards_table
130
+
131
+ Group.switch_connection_to(connection)
132
+ Group.table_name = groups_table
133
+ end
134
+ end
135
+
136
+ # To be mixed in AR#base.
137
+ # Allows a schema name to be set at the same time a db connection is selected.
138
+ module SchemaTableNamePrefix
139
+ def set_schema_table_name_prefix(con)
140
+
141
+ #Rails.logger.debug "set_schema_table_name_prefix: self=#{self} @dbcharmer_table_name_prefix=#{@dbcharmer_table_name_prefix}"
142
+ if con.is_a?(Hash) && con[:schema_name]
143
+ new_prefix = con[:schema_name] + '.'
144
+ else
145
+ new_prefix = '' if self.to_s=='ActiveRecord::Base' # this is for migrations
146
+ end
147
+
148
+ # remove the old dbcharmer prefix first
149
+ if !@dbcharmer_table_name_prefix.blank? && @dbcharmer_table_name_prefix != new_prefix
150
+ self.table_name_prefix = self.table_name_prefix.gsub(/#{Regexp.escape(@dbcharmer_table_name_prefix)}/, '')
151
+ end
152
+
153
+ # Set the new table_name_prefix
154
+ if new_prefix && @dbcharmer_table_name_prefix != new_prefix
155
+ @dbcharmer_table_name_prefix = new_prefix
156
+ self.table_name_prefix = "#{new_prefix}#{self.table_name_prefix}"
157
+
158
+ # Reset all forms of table_name that were memoized in Rails.
159
+ # Don't do it in the context of migrations where the current class
160
+ # is AR::Base and there is no actual table name.
161
+ unless self.to_s=='ActiveRecord::Base'
162
+ #Rails.logger.debug "set_schema_table_name_prefix: resetting"
163
+ reset_cached_table_name
164
+ end
165
+ end
166
+ end
167
+
168
+ # Rails memoizes table_name and table_name_prefix at the time the models are loaded.
169
+ # This method forces refreshing those names
170
+ def reset_cached_table_name
171
+ self.reset_table_name
172
+ @arel_table = nil
173
+ @relation = nil
174
+ end
175
+ end
176
+
177
+ end
178
+ end
179
+ end
@@ -4,7 +4,9 @@ module DbCharmer
4
4
  autoload :Range, 'db_charmer/sharding/method/range'
5
5
  autoload :HashMap, 'db_charmer/sharding/method/hash_map'
6
6
  autoload :DbBlockMap, 'db_charmer/sharding/method/db_block_map'
7
+ autoload :DbBlockGroupMapBase, 'db_charmer/sharding/method/db_block_group_map_base'
7
8
  autoload :DbBlockGroupMap, 'db_charmer/sharding/method/db_block_group_map'
9
+ autoload :DbBlockSchemaMap, 'db_charmer/sharding/method/db_block_schema_map'
8
10
  end
9
11
  end
10
12
  end
@@ -9,10 +9,38 @@ module DbCharmer
9
9
  def self.register_connection(config)
10
10
  name = config[:name] or raise ArgumentError, "No :name in connection!"
11
11
  @@sharded_connections[name] = DbCharmer::Sharding::Connection.new(config)
12
+
13
+ # Enable multi-db migrations
14
+ if config[:method] == :db_block_schema_map
15
+ ::ActiveRecord::Base.extend(DbCharmer::Sharding::Method::SchemaTableNamePrefix)
16
+ end
12
17
  end
13
18
 
19
+ # name is the is the sharded_connection name passed to db_magic in the model
14
20
  def self.sharded_connection(name)
15
21
  @@sharded_connections[name] or raise ArgumentError, "Invalid sharded connection name!"
16
22
  end
23
+
24
+ # Return the DbCharmer::Sharding::Connection that matches name
25
+ # name is the the connection name calculated in the sharder for a specific database
26
+ def self.sharded_connection_by_connection_name(name)
27
+ connection = @@sharded_connections.detect do |c|
28
+ c[1].config[:connection] == name.to_sym
29
+ end
30
+ connection[1] if connection
31
+ end
32
+
33
+ def self.sharder_for_connection_name(sharder_name)
34
+ connection = sharded_connection(sharder_name)
35
+ return connection.sharder if connection
36
+ end
37
+
38
+ # Return shard record for the given config hash.
39
+ # connection_config is the config hash taken from a PostgreSQLAdapter instance.
40
+ def self.shard_for_connection_name(connection_config)
41
+ sharder = sharder_for_connection_name(connection_config[:sharder_name])
42
+ return unless sharder
43
+ sharder.shard_class.where(:id => connection_config[:shard_id]).first
44
+ end
17
45
  end
18
46
  end
@@ -2,8 +2,8 @@ module DbCharmer
2
2
  module Version
3
3
  MAJOR = 1
4
4
  MINOR = 7
5
- PATCH = "01"
6
- BUILD = nil
5
+ PATCH = 4
6
+ BUILD = 0
7
7
 
8
8
  STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
9
9
  end
data/lib/db_charmer.rb CHANGED
@@ -1,6 +1,10 @@
1
1
  # In Rails 2.2 they did not add it to the autoload so it won't work w/o this require
2
2
  require 'active_record/version' unless defined?(::ActiveRecord::VERSION::MAJOR)
3
3
 
4
+ # Load the rake tasks so that they become available in the application
5
+ path = File.join(File.dirname(__FILE__),'tasks/*.rake')
6
+ Dir[path].each { |ext| load ext } if defined?(Rake)
7
+
4
8
  module DbCharmer
5
9
  # Configure autoload
6
10
  autoload :Sharding, 'db_charmer/sharding'
@@ -14,6 +18,11 @@ module DbCharmer
14
18
  ::ActiveRecord::VERSION::MAJOR > 2
15
19
  end
16
20
 
21
+ # Used in all Rails3.1-specific places
22
+ def self.rails31?
23
+ rails3? && ::ActiveRecord::VERSION::MINOR >= 1
24
+ end
25
+
17
26
  # Used in all Rails2-specific places
18
27
  def self.rails2?
19
28
  ::ActiveRecord::VERSION::MAJOR == 2
@@ -134,32 +143,43 @@ end
134
143
 
135
144
  # Enable connection proxy for associations
136
145
  # WARNING: Inject methods to association class right here (they proxy include calls somewhere else, so include does not work)
137
- module ActiveRecord
138
- module Associations
139
- class AssociationProxy
140
- def proxy?
141
- true
142
- end
146
+ association_proxy_class = DbCharmer.rails31? ? ActiveRecord::Associations::CollectionProxy : ActiveRecord::Associations::AssociationProxy
147
+ association_proxy_class.class_eval do
148
+ def proxy?
149
+ true
150
+ end
143
151
 
144
- def on_db(con, proxy_target = nil, &block)
145
- proxy_target ||= self
146
- @reflection.klass.on_db(con, proxy_target, &block)
147
- end
152
+ if DbCharmer.rails31?
153
+ def on_db(con, proxy_target = nil, &block)
154
+ proxy_target ||= self
155
+ @association.klass.on_db(con, proxy_target, &block)
156
+ end
148
157
 
149
- def on_slave(con = nil, &block)
150
- @reflection.klass.on_slave(con, self, &block)
151
- end
158
+ def on_slave(con = nil, &block)
159
+ @association.klass.on_slave(con, self, &block)
160
+ end
152
161
 
153
- def on_master(&block)
154
- @reflection.klass.on_master(self, &block)
155
- end
162
+ def on_master(&block)
163
+ @association.klass.on_master(self, &block)
164
+ end
165
+ else
166
+ def on_db(con, proxy_target = nil, &block)
167
+ proxy_target ||= self
168
+ @reflection.klass.on_db(con, proxy_target, &block)
169
+ end
170
+
171
+ def on_slave(con = nil, &block)
172
+ @reflection.klass.on_slave(con, self, &block)
173
+ end
174
+
175
+ def on_master(&block)
176
+ @reflection.klass.on_master(self, &block)
156
177
  end
157
178
  end
158
179
  end
159
180
 
160
- # Enable multi-db migrations
161
181
  require 'db_charmer/active_record/migration/multi_db_migrations'
162
- ActiveRecord::Migration.extend(DbCharmer::ActiveRecord::Migration::MultiDbMigrations)
182
+ ActiveRecord::Migration.send(:include, DbCharmer::ActiveRecord::Migration::MultiDbMigrations)
163
183
 
164
184
  # Enable the magic
165
185
  if DbCharmer.rails3?
@@ -173,11 +193,19 @@ require 'db_charmer/active_record/db_magic'
173
193
  ActiveRecord::Base.extend(DbCharmer::ActiveRecord::DbMagic)
174
194
 
175
195
  # Setup association preload magic
176
- require 'db_charmer/active_record/association_preload'
177
- ActiveRecord::Base.extend(DbCharmer::ActiveRecord::AssociationPreload)
196
+ if DbCharmer.rails31?
197
+ require 'db_charmer/rails31/active_record/preloader/association'
198
+ ActiveRecord::Associations::Preloader::Association.send(:include, DbCharmer::ActiveRecord::Preloader::Association)
199
+ require 'db_charmer/rails31/active_record/preloader/has_and_belongs_to_many'
200
+ ActiveRecord::Associations::Preloader::HasAndBelongsToMany.send(:include, DbCharmer::ActiveRecord::Preloader::HasAndBelongsToMany)
201
+ else
202
+ require 'db_charmer/active_record/association_preload'
203
+ ActiveRecord::Base.extend(DbCharmer::ActiveRecord::AssociationPreload)
204
+
205
+ # Open up really useful API method
206
+ ActiveRecord::AssociationPreload::ClassMethods.send(:public, :preload_associations)
207
+ end
178
208
 
179
- # Open up really useful API method
180
- ActiveRecord::AssociationPreload::ClassMethods.send(:public, :preload_associations)
181
209
 
182
210
  class ::ActiveRecord::Base
183
211
  class << self
@@ -23,7 +23,7 @@ namespace :db_charmer do
23
23
 
24
24
  desc 'Create the databases defined in config/database.yml for the current RAILS_ENV'
25
25
  task :create => "db:load_config" do
26
- create_core_and_sub_database(ActiveRecord::Base.configurations[RAILS_ENV])
26
+ create_core_and_sub_database(ActiveRecord::Base.configurations[Rails.env])
27
27
  end
28
28
 
29
29
  def create_core_and_sub_database(config)
@@ -45,11 +45,16 @@ namespace :db_charmer do
45
45
  local_database?(config) { drop_core_and_sub_database(config) }
46
46
  end
47
47
  end
48
+
49
+ task :schema_shards => :environment do
50
+ config = ::ActiveRecord::Base.configurations[Rails.env || 'development']
51
+ drop_schema_shard_databases(config)
52
+ end
48
53
  end
49
54
 
50
55
  desc 'Drops the database for the current RAILS_ENV'
51
56
  task :drop => "db:load_config" do
52
- config = ::ActiveRecord::Base.configurations[RAILS_ENV || 'development']
57
+ config = ::ActiveRecord::Base.configurations[Rails.env || 'development']
53
58
  begin
54
59
  drop_core_and_sub_database(config)
55
60
  rescue Exception => e
@@ -68,15 +73,36 @@ namespace :db_charmer do
68
73
  end
69
74
 
70
75
  def drop_core_and_sub_database(config)
76
+ exit unless Rails.env=='test'
71
77
  drop_database(config)
72
78
  config.each_value do | sub_config |
73
79
  next unless sub_config.is_a?(Hash)
74
80
  next unless sub_config['database']
75
81
  begin
76
82
  drop_database(sub_config)
77
- rescue
78
- $stderr.puts "#{config['database']} not exists"
83
+ rescue => e
84
+ $stderr.puts "#{e.to_s}, #{config['database']} not exists"
79
85
  end
80
86
  end
81
87
  end
82
88
 
89
+ # find all schema sharded databases and drop them
90
+ def drop_schema_shard_databases(config)
91
+ exit unless Rails.env=='test'
92
+ config.each do |name, sub_config|
93
+ next unless sub_config.is_a?(Hash)
94
+ next unless sub_config['database']
95
+
96
+ # find the database connection for the schema admin db
97
+ next unless sub_config['shard_db_name_prefix']
98
+ connection = DbCharmer::Sharding.sharded_connection_by_connection_name(name)
99
+ next unless connection
100
+
101
+ # itereate through entries in the shards_info table to find the
102
+ # databases that will be dropped
103
+ dbgm = DbCharmer::Sharding::Method::DbBlockSchemaMap.new(connection.config)
104
+ dbgm.drop_all_shard_databases
105
+ end
106
+ end
107
+
108
+