yam-db-charmer 1.7.01 → 1.7.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,6 +15,10 @@ module DbCharmer
15
15
  sharder_class.new(config)
16
16
  end
17
17
 
18
+ def reload_sharder
19
+ @sharder = instantiate_sharder
20
+ end
21
+
18
22
  def shard_connections
19
23
  sharder.respond_to?(:shard_connections) ? sharder.shard_connections : nil
20
24
  end
@@ -9,6 +9,9 @@ module DbCharmer
9
9
  module Sharding
10
10
  module Method
11
11
  class DbBlockGroupMap
12
+
13
+ include DbBlockGroupMapBase
14
+
12
15
  # Shard connection info model
13
16
  class Shard < ::ActiveRecord::Base
14
17
  validates_presence_of :db_host
@@ -26,169 +29,6 @@ module DbCharmer
26
29
  belongs_to :shard, :class_name => 'DbCharmer::Sharding::Method::DbBlockGroupMap::Shard'
27
30
  end
28
31
 
29
- #---------------------------------------------------------------------------------------------------------------
30
- # Sharder name
31
- attr_accessor :name
32
-
33
- # Mapping db connection
34
- attr_accessor :connection, :connection_name
35
-
36
- # Mapping table name
37
- attr_accessor :map_table
38
-
39
- # Tablegroups table name
40
- attr_accessor :groups_table
41
-
42
- # Shards table name
43
- attr_accessor :shards_table
44
-
45
- # Sharding keys block size
46
- attr_accessor :block_size
47
-
48
- def initialize(config)
49
- @name = config[:name] or raise(ArgumentError, "Missing required :name parameter!")
50
- @connection = DbCharmer::ConnectionFactory.connect(config[:connection], true)
51
- @block_size = (config[:block_size] || 10000).to_i
52
-
53
- @map_table = config[:map_table] or raise(ArgumentError, "Missing required :map_table parameter!")
54
- @groups_table = config[:groups_table] or raise(ArgumentError, "Missing required :groups_table parameter!")
55
- @shards_table = config[:shards_table] or raise(ArgumentError, "Missing required :shards_table parameter!")
56
-
57
- # Local caches
58
- @shard_info_cache = {}
59
- @group_info_cache = {}
60
-
61
- @blocks_cache = Rails.cache
62
- @blocks_cache_prefix = config[:blocks_cache_prefix] || "#{@name}_block:"
63
- end
64
-
65
- #---------------------------------------------------------------------------------------------------------------
66
- def shard_for_key(key)
67
- block = block_for_key(key)
68
-
69
- # Auto-allocate new blocks
70
- block ||= allocate_new_block_for_key(key)
71
- raise ArgumentError, "Invalid key value, no shards found for this key and could not create a new block!" unless block
72
-
73
- # Load shard
74
- group_id = block['group_id'].to_i
75
- shard_info = shard_info_by_group_id(group_id)
76
-
77
- # Get config
78
- shard_connection_config(shard_info, group_id)
79
- end
80
-
81
- #---------------------------------------------------------------------------------------------------------------
82
- # Returns a block for a key
83
- def block_for_key(key, cache = true)
84
- # Cleanup the cache if asked to
85
- key_range = [ block_start_for_key(key), block_end_for_key(key) ]
86
- block_cache_key = "%d-%d" % key_range
87
-
88
- if cache
89
- cached_block = get_cached_block(block_cache_key)
90
- return cached_block if cached_block
91
- end
92
-
93
- # Fetch cached value or load from db
94
- block = begin
95
- sql = "SELECT * FROM #{map_table} WHERE start_id = #{key_range.first} AND end_id = #{key_range.last} LIMIT 1"
96
- connection.select_one(sql, 'Find a shard block')
97
- end
98
-
99
- set_cached_block(block_cache_key, block)
100
-
101
- return block
102
- end
103
-
104
- #---------------------------------------------------------------------------------------------------------------
105
- def get_cached_block(block_cache_key)
106
- @blocks_cache.read("#{@blocks_cache_prefix}#{block_cache_key}")
107
- end
108
-
109
- def set_cached_block(block_cache_key, block)
110
- @blocks_cache.write("#{@blocks_cache_prefix}#{block_cache_key}", block)
111
- end
112
-
113
- #---------------------------------------------------------------------------------------------------------------
114
- # Load group info
115
- def group_info_by_id(group_id, cache = true)
116
- # Cleanup the cache if asked to
117
- @group_info_cache[group_id] = nil unless cache
118
-
119
- # Either load from cache or from db
120
- @group_info_cache[group_id] ||= begin
121
- prepare_shard_models
122
- Group.find_by_id(group_id)
123
- end
124
- end
125
-
126
- # Load shard info
127
- def shard_info_by_id(shard_id, cache = true)
128
- # Cleanup the cache if asked to
129
- @shard_info_cache[shard_id] = nil unless cache
130
-
131
- # Either load from cache or from db
132
- @shard_info_cache[shard_id] ||= begin
133
- prepare_shard_models
134
- Shard.find_by_id(shard_id)
135
- end
136
- end
137
-
138
- # Load shard info using mapping info for a group
139
- def shard_info_by_group_id(group_id)
140
- # Load group
141
- group_info = group_info_by_id(group_id)
142
- raise ArgumentError, "Invalid group_id: #{group_id}" unless group_info
143
-
144
- shard_info = shard_info_by_id(group_info.shard_id)
145
- raise ArgumentError, "Invalid shard_id: #{group_info.shard_id}" unless shard_info
146
-
147
- return shard_info
148
- end
149
-
150
- #---------------------------------------------------------------------------------------------------------------
151
- def allocate_new_block_for_key(key)
152
- # Can't find any groups to use for blocks allocation!
153
- return nil unless group = least_loaded_group
154
-
155
- # Figure out block limits
156
- start_id = block_start_for_key(key)
157
- end_id = block_end_for_key(key)
158
-
159
- # Try to insert a new mapping (ignore duplicate key errors)
160
- sql = <<-SQL
161
- INSERT INTO #{map_table}
162
- (start_id, end_id, group_id, block_size, created_at, updated_at) VALUES
163
- (#{start_id}, #{end_id}, #{group.id}, #{block_size}, NOW(), NOW())
164
- SQL
165
- connection.execute(sql, "Allocate new block")
166
-
167
- # Increment the blocks counter on the shard
168
- Group.update_counters(group.id, :blocks_count => +1)
169
-
170
- # Retry block search after creation
171
- block_for_key(key)
172
- end
173
-
174
- def least_loaded_group
175
- prepare_shard_models
176
-
177
- # Select group
178
- group = Group.first(:conditions => { :enabled => true, :open => true }, :order => 'blocks_count ASC')
179
- raise "Can't find any tablegroups to use for blocks allocation!" unless group
180
- return group
181
- end
182
-
183
- #---------------------------------------------------------------------------------------------------------------
184
- def block_start_for_key(key)
185
- block_size.to_i * (key.to_i / block_size.to_i)
186
- end
187
-
188
- def block_end_for_key(key)
189
- block_size.to_i + block_start_for_key(key)
190
- end
191
-
192
32
  #---------------------------------------------------------------------------------------------------------------
193
33
  # Create configuration (use mapping connection as a template)
194
34
  def shard_connection_config(shard, group_id)
@@ -234,23 +74,18 @@ module DbCharmer
234
74
  end
235
75
  end
236
76
 
237
- def shard_connections
238
- # Find all groups
77
+ def create_group(shard_id, open, enabled)
78
+ # Prepare model
239
79
  prepare_shard_models
240
- groups = Group.all(:conditions => { :enabled => true }, :include => :shard)
241
- # Map them to shards
242
- groups.map { |group| shard_connection_config(group.shard, group.id) }
243
- end
244
80
 
245
- # Prepare model for working with our shards table
246
- def prepare_shard_models
247
- Shard.set_table_name(shards_table)
248
- Shard.switch_connection_to(connection)
249
-
250
- Group.set_table_name(groups_table)
251
- Group.switch_connection_to(connection)
81
+ # Create the record
82
+ Group.create! do |group|
83
+ group.shard_id = shard_id
84
+ group.open = open
85
+ group.enabled = enabled
86
+ end
252
87
  end
253
-
88
+
254
89
  end
255
90
  end
256
91
  end
@@ -0,0 +1,270 @@
1
+ # This is a more sophisticated sharding method based on a two layer database-backed
2
+ # blocks map that holds block-shard associations. Record blocks are mapped to tablegroups
3
+ # and groups are mapped to shards.
4
+ #
5
+ # It automatically creates new blocks for new keys and assigns them to existing groups.
6
+ # Warning: make sure to create at least one shard and one group before inserting any records.
7
+ #
8
+ module DbCharmer
9
+ module Sharding
10
+ module Method
11
+ module DbBlockGroupMapBase
12
+
13
+ #---------------------------------------------------------------------------------------------------------------
14
+ # Sharder name
15
+ attr_accessor :name
16
+
17
+ # Mapping db connection
18
+ attr_accessor :connection, :connection_name
19
+
20
+ # Mapping table name
21
+ attr_accessor :map_table
22
+
23
+ # Tablegroups table name
24
+ attr_accessor :groups_table
25
+
26
+ # Shards table name
27
+ attr_accessor :shards_table
28
+
29
+ # Sharding keys block size
30
+ attr_accessor :block_size
31
+
32
+ def initialize(config)
33
+ @name = config[:name] or raise(ArgumentError, "Missing required :name parameter!")
34
+
35
+ if config[:method] == :db_block_schema_map
36
+ if ::ActiveRecord::Base.configurations[DbCharmer.env]['adapter'] != 'postgresql'
37
+ raise(ArgumentError, 'DbBlockSchemaMap method can only be used with the postgresql adapter')
38
+ end
39
+ end
40
+
41
+ @connection = DbCharmer::ConnectionFactory.connect(config[:connection], true)
42
+ @block_size = (config[:block_size] || 10000).to_i
43
+
44
+ @map_table = config[:map_table] or raise(ArgumentError, "Missing required :map_table parameter!")
45
+ @groups_table = config[:groups_table] or raise(ArgumentError, "Missing required :groups_table parameter!")
46
+ @shards_table = config[:shards_table] or raise(ArgumentError, "Missing required :shards_table parameter!")
47
+
48
+ # Local caches
49
+ @shard_info_cache = {}
50
+ @group_info_cache = {}
51
+
52
+ @blocks_cache = Rails.cache
53
+ @blocks_cache_prefix = config[:blocks_cache_prefix] || "#{@name}_block:"
54
+ end
55
+
56
+ #---------------------------------------------------------------------------------------------------------------
57
+ def shard_for_key(key)
58
+ block = block_for_key(key)
59
+
60
+ # Auto-allocate new blocks
61
+ block ||= allocate_new_block_for_key(key)
62
+ raise ArgumentError, "Invalid key value, no shards found for this key and could not create a new block!" unless block
63
+
64
+ # Load shard
65
+ group_id = block['group_id'].to_i
66
+ shard_info = shard_info_by_group_id(group_id)
67
+
68
+ # Get config
69
+ shard_connection_config(shard_info, group_id)
70
+ end
71
+
72
+ #---------------------------------------------------------------------------------------------------------------
73
+ # Returns a block for a key
74
+ def block_for_key(key, cache = true)
75
+ # Cleanup the cache if asked to
76
+ key_range = [ block_start_for_key(key), block_end_for_key(key) ]
77
+ block_cache_key = "%d-%d" % key_range
78
+
79
+ if cache
80
+ cached_block = get_cached_block(block_cache_key)
81
+ return cached_block if cached_block
82
+ end
83
+
84
+ # Fetch cached value or load from db
85
+ block = begin
86
+ sql = "SELECT * FROM #{map_table} WHERE start_id = #{key_range.first} AND end_id = #{key_range.last} LIMIT 1"
87
+ connection.select_one(sql, 'Find a shard block')
88
+ end
89
+
90
+ set_cached_block(block_cache_key, block)
91
+
92
+ return block
93
+ end
94
+
95
+ #---------------------------------------------------------------------------------------------------------------
96
+ def get_cached_block(block_cache_key)
97
+ @blocks_cache.read("#{@blocks_cache_prefix}#{block_cache_key}")
98
+ end
99
+
100
+ def set_cached_block(block_cache_key, block)
101
+ @blocks_cache.write("#{@blocks_cache_prefix}#{block_cache_key}", block)
102
+ end
103
+
104
+ #---------------------------------------------------------------------------------------------------------------
105
+ # Load group info
106
+ def group_info_by_id(group_id, cache = true)
107
+ # Cleanup the cache if asked to
108
+ @group_info_cache[group_id] = nil unless cache
109
+
110
+ # Either load from cache or from db
111
+ @group_info_cache[group_id] ||= begin
112
+ prepare_shard_models
113
+ group_class.find_by_id(group_id)
114
+ end
115
+ end
116
+
117
+ # Load shard info
118
+ def shard_info_by_id(shard_id, cache = true)
119
+ # Cleanup the cache if asked to
120
+ @shard_info_cache[shard_id] = nil unless cache
121
+
122
+ # Either load from cache or from db
123
+ @shard_info_cache[shard_id] ||= begin
124
+ prepare_shard_models
125
+ shard_class.find_by_id(shard_id)
126
+ end
127
+ end
128
+
129
+ def clear_shard_info_cache
130
+ @shard_info_cache = {}
131
+ end
132
+
133
+ def clear_group_info_cache
134
+ @group_info_cache = {}
135
+ end
136
+
137
+ # Load shard info using mapping info for a group
138
+ def shard_info_by_group_id(group_id)
139
+ # Load group
140
+ group_info = group_info_by_id(group_id)
141
+ raise ArgumentError, "Invalid group_id: #{group_id}" unless group_info
142
+
143
+ shard_info = shard_info_by_id(group_info.shard_id)
144
+ raise ArgumentError, "Invalid shard_id: #{group_info.shard_id}" unless shard_info
145
+
146
+ return shard_info
147
+ end
148
+
149
+ #---------------------------------------------------------------------------------------------------------------
150
+ def allocate_new_block_for_key(key)
151
+ # Can't find any groups to use for blocks allocation!
152
+ return nil unless group = least_loaded_group
153
+
154
+ # Figure out block limits
155
+ start_id = block_start_for_key(key)
156
+ end_id = block_end_for_key(key)
157
+
158
+ # Try to insert a new mapping (ignore duplicate key errors)
159
+ sql = <<-SQL
160
+ INSERT INTO #{map_table}
161
+ (start_id, end_id, group_id, block_size, created_at, updated_at) VALUES
162
+ (#{start_id}, #{end_id}, #{group.id}, #{block_size}, NOW(), NOW())
163
+ SQL
164
+ connection.execute(sql, "Allocate new block")
165
+
166
+ # Increment the blocks counter on the shard
167
+ group_class.update_counters(group.id, :blocks_count => +1)
168
+
169
+ # Retry block search after creation
170
+ block_for_key(key)
171
+ end
172
+
173
+ def least_loaded_group
174
+ prepare_shard_models
175
+
176
+ # Select group
177
+ group = group_class.first(:conditions => { :enabled => true, :open => true }, :order => 'blocks_count ASC')
178
+ raise "Can't find any tablegroups to use for blocks allocation!" unless group
179
+ return group
180
+ end
181
+
182
+ #---------------------------------------------------------------------------------------------------------------
183
+ def block_start_for_key(key)
184
+ block_size.to_i * (key.to_i / block_size.to_i)
185
+ end
186
+
187
+ def block_end_for_key(key)
188
+ block_size.to_i + block_start_for_key(key)
189
+ end
190
+
191
+ def group_class
192
+ if self.is_a?(DbCharmer::Sharding::Method::DbBlockGroupMap)
193
+ "DbCharmer::Sharding::Method::DbBlockGroupMap::Group".classify.constantize
194
+ elsif self.is_a?(DbCharmer::Sharding::Method::DbBlockSchemaMap)
195
+ "DbCharmer::Sharding::Method::DbBlockSchemaMap::Group".classify.constantize
196
+ end
197
+ end
198
+
199
+ def shard_class
200
+ if self.is_a?(DbCharmer::Sharding::Method::DbBlockGroupMap)
201
+ "DbCharmer::Sharding::Method::DbBlockGroupMap::Shard".classify.constantize
202
+ elsif self.is_a?(DbCharmer::Sharding::Method::DbBlockSchemaMap)
203
+ "DbCharmer::Sharding::Method::DbBlockSchemaMap::Shard".classify.constantize
204
+ end
205
+ end
206
+
207
+ def shard_connections
208
+ # Find all groups
209
+ prepare_shard_models
210
+ groups = group_class.all(:conditions => { :enabled => true }, :include => :shard)
211
+ # Map them to shards
212
+ groups.map { |group| shard_connection_config(group.shard, group.id) }
213
+ end
214
+
215
+ # Prepare model for working with our shards table
216
+ def prepare_shard_models
217
+ shard_class.switch_connection_to(connection)
218
+ shard_class.set_table_name(shards_table)
219
+
220
+ group_class.switch_connection_to(connection)
221
+ group_class.set_table_name(groups_table)
222
+ end
223
+
224
+ # This connections settings can be used to drop and create databases
225
+ def shard_connection_config_no_dbname(shard)
226
+ # Format connection name
227
+ connection_name = "db_charmer_db_block_group_map_#{name}_s%d_no_db" % shard.id
228
+ connection.instance_variable_get(:@config).clone.merge(
229
+ # Name for the connection factory
230
+ :connection_name => connection_name,
231
+ # Connection params
232
+ :host => shard.db_host,
233
+ :port => shard.db_port,
234
+ :username => shard.db_user,
235
+ :password => shard.db_pass,
236
+ :database => nil,
237
+ :schema_name => ''
238
+ )
239
+ end
240
+
241
+ def create_shard_database(shard)
242
+ conn_config = shard_connection_config_no_dbname(shard)
243
+ old_proxy = ::ActiveRecord::Base.db_charmer_connection_proxy
244
+ ::ActiveRecord::Base.switch_connection_to(conn_config)
245
+ sql = "SELECT datname FROM pg_database WHERE datname='#{shard.db_name}'"
246
+ existing_dbs = ::ActiveRecord::Base.connection.execute(sql)
247
+ unless existing_dbs.first
248
+ ::ActiveRecord::Base.connection.create_database(shard.db_name, conn_config)
249
+ end
250
+ ::ActiveRecord::Base.switch_connection_to(old_proxy)
251
+ end
252
+
253
+ def drop_shard_database(shard)
254
+ conn_config = shard_connection_config_no_dbname(shard)
255
+ old_proxy = ::ActiveRecord::Base.db_charmer_connection_proxy
256
+ ::ActiveRecord::Base.switch_connection_to(conn_config)
257
+ ::ActiveRecord::Base.connection.drop_database(shard.db_name)
258
+ ::ActiveRecord::Base.switch_connection_to(old_proxy)
259
+ end
260
+
261
+ def drop_all_shard_databases
262
+ prepare_shard_models
263
+ shard_class.all.each do |shard|
264
+ drop_shard_database(shard)
265
+ end
266
+ end
267
+ end
268
+ end
269
+ end
270
+ end