activerecord-turntable 2.5.0 → 3.0.0.alpha1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +7 -0
- data/.travis.yml +3 -7
- data/CHANGELOG.md +4 -14
- data/Gemfile +3 -0
- data/Guardfile +12 -7
- data/README.md +11 -19
- data/Rakefile +14 -15
- data/activerecord-turntable.gemspec +24 -27
- data/gemfiles/rails5_0.gemfile +6 -0
- data/lib/active_record/turntable/active_record_ext/abstract_adapter.rb +24 -20
- data/lib/active_record/turntable/active_record_ext/activerecord_import_ext.rb +6 -16
- data/lib/active_record/turntable/active_record_ext/acts_as_archive_extension.rb +25 -16
- data/lib/active_record/turntable/active_record_ext/association.rb +33 -14
- data/lib/active_record/turntable/active_record_ext/association_preloader.rb +4 -24
- data/lib/active_record/turntable/active_record_ext/clever_load.rb +2 -2
- data/lib/active_record/turntable/active_record_ext/connection_handler_extension.rb +11 -15
- data/lib/active_record/turntable/active_record_ext/database_tasks.rb +9 -9
- data/lib/active_record/turntable/active_record_ext/fixtures.rb +11 -41
- data/lib/active_record/turntable/active_record_ext/locking_optimistic.rb +40 -147
- data/lib/active_record/turntable/active_record_ext/log_subscriber.rb +6 -37
- data/lib/active_record/turntable/active_record_ext/migration_proxy.rb +1 -1
- data/lib/active_record/turntable/active_record_ext/persistence.rb +54 -148
- data/lib/active_record/turntable/active_record_ext/relation.rb +17 -45
- data/lib/active_record/turntable/active_record_ext/schema_dumper.rb +80 -78
- data/lib/active_record/turntable/active_record_ext/sequencer.rb +6 -15
- data/lib/active_record/turntable/active_record_ext/transactions.rb +14 -9
- data/lib/active_record/turntable/active_record_ext.rb +15 -16
- data/lib/active_record/turntable/algorithm/modulo_algorithm.rb +1 -1
- data/lib/active_record/turntable/algorithm/range_algorithm.rb +9 -9
- data/lib/active_record/turntable/algorithm/range_bsearch_algorithm.rb +12 -12
- data/lib/active_record/turntable/base.rb +27 -14
- data/lib/active_record/turntable/cluster.rb +10 -13
- data/lib/active_record/turntable/cluster_helper_methods.rb +6 -6
- data/lib/active_record/turntable/config.rb +3 -3
- data/lib/active_record/turntable/connection_proxy/mixable.rb +1 -1
- data/lib/active_record/turntable/connection_proxy.rb +23 -22
- data/lib/active_record/turntable/helpers/test_helper.rb +4 -4
- data/lib/active_record/turntable/master_shard.rb +12 -7
- data/lib/active_record/turntable/migration.rb +41 -47
- data/lib/active_record/turntable/mixer/fader/calculate_shards_sum_result.rb +7 -7
- data/lib/active_record/turntable/mixer/fader/select_shards_merge_result.rb +12 -12
- data/lib/active_record/turntable/mixer.rb +121 -121
- data/lib/active_record/turntable/plugin.rb +1 -1
- data/lib/active_record/turntable/pool_proxy.rb +7 -19
- data/lib/active_record/turntable/query_cache.rb +41 -0
- data/lib/active_record/turntable/railtie.rb +7 -5
- data/lib/active_record/turntable/railties/databases.rake +19 -20
- data/lib/active_record/turntable/seq_shard.rb +15 -15
- data/lib/active_record/turntable/sequencer/api.rb +7 -7
- data/lib/active_record/turntable/sequencer/barrage.rb +6 -7
- data/lib/active_record/turntable/sequencer/mysql.rb +2 -2
- data/lib/active_record/turntable/sequencer.rb +15 -15
- data/lib/active_record/turntable/shard.rb +23 -20
- data/lib/active_record/turntable/sql_tree_patch.rb +59 -57
- data/lib/active_record/turntable/util.rb +1 -21
- data/lib/active_record/turntable/version.rb +1 -1
- data/lib/active_record/turntable.rb +14 -19
- data/lib/activerecord-turntable.rb +2 -2
- data/script/performance/algorithm +8 -9
- data/spec/active_record/turntable/active_record_ext/association_preloader_spec.rb +4 -4
- data/spec/active_record/turntable/active_record_ext/association_spec.rb +5 -14
- data/spec/active_record/turntable/active_record_ext/clever_load_spec.rb +6 -6
- data/spec/active_record/turntable/active_record_ext/fixture_set_spec.rb +4 -4
- data/spec/active_record/turntable/active_record_ext/locking_optimistic_spec.rb +2 -15
- data/spec/active_record/turntable/active_record_ext/migration_spec.rb +4 -4
- data/spec/active_record/turntable/active_record_ext/persistence_spec.rb +17 -15
- data/spec/active_record/turntable/active_record_ext/sequencer_spec.rb +1 -1
- data/spec/active_record/turntable/active_record_ext/test_fixtures_spec.rb +3 -3
- data/spec/active_record/turntable/algorithm/modulo_algorithm_spec.rb +2 -3
- data/spec/active_record/turntable/algorithm/range_algorithm_spec.rb +2 -3
- data/spec/active_record/turntable/algorithm/range_bsearch_algorithm_spec.rb +2 -3
- data/spec/active_record/turntable/algorithm_spec.rb +5 -5
- data/spec/active_record/turntable/base_spec.rb +1 -1
- data/spec/active_record/turntable/cluster_spec.rb +1 -1
- data/spec/active_record/turntable/config_spec.rb +1 -1
- data/spec/active_record/turntable/connection_proxy_spec.rb +16 -17
- data/spec/active_record/turntable/finder_spec.rb +1 -1
- data/spec/active_record/turntable/mixer/fader_spec.rb +1 -1
- data/spec/active_record/turntable/mixer_spec.rb +2 -4
- data/spec/active_record/turntable/query_cache_spec.rb +28 -0
- data/spec/active_record/turntable/sequencer/api_spec.rb +4 -4
- data/spec/active_record/turntable/sequencer/barrage_spec.rb +2 -2
- data/spec/active_record/turntable/sequencer/mysql_spec.rb +1 -1
- data/spec/active_record/turntable/shard_spec.rb +1 -1
- data/spec/active_record/turntable/sql_tree_patch_spec.rb +2 -2
- data/spec/active_record/turntable/transaction_spec.rb +2 -2
- data/spec/active_record/turntable_spec.rb +3 -3
- data/spec/migrations/002_create_user_statuses.rb +4 -4
- data/spec/migrations/003_create_cards.rb +3 -3
- data/spec/migrations/004_create_cards_users.rb +2 -2
- data/spec/models/user_status.rb +0 -1
- data/spec/spec_helper.rb +15 -14
- data/spec/support/turntable_helper.rb +4 -4
- metadata +98 -59
- data/gemfiles/rails4_0.gemfile +0 -7
- data/gemfiles/rails4_1.gemfile +0 -7
- data/gemfiles/rails4_2.gemfile +0 -7
- data/lib/active_record/turntable/rack/connection_management.rb +0 -18
- data/lib/active_record/turntable/rack/query_cache.rb +0 -40
- data/lib/active_record/turntable/rack.rb +0 -8
- data/spec/active_record/turntable/rack/query_cache_spec.rb +0 -19
@@ -13,13 +13,13 @@ module ActiveRecord::Turntable
|
|
13
13
|
|
14
14
|
private
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
16
|
+
def merge_results(results)
|
17
|
+
ActiveRecord::Result.new(
|
18
|
+
results.first.columns,
|
19
|
+
results[0].rows.zip(*results[1..-1].map(&:rows)).map { |r| [r.map(&:first).inject(&:+)] },
|
20
|
+
results.first.column_types
|
21
|
+
)
|
22
|
+
end
|
23
23
|
end
|
24
24
|
end
|
25
25
|
end
|
@@ -13,20 +13,20 @@ module ActiveRecord::Turntable
|
|
13
13
|
|
14
14
|
private
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
16
|
+
def merge_results(results)
|
17
|
+
if results.any? { |r| r.is_a?(ActiveRecord::Result) }
|
18
|
+
first_result = results.find(&:present?)
|
19
|
+
return results.first unless first_result
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
21
|
+
ActiveRecord::Result.new(
|
22
|
+
first_result.columns,
|
23
|
+
results.flat_map(&:rows),
|
24
|
+
first_result.column_types
|
25
|
+
)
|
26
|
+
else
|
27
|
+
results.compact.inject(&:+)
|
28
|
+
end
|
28
29
|
end
|
29
|
-
end
|
30
30
|
end
|
31
31
|
end
|
32
32
|
end
|
@@ -1,5 +1,5 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "active_support/core_ext/object/try"
|
2
|
+
require "active_record/turntable/sql_tree_patch"
|
3
3
|
|
4
4
|
module ActiveRecord::Turntable
|
5
5
|
class Mixer
|
@@ -9,7 +9,7 @@ module ActiveRecord::Turntable
|
|
9
9
|
autoload :Fader
|
10
10
|
end
|
11
11
|
|
12
|
-
delegate :logger, :
|
12
|
+
delegate :logger, to: ActiveRecord::Base
|
13
13
|
|
14
14
|
NOT_USED_FOR_SHARDING_OPERATORS_REGEXP = /\A(NOT IN|IS|IS NOT|BETWEEN|LIKE|\!\=|<<|>>|<>|>\=|<=|[\*\+\-\/\%\|\&><])\z/
|
15
15
|
|
@@ -24,7 +24,7 @@ module ActiveRecord::Turntable
|
|
24
24
|
{ @proxy.fixed_shard => query },
|
25
25
|
method, query, *args, &block)
|
26
26
|
end
|
27
|
-
binds = (method ==
|
27
|
+
binds = (method == "insert") ? args[4] : args[1]
|
28
28
|
binded_query = bind_sql(query, binds)
|
29
29
|
|
30
30
|
begin
|
@@ -44,8 +44,8 @@ module ActiveRecord::Turntable
|
|
44
44
|
else
|
45
45
|
# send to master shard
|
46
46
|
Fader::SpecifiedShard.new(@proxy,
|
47
|
-
|
48
|
-
|
47
|
+
{ @proxy.master => query },
|
48
|
+
method, query, *args, &block)
|
49
49
|
end
|
50
50
|
rescue Exception => err
|
51
51
|
logger.warn { "[ActiveRecord::Turntable] Error on Building Fader: #{binded_query}, on_method: #{method_name}, err: #{err}" }
|
@@ -59,7 +59,7 @@ module ActiveRecord::Turntable
|
|
59
59
|
when "OR"
|
60
60
|
lkeys = find_shard_keys(tree.lhs, table_name, shard_key)
|
61
61
|
rkeys = find_shard_keys(tree.rhs, table_name, shard_key)
|
62
|
-
if lkeys.present?
|
62
|
+
if lkeys.present? && rkeys.present?
|
63
63
|
lkeys + rkeys
|
64
64
|
else
|
65
65
|
[]
|
@@ -67,7 +67,7 @@ module ActiveRecord::Turntable
|
|
67
67
|
when "AND"
|
68
68
|
lkeys = find_shard_keys(tree.lhs, table_name, shard_key)
|
69
69
|
rkeys = find_shard_keys(tree.rhs, table_name, shard_key)
|
70
|
-
if lkeys.present?
|
70
|
+
if lkeys.present? || rkeys.present?
|
71
71
|
lkeys + rkeys
|
72
72
|
else
|
73
73
|
[]
|
@@ -93,141 +93,141 @@ module ActiveRecord::Turntable
|
|
93
93
|
[]
|
94
94
|
else
|
95
95
|
raise ActiveRecord::Turntable::UnknownOperatorError,
|
96
|
-
|
96
|
+
"[ActiveRecord::Turntable] Found Unknown SQL Operator:'#{tree.operator if tree.respond_to?(:operaor)}', Please report this bug."
|
97
97
|
end
|
98
98
|
end
|
99
99
|
|
100
100
|
private
|
101
101
|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
102
|
+
def divide_insert_values(tree, shard_key_name)
|
103
|
+
idx = tree.fields.find_index { |f| f.name == shard_key_name.to_s }
|
104
|
+
result = {}
|
105
|
+
tree.values.each do |val|
|
106
|
+
(result[val[idx].value] ||= []) << val
|
107
|
+
end
|
108
|
+
result
|
107
109
|
end
|
108
|
-
return result
|
109
|
-
end
|
110
110
|
|
111
|
-
|
112
|
-
|
113
|
-
|
111
|
+
def build_shards_with_same_query(shards, query)
|
112
|
+
Hash[shards.map { |s| [s, query] }]
|
113
|
+
end
|
114
114
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
115
|
+
def bind_sql(sql, binds)
|
116
|
+
# TODO: substitution value should be determined by adapter
|
117
|
+
query = sql.is_a?(String) ? sql : @proxy.to_sql(sql, binds ? binds.dup : [])
|
118
|
+
query = if query.include?("\0") && binds.is_a?(Array) && binds[0].is_a?(Array) && binds[0][0].is_a?(ActiveRecord::ConnectionAdapters::Column)
|
119
|
+
binds = binds.dup
|
120
|
+
query.gsub("\0") { @proxy.master.connection.quote(*binds.shift.reverse) }
|
121
|
+
else
|
122
|
+
query
|
123
|
+
end
|
124
|
+
end
|
125
125
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
tree.select.first.to_sql == '1 AS "one"'
|
143
|
-
return Fader::SelectShardsMergeResult.new(@proxy,
|
144
|
-
build_shards_with_same_query(@proxy.shards.values, query),
|
145
|
-
method, query, *args, &block
|
146
|
-
)
|
147
|
-
elsif tree.group_by or tree.order_by or tree.limit.try(:value).to_i > 0
|
148
|
-
raise CannotSpecifyShardError, "cannot specify shard for query: #{tree.to_sql}"
|
149
|
-
elsif shard_keys.present?
|
150
|
-
if SQLTree::Node::SelectDeclaration === tree.select.first and
|
151
|
-
SQLTree::Node::CountAggregrate === tree.select.first.expression
|
152
|
-
return Fader::CalculateShardsSumResult.new(@proxy,
|
153
|
-
build_shards_with_same_query(@proxy.shards.values, query),
|
154
|
-
method, query, *args, &block)
|
155
|
-
else
|
126
|
+
def build_select_fader(tree, method, query, *args, &block)
|
127
|
+
shard_keys = if !tree.where && tree.from.size == 1 && SQLTree::Node::SubQuery === tree.from.first.table_reference.table
|
128
|
+
find_shard_keys(tree.from.first.table_reference.table.where,
|
129
|
+
@proxy.klass.table_name,
|
130
|
+
@proxy.klass.turntable_shard_key.to_s)
|
131
|
+
else
|
132
|
+
find_shard_keys(tree.where,
|
133
|
+
@proxy.klass.table_name,
|
134
|
+
@proxy.klass.turntable_shard_key.to_s)
|
135
|
+
end
|
136
|
+
|
137
|
+
if shard_keys.size == 1 # shard
|
138
|
+
return Fader::SpecifiedShard.new(@proxy,
|
139
|
+
{ @proxy.cluster.shard_for(shard_keys.first) => query },
|
140
|
+
method, query, *args, &block)
|
141
|
+
elsif SQLTree::Node::SelectDeclaration === tree.select.first &&
|
142
|
+
tree.select.first.to_sql == '1 AS "one"' # for `SELECT 1 AS one` (AR::Base.exists?)
|
156
143
|
return Fader::SelectShardsMergeResult.new(@proxy,
|
157
|
-
|
144
|
+
build_shards_with_same_query(@proxy.shards.values, query),
|
158
145
|
method, query, *args, &block
|
159
|
-
|
146
|
+
)
|
147
|
+
elsif tree.group_by || tree.order_by || tree.limit.try(:value).to_i > 0
|
148
|
+
raise CannotSpecifyShardError, "cannot specify shard for query: #{tree.to_sql}"
|
149
|
+
elsif shard_keys.present?
|
150
|
+
if SQLTree::Node::SelectDeclaration === tree.select.first &&
|
151
|
+
SQLTree::Node::CountAggregrate === tree.select.first.expression
|
152
|
+
return Fader::CalculateShardsSumResult.new(@proxy,
|
153
|
+
build_shards_with_same_query(@proxy.shards.values, query),
|
154
|
+
method, query, *args, &block)
|
155
|
+
else
|
156
|
+
return Fader::SelectShardsMergeResult.new(@proxy,
|
157
|
+
Hash[shard_keys.map { |k| [@proxy.cluster.shard_for(k), query] }],
|
158
|
+
method, query, *args, &block
|
159
|
+
)
|
160
|
+
end
|
161
|
+
else # scan all shards
|
162
|
+
if SQLTree::Node::SelectDeclaration === tree.select.first &&
|
163
|
+
SQLTree::Node::CountAggregrate === tree.select.first.expression
|
164
|
+
|
165
|
+
if raise_on_not_specified_shard_query?
|
166
|
+
raise CannotSpecifyShardError, "[Performance Notice] PLEASE FIX: #{tree.to_sql}"
|
167
|
+
end
|
168
|
+
return Fader::CalculateShardsSumResult.new(@proxy,
|
169
|
+
build_shards_with_same_query(@proxy.shards.values, query),
|
170
|
+
method, query, *args, &block)
|
171
|
+
elsif SQLTree::Node::AllFieldsDeclaration === tree.select.first ||
|
172
|
+
SQLTree::Node::Expression::Value === tree.select.first.expression ||
|
173
|
+
SQLTree::Node::Expression::Variable === tree.select.first.expression
|
174
|
+
|
175
|
+
if raise_on_not_specified_shard_query?
|
176
|
+
raise CannotSpecifyShardError, "[Performance Notice] PLEASE FIX: #{tree.to_sql}"
|
177
|
+
end
|
178
|
+
return Fader::SelectShardsMergeResult.new(@proxy,
|
179
|
+
build_shards_with_same_query(@proxy.shards.values, query),
|
180
|
+
method, query, *args, &block
|
181
|
+
)
|
182
|
+
else
|
183
|
+
raise CannotSpecifyShardError, "cannot specify shard for query: #{tree.to_sql}"
|
184
|
+
end
|
160
185
|
end
|
161
|
-
|
162
|
-
if SQLTree::Node::SelectDeclaration === tree.select.first and
|
163
|
-
SQLTree::Node::CountAggregrate === tree.select.first.expression
|
186
|
+
end
|
164
187
|
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
188
|
+
def build_update_fader(tree, method, query, *args, &block)
|
189
|
+
shard_keys = find_shard_keys(tree.where, @proxy.klass.table_name, @proxy.klass.turntable_shard_key.to_s)
|
190
|
+
shards_with_query = if shard_keys.present?
|
191
|
+
build_shards_with_same_query(shard_keys.map { |k| @proxy.cluster.shard_for(k) }, query)
|
192
|
+
else
|
193
|
+
build_shards_with_same_query(@proxy.shards.values, query)
|
194
|
+
end
|
195
|
+
|
196
|
+
if shards_with_query.size == 1
|
197
|
+
Fader::SpecifiedShard.new(@proxy,
|
198
|
+
shards_with_query,
|
199
|
+
method, query, *args, &block)
|
200
|
+
else
|
201
|
+
if raise_on_not_specified_shard_update?
|
176
202
|
raise CannotSpecifyShardError, "[Performance Notice] PLEASE FIX: #{tree.to_sql}"
|
177
203
|
end
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
)
|
182
|
-
else
|
183
|
-
raise CannotSpecifyShardError, "cannot specify shard for query: #{tree.to_sql}"
|
204
|
+
Fader::UpdateShardsMergeResult.new(@proxy,
|
205
|
+
shards_with_query,
|
206
|
+
method, query, *args, &block)
|
184
207
|
end
|
185
208
|
end
|
186
|
-
end
|
187
209
|
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
method, query, *args, &block)
|
200
|
-
else
|
201
|
-
if raise_on_not_specified_shard_update?
|
202
|
-
raise CannotSpecifyShardError, "[Performance Notice] PLEASE FIX: #{tree.to_sql}"
|
210
|
+
def build_insert_fader(tree, method, query, *args, &block)
|
211
|
+
values_hash = divide_insert_values(tree, @proxy.klass.turntable_shard_key)
|
212
|
+
shards_with_query = {}
|
213
|
+
values_hash.each do |k, vs|
|
214
|
+
tree.values = [[SQLTree::Node::Expression::Variable.new("\\0")]]
|
215
|
+
sql = tree.to_sql
|
216
|
+
value_sql = vs.map do |val|
|
217
|
+
"(#{val.map { |v| "#{v.escape}#{@proxy.connection.quote(v.value)}" }.join(', ')})"
|
218
|
+
end.join(", ")
|
219
|
+
sql.gsub!('("\0")') { value_sql }
|
220
|
+
shards_with_query[@proxy.cluster.shard_for(k)] = sql
|
203
221
|
end
|
204
|
-
Fader::
|
205
|
-
shards_with_query,
|
206
|
-
method, query, *args, &block)
|
222
|
+
Fader::InsertShardsMergeResult.new(@proxy, shards_with_query, method, query, *args, &block)
|
207
223
|
end
|
208
|
-
end
|
209
224
|
|
210
|
-
|
211
|
-
|
212
|
-
shards_with_query = {}
|
213
|
-
values_hash.each do |k,vs|
|
214
|
-
tree.values = [[SQLTree::Node::Expression::Variable.new("\\0")]]
|
215
|
-
sql = tree.to_sql
|
216
|
-
value_sql = vs.map do |val|
|
217
|
-
"(#{val.map { |v| "#{v.escape}#{@proxy.connection.quote(v.value)}" }.join(', ')})"
|
218
|
-
end.join(', ')
|
219
|
-
sql.gsub!('("\0")') { value_sql }
|
220
|
-
shards_with_query[@proxy.cluster.shard_for(k)] = sql
|
225
|
+
def raise_on_not_specified_shard_query?
|
226
|
+
ActiveRecord::Base.turntable_config[:raise_on_not_specified_shard_query]
|
221
227
|
end
|
222
|
-
Fader::InsertShardsMergeResult.new(@proxy, shards_with_query, method, query, *args, &block)
|
223
|
-
end
|
224
228
|
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
def raise_on_not_specified_shard_update?
|
230
|
-
ActiveRecord::Base.turntable_config[:raise_on_not_specified_shard_update]
|
231
|
-
end
|
229
|
+
def raise_on_not_specified_shard_update?
|
230
|
+
ActiveRecord::Base.turntable_config[:raise_on_not_specified_shard_update]
|
231
|
+
end
|
232
232
|
end
|
233
233
|
end
|
@@ -1,22 +1,10 @@
|
|
1
1
|
module ActiveRecord::Turntable
|
2
|
-
module ActiveRecordConnectionMethods
|
3
|
-
def self.included(base)
|
4
|
-
base.alias_method_chain :reload, :master
|
5
|
-
end
|
6
|
-
|
7
|
-
def reload_with_master(*args, &block)
|
8
|
-
connection.with_master { reload_without_master }
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
2
|
class PoolProxy
|
13
3
|
def initialize(proxy)
|
14
4
|
@proxy = proxy
|
15
5
|
end
|
16
6
|
|
17
|
-
|
18
|
-
@proxy
|
19
|
-
end
|
7
|
+
attr_reader :proxy
|
20
8
|
alias_method :connection, :proxy
|
21
9
|
|
22
10
|
def with_connection
|
@@ -24,7 +12,7 @@ module ActiveRecord::Turntable
|
|
24
12
|
end
|
25
13
|
|
26
14
|
delegate :connected?, :automatic_reconnect, :automatic_reconnect=, :checkout_timeout, :dead_connection_timeout,
|
27
|
-
|
15
|
+
:spec, :connections, :size, :reaper, :table_exists?, to: :proxy
|
28
16
|
|
29
17
|
%w(columns_hash column_defaults primary_keys).each do |name|
|
30
18
|
define_method(name.to_sym) do
|
@@ -39,12 +27,12 @@ module ActiveRecord::Turntable
|
|
39
27
|
end
|
40
28
|
|
41
29
|
%w(active_connection?).each do |name|
|
42
|
-
define_method(name.to_sym) do |*
|
30
|
+
define_method(name.to_sym) do |*_args|
|
43
31
|
@proxy.master.connection_pool.send(name.to_sym) ||
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
32
|
+
@proxy.seq.connection_pool.try(name.to_sym) if @proxy.respond_to?(:seq) ||
|
33
|
+
@proxy.shards.values.any? do |pool|
|
34
|
+
pool.connection_pool.send(name.to_sym)
|
35
|
+
end
|
48
36
|
end
|
49
37
|
end
|
50
38
|
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require "rack/body_proxy"
|
2
|
+
require "active_record/query_cache"
|
3
|
+
|
4
|
+
module ActiveRecord
|
5
|
+
module Turntable
|
6
|
+
class QueryCache < ActiveRecord::QueryCache
|
7
|
+
def self.run
|
8
|
+
klasses = ActiveRecord::Base.turntable_connections.values
|
9
|
+
enables = klasses.map do |k|
|
10
|
+
enabled = k.connection.query_cache_enabled
|
11
|
+
k.connection.enable_query_cache!
|
12
|
+
|
13
|
+
enabled
|
14
|
+
end
|
15
|
+
|
16
|
+
enables.all?
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.complete(enabled)
|
20
|
+
klasses = ActiveRecord::Base.turntable_connections.values
|
21
|
+
klasses.each do |k|
|
22
|
+
k.connection.clear_query_cache
|
23
|
+
k.connection.disable_query_cache! unless enabled
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.install_executor_hooks(executor = ActiveSupport::Executor)
|
28
|
+
executor.register_hook(self)
|
29
|
+
|
30
|
+
executor.to_complete do
|
31
|
+
klasses = ActiveRecord::Base.turntable_connection_classes
|
32
|
+
klasses.each do |k|
|
33
|
+
unless k.connected? && k.connection.transaction_open?
|
34
|
+
k.clear_active_connections!
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -1,14 +1,14 @@
|
|
1
1
|
module ActiveRecord::Turntable
|
2
2
|
class Railtie < Rails::Railtie
|
3
3
|
rake_tasks do
|
4
|
-
require
|
4
|
+
require "active_record/turntable/active_record_ext/database_tasks"
|
5
5
|
load "active_record/turntable/railties/databases.rake"
|
6
6
|
end
|
7
7
|
|
8
8
|
# rails loading hook
|
9
9
|
ActiveSupport.on_load(:before_initialize) do
|
10
10
|
ActiveSupport.on_load(:active_record) do
|
11
|
-
ActiveRecord::Base.
|
11
|
+
ActiveRecord::Base.include(ActiveRecord::Turntable)
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
@@ -23,9 +23,11 @@ module ActiveRecord::Turntable
|
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
-
# QueryCache
|
27
|
-
initializer "
|
28
|
-
|
26
|
+
# set QueryCache executor hooks for turntable clusters
|
27
|
+
initializer "active_record.set_executor_hooks" do
|
28
|
+
ActiveSupport.on_load(:active_record) do
|
29
|
+
ActiveRecord::Turntable::QueryCache.install_executor_hooks
|
30
|
+
end
|
29
31
|
end
|
30
32
|
end
|
31
33
|
end
|
@@ -1,5 +1,5 @@
|
|
1
|
-
require
|
2
|
-
ActiveRecord::SchemaDumper.
|
1
|
+
require "active_record/turntable"
|
2
|
+
ActiveRecord::SchemaDumper.prepend(ActiveRecord::Turntable::ActiveRecordExt::SchemaDumper)
|
3
3
|
|
4
4
|
turntable_namespace = nil
|
5
5
|
|
@@ -11,9 +11,9 @@ db_namespace = namespace :db do
|
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
-
desc
|
14
|
+
desc "Create current turntable databases config/database.yml for the current Rails.env"
|
15
15
|
task :create do
|
16
|
-
unless ENV[
|
16
|
+
unless ENV["DATABASE_URL"]
|
17
17
|
ActiveRecord::Tasks::DatabaseTasks.create_current_turntable_cluster
|
18
18
|
end
|
19
19
|
end
|
@@ -24,9 +24,9 @@ db_namespace = namespace :db do
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
-
desc
|
27
|
+
desc "Drops current turntable databases for the current Rails.env"
|
28
28
|
task :drop do
|
29
|
-
unless ENV[
|
29
|
+
unless ENV["DATABASE_URL"]
|
30
30
|
ActiveRecord::Tasks::DatabaseTasks.drop_current_turntable_cluster
|
31
31
|
end
|
32
32
|
end
|
@@ -34,14 +34,14 @@ db_namespace = namespace :db do
|
|
34
34
|
namespace :schema do
|
35
35
|
# TODO: implement schema:cache:xxxx
|
36
36
|
task :dump do
|
37
|
-
require
|
37
|
+
require "active_record/schema_dumper"
|
38
38
|
config = ActiveRecord::Base.configurations[Rails.env]
|
39
39
|
shard_configs = config["shards"]
|
40
40
|
shard_configs.merge!(config["seq"]) if config["seq"]
|
41
41
|
if shard_configs
|
42
42
|
shard_configs.each do |name, config|
|
43
43
|
next unless config["database"]
|
44
|
-
filename = ENV[
|
44
|
+
filename = ENV["SCHEMA"] || "#{Rails.root}/db/schema-#{name}.rb"
|
45
45
|
File.open(filename, "w:utf-8") do |file|
|
46
46
|
ActiveRecord::Base.establish_connection(config)
|
47
47
|
ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, file)
|
@@ -49,10 +49,10 @@ db_namespace = namespace :db do
|
|
49
49
|
end
|
50
50
|
end
|
51
51
|
ActiveRecord::Base.establish_connection(config)
|
52
|
-
turntable_namespace[
|
52
|
+
turntable_namespace["schema:dump"].reenable
|
53
53
|
end
|
54
54
|
|
55
|
-
desc
|
55
|
+
desc "Load a schema.rb file into the database"
|
56
56
|
task :load do
|
57
57
|
config = ActiveRecord::Base.configurations[Rails.env]
|
58
58
|
shard_configs = config["shards"]
|
@@ -61,11 +61,11 @@ db_namespace = namespace :db do
|
|
61
61
|
shard_configs.each do |name, config|
|
62
62
|
next unless config["database"]
|
63
63
|
ActiveRecord::Base.establish_connection(config)
|
64
|
-
file = ENV[
|
65
|
-
if File.
|
64
|
+
file = ENV["SCHEMA"] || "#{Rails.root}/db/schema-#{name}.rb"
|
65
|
+
if File.exist?(file)
|
66
66
|
load(file)
|
67
67
|
else
|
68
|
-
abort %
|
68
|
+
abort %(#{file} doesn't exist yet. Run "rake db:migrate" to create it then try again. If you do not intend to use a database, you should instead alter #{Rails.root}/config/application.rb to limit the frameworks that will be loaded')
|
69
69
|
end
|
70
70
|
end
|
71
71
|
end
|
@@ -74,7 +74,7 @@ db_namespace = namespace :db do
|
|
74
74
|
end
|
75
75
|
|
76
76
|
namespace :structure do
|
77
|
-
desc
|
77
|
+
desc "Dump the database structure to an SQL file"
|
78
78
|
task :dump do
|
79
79
|
current_config = ActiveRecord::Tasks::DatabaseTasks.current_config
|
80
80
|
shard_configs = current_config["shards"]
|
@@ -86,15 +86,14 @@ db_namespace = namespace :db do
|
|
86
86
|
filename = File.join(ActiveRecord::Tasks::DatabaseTasks.db_dir, "structure_#{name}.sql")
|
87
87
|
ActiveRecord::Tasks::DatabaseTasks.structure_dump(config, filename)
|
88
88
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
end
|
89
|
+
next unless ActiveRecord::Base.connection.supports_migrations?
|
90
|
+
File.open(filename, "a") do |f|
|
91
|
+
f.puts ActiveRecord::Base.connection.dump_schema_information
|
93
92
|
end
|
94
93
|
end
|
95
94
|
ActiveRecord::Base.establish_connection(current_config)
|
96
95
|
end
|
97
|
-
turntable_namespace[
|
96
|
+
turntable_namespace["structure:dump"].reenable
|
98
97
|
end
|
99
98
|
|
100
99
|
# desc "Recreate the databases from the structure.sql file"
|
@@ -121,7 +120,7 @@ db_namespace = namespace :db do
|
|
121
120
|
shard_configs = config["shards"]
|
122
121
|
shard_configs.merge!(config["seq"]) if config["seq"]
|
123
122
|
if shard_configs
|
124
|
-
shard_configs.each do |
|
123
|
+
shard_configs.each do |_name, config|
|
125
124
|
next unless config["database"]
|
126
125
|
ActiveRecord::Tasks::DatabaseTasks.purge config
|
127
126
|
end
|
@@ -2,20 +2,20 @@ module ActiveRecord::Turntable
|
|
2
2
|
class SeqShard < Shard
|
3
3
|
private
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
5
|
+
def create_connection_class
|
6
|
+
klass = get_or_set_connection_class
|
7
|
+
klass.remove_connection
|
8
|
+
klass.establish_connection ActiveRecord::Base.connection_pool.spec.config[:seq][name].with_indifferent_access
|
9
|
+
klass
|
10
|
+
end
|
11
|
+
|
12
|
+
def retrieve_connection_pool
|
13
|
+
ActiveRecord::Base.turntable_connections[name] ||=
|
14
|
+
begin
|
15
|
+
config = ActiveRecord::Base.configurations[Rails.env]["seq"][name]
|
16
|
+
raise ArgumentError, "Unknown database config: #{name}, have #{ActiveRecord::Base.configurations.inspect}" unless config
|
17
|
+
ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec_for(config))
|
18
|
+
end
|
19
|
+
end
|
20
20
|
end
|
21
21
|
end
|