activerecord-turntable 2.5.0 → 3.0.0.alpha1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +7 -0
- data/.travis.yml +3 -7
- data/CHANGELOG.md +4 -14
- data/Gemfile +3 -0
- data/Guardfile +12 -7
- data/README.md +11 -19
- data/Rakefile +14 -15
- data/activerecord-turntable.gemspec +24 -27
- data/gemfiles/rails5_0.gemfile +6 -0
- data/lib/active_record/turntable/active_record_ext/abstract_adapter.rb +24 -20
- data/lib/active_record/turntable/active_record_ext/activerecord_import_ext.rb +6 -16
- data/lib/active_record/turntable/active_record_ext/acts_as_archive_extension.rb +25 -16
- data/lib/active_record/turntable/active_record_ext/association.rb +33 -14
- data/lib/active_record/turntable/active_record_ext/association_preloader.rb +4 -24
- data/lib/active_record/turntable/active_record_ext/clever_load.rb +2 -2
- data/lib/active_record/turntable/active_record_ext/connection_handler_extension.rb +11 -15
- data/lib/active_record/turntable/active_record_ext/database_tasks.rb +9 -9
- data/lib/active_record/turntable/active_record_ext/fixtures.rb +11 -41
- data/lib/active_record/turntable/active_record_ext/locking_optimistic.rb +40 -147
- data/lib/active_record/turntable/active_record_ext/log_subscriber.rb +6 -37
- data/lib/active_record/turntable/active_record_ext/migration_proxy.rb +1 -1
- data/lib/active_record/turntable/active_record_ext/persistence.rb +54 -148
- data/lib/active_record/turntable/active_record_ext/relation.rb +17 -45
- data/lib/active_record/turntable/active_record_ext/schema_dumper.rb +80 -78
- data/lib/active_record/turntable/active_record_ext/sequencer.rb +6 -15
- data/lib/active_record/turntable/active_record_ext/transactions.rb +14 -9
- data/lib/active_record/turntable/active_record_ext.rb +15 -16
- data/lib/active_record/turntable/algorithm/modulo_algorithm.rb +1 -1
- data/lib/active_record/turntable/algorithm/range_algorithm.rb +9 -9
- data/lib/active_record/turntable/algorithm/range_bsearch_algorithm.rb +12 -12
- data/lib/active_record/turntable/base.rb +27 -14
- data/lib/active_record/turntable/cluster.rb +10 -13
- data/lib/active_record/turntable/cluster_helper_methods.rb +6 -6
- data/lib/active_record/turntable/config.rb +3 -3
- data/lib/active_record/turntable/connection_proxy/mixable.rb +1 -1
- data/lib/active_record/turntable/connection_proxy.rb +23 -22
- data/lib/active_record/turntable/helpers/test_helper.rb +4 -4
- data/lib/active_record/turntable/master_shard.rb +12 -7
- data/lib/active_record/turntable/migration.rb +41 -47
- data/lib/active_record/turntable/mixer/fader/calculate_shards_sum_result.rb +7 -7
- data/lib/active_record/turntable/mixer/fader/select_shards_merge_result.rb +12 -12
- data/lib/active_record/turntable/mixer.rb +121 -121
- data/lib/active_record/turntable/plugin.rb +1 -1
- data/lib/active_record/turntable/pool_proxy.rb +7 -19
- data/lib/active_record/turntable/query_cache.rb +41 -0
- data/lib/active_record/turntable/railtie.rb +7 -5
- data/lib/active_record/turntable/railties/databases.rake +19 -20
- data/lib/active_record/turntable/seq_shard.rb +15 -15
- data/lib/active_record/turntable/sequencer/api.rb +7 -7
- data/lib/active_record/turntable/sequencer/barrage.rb +6 -7
- data/lib/active_record/turntable/sequencer/mysql.rb +2 -2
- data/lib/active_record/turntable/sequencer.rb +15 -15
- data/lib/active_record/turntable/shard.rb +23 -20
- data/lib/active_record/turntable/sql_tree_patch.rb +59 -57
- data/lib/active_record/turntable/util.rb +1 -21
- data/lib/active_record/turntable/version.rb +1 -1
- data/lib/active_record/turntable.rb +14 -19
- data/lib/activerecord-turntable.rb +2 -2
- data/script/performance/algorithm +8 -9
- data/spec/active_record/turntable/active_record_ext/association_preloader_spec.rb +4 -4
- data/spec/active_record/turntable/active_record_ext/association_spec.rb +5 -14
- data/spec/active_record/turntable/active_record_ext/clever_load_spec.rb +6 -6
- data/spec/active_record/turntable/active_record_ext/fixture_set_spec.rb +4 -4
- data/spec/active_record/turntable/active_record_ext/locking_optimistic_spec.rb +2 -15
- data/spec/active_record/turntable/active_record_ext/migration_spec.rb +4 -4
- data/spec/active_record/turntable/active_record_ext/persistence_spec.rb +17 -15
- data/spec/active_record/turntable/active_record_ext/sequencer_spec.rb +1 -1
- data/spec/active_record/turntable/active_record_ext/test_fixtures_spec.rb +3 -3
- data/spec/active_record/turntable/algorithm/modulo_algorithm_spec.rb +2 -3
- data/spec/active_record/turntable/algorithm/range_algorithm_spec.rb +2 -3
- data/spec/active_record/turntable/algorithm/range_bsearch_algorithm_spec.rb +2 -3
- data/spec/active_record/turntable/algorithm_spec.rb +5 -5
- data/spec/active_record/turntable/base_spec.rb +1 -1
- data/spec/active_record/turntable/cluster_spec.rb +1 -1
- data/spec/active_record/turntable/config_spec.rb +1 -1
- data/spec/active_record/turntable/connection_proxy_spec.rb +16 -17
- data/spec/active_record/turntable/finder_spec.rb +1 -1
- data/spec/active_record/turntable/mixer/fader_spec.rb +1 -1
- data/spec/active_record/turntable/mixer_spec.rb +2 -4
- data/spec/active_record/turntable/query_cache_spec.rb +28 -0
- data/spec/active_record/turntable/sequencer/api_spec.rb +4 -4
- data/spec/active_record/turntable/sequencer/barrage_spec.rb +2 -2
- data/spec/active_record/turntable/sequencer/mysql_spec.rb +1 -1
- data/spec/active_record/turntable/shard_spec.rb +1 -1
- data/spec/active_record/turntable/sql_tree_patch_spec.rb +2 -2
- data/spec/active_record/turntable/transaction_spec.rb +2 -2
- data/spec/active_record/turntable_spec.rb +3 -3
- data/spec/migrations/002_create_user_statuses.rb +4 -4
- data/spec/migrations/003_create_cards.rb +3 -3
- data/spec/migrations/004_create_cards_users.rb +2 -2
- data/spec/models/user_status.rb +0 -1
- data/spec/spec_helper.rb +15 -14
- data/spec/support/turntable_helper.rb +4 -4
- metadata +98 -59
- data/gemfiles/rails4_0.gemfile +0 -7
- data/gemfiles/rails4_1.gemfile +0 -7
- data/gemfiles/rails4_2.gemfile +0 -7
- data/lib/active_record/turntable/rack/connection_management.rb +0 -18
- data/lib/active_record/turntable/rack/query_cache.rb +0 -40
- data/lib/active_record/turntable/rack.rb +0 -8
- data/spec/active_record/turntable/rack/query_cache_spec.rb +0 -19
@@ -7,31 +7,31 @@ module ActiveRecord::Turntable::Algorithm
|
|
7
7
|
|
8
8
|
def calculate(key)
|
9
9
|
idx = calculate_idx(key)
|
10
|
-
@config[
|
10
|
+
@config[:shards][idx][:connection]
|
11
11
|
rescue
|
12
12
|
raise ActiveRecord::Turntable::CannotSpecifyShardError, "cannot specify shard for key:#{key}"
|
13
13
|
end
|
14
14
|
|
15
15
|
def calculate_idx(key)
|
16
|
-
@config[
|
16
|
+
@config[:shards].find_index { |h| h[:less_than] > key }
|
17
17
|
end
|
18
18
|
|
19
19
|
# { connection_name => weight, ... }
|
20
20
|
def calculate_used_shards_with_weight(sequence_value)
|
21
21
|
idx = calculate_idx(sequence_value)
|
22
22
|
last_connection = calculate(sequence_value)
|
23
|
-
shards = @config[
|
24
|
-
weighted_hash = Hash.new {|h,k| h[k]=0}
|
23
|
+
shards = @config[:shards][0..idx]
|
24
|
+
weighted_hash = Hash.new { |h, k| h[k] = 0 }
|
25
25
|
prev_max = 0
|
26
|
-
shards.each_with_index do |h,idx|
|
27
|
-
weighted_hash[h[
|
28
|
-
h[
|
26
|
+
shards.each_with_index do |h, idx|
|
27
|
+
weighted_hash[h[:connection]] += if idx < shards.size - 1
|
28
|
+
h[:less_than] - prev_max - 1
|
29
29
|
else
|
30
30
|
sequence_value - prev_max
|
31
31
|
end
|
32
|
-
prev_max = h[
|
32
|
+
prev_max = h[:less_than] - 1
|
33
33
|
end
|
34
|
-
|
34
|
+
weighted_hash
|
35
35
|
end
|
36
36
|
end
|
37
37
|
end
|
@@ -1,22 +1,22 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
|
-
require
|
2
|
+
require "bsearch"
|
3
3
|
module ActiveRecord::Turntable::Algorithm
|
4
4
|
class RangeBsearchAlgorithm < Base
|
5
5
|
def initialize(config)
|
6
6
|
@config = config
|
7
|
-
@config[
|
7
|
+
@config[:shards].sort_by! { |a| a[:less_than] }
|
8
8
|
end
|
9
9
|
|
10
10
|
def calculate(key)
|
11
11
|
idx = calculate_idx(key)
|
12
|
-
@config[
|
12
|
+
@config[:shards][idx][:connection]
|
13
13
|
rescue
|
14
14
|
raise ActiveRecord::Turntable::CannotSpecifyShardError, "cannot specify shard for key:#{key}"
|
15
15
|
end
|
16
16
|
|
17
17
|
def calculate_idx(key)
|
18
|
-
@config[
|
19
|
-
h[
|
18
|
+
@config[:shards].bsearch_upper_boundary { |h|
|
19
|
+
h[:less_than] <=> key
|
20
20
|
}
|
21
21
|
end
|
22
22
|
|
@@ -24,18 +24,18 @@ module ActiveRecord::Turntable::Algorithm
|
|
24
24
|
def calculate_used_shards_with_weight(sequence_value)
|
25
25
|
idx = calculate_idx(sequence_value)
|
26
26
|
last_connection = calculate(sequence_value)
|
27
|
-
shards = @config[
|
28
|
-
weighted_hash = Hash.new {|h,k| h[k]=0}
|
27
|
+
shards = @config[:shards][0..idx]
|
28
|
+
weighted_hash = Hash.new { |h, k| h[k] = 0 }
|
29
29
|
prev_max = 0
|
30
|
-
shards.each_with_index do |h,idx|
|
31
|
-
weighted_hash[h[
|
32
|
-
h[
|
30
|
+
shards.each_with_index do |h, idx|
|
31
|
+
weighted_hash[h[:connection]] += if idx < shards.size - 1
|
32
|
+
h[:less_than] - prev_max - 1
|
33
33
|
else
|
34
34
|
sequence_value - prev_max
|
35
35
|
end
|
36
|
-
prev_max = h[
|
36
|
+
prev_max = h[:less_than] - 1
|
37
37
|
end
|
38
|
-
|
38
|
+
weighted_hash
|
39
39
|
end
|
40
40
|
end
|
41
41
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "active_support/lazy_load_hooks"
|
2
2
|
|
3
3
|
module ActiveRecord::Turntable
|
4
4
|
module Base
|
@@ -6,14 +6,14 @@ module ActiveRecord::Turntable
|
|
6
6
|
|
7
7
|
included do
|
8
8
|
class_attribute :turntable_connections, :turntable_clusters,
|
9
|
-
|
9
|
+
:turntable_enabled, :turntable_sequencer_enabled
|
10
10
|
|
11
11
|
self.turntable_connections = {}
|
12
12
|
self.turntable_clusters = {}.with_indifferent_access
|
13
13
|
self.turntable_enabled = false
|
14
14
|
self.turntable_sequencer_enabled = false
|
15
15
|
class << self
|
16
|
-
delegate :shards_transaction, :with_all, :
|
16
|
+
delegate :shards_transaction, :with_all, to: :connection
|
17
17
|
end
|
18
18
|
|
19
19
|
ActiveSupport.on_load(:turntable_config_loaded) do
|
@@ -22,37 +22,52 @@ module ActiveRecord::Turntable
|
|
22
22
|
include ClusterHelperMethods
|
23
23
|
end
|
24
24
|
|
25
|
+
module ConnectionExtension
|
26
|
+
module ClassMethods
|
27
|
+
def connection_specification_name
|
28
|
+
return super unless turntable_enabled?
|
29
|
+
self.connection_specification_name = "turntable_pool_proxy::#{name}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.prepended(base)
|
34
|
+
class << base
|
35
|
+
prepend ClassMethods
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
25
40
|
module ClassMethods
|
26
41
|
# @param [Symbol] cluster_name cluster name for this class
|
27
42
|
# @param [Symbol] shard_key_name shard key attribute name
|
28
43
|
# @param [Hash] options
|
29
44
|
def turntable(cluster_name, shard_key_name, options = {})
|
30
45
|
class_attribute :turntable_shard_key,
|
31
|
-
|
46
|
+
:turntable_cluster, :turntable_cluster_name
|
32
47
|
|
33
48
|
self.turntable_enabled = true
|
34
49
|
self.turntable_cluster_name = cluster_name
|
35
50
|
self.turntable_shard_key = shard_key_name
|
36
51
|
self.turntable_cluster =
|
37
52
|
self.turntable_clusters[cluster_name] ||= Cluster.new(
|
38
|
-
|
39
|
-
|
40
|
-
|
53
|
+
turntable_config[:clusters][cluster_name],
|
54
|
+
options
|
55
|
+
)
|
56
|
+
prepend ConnectionExtension
|
41
57
|
turntable_replace_connection_pool
|
42
58
|
end
|
43
59
|
|
44
|
-
|
45
60
|
def turntable_replace_connection_pool
|
46
61
|
ch = connection_handler
|
47
62
|
cp = ConnectionProxy.new(self, turntable_cluster)
|
48
63
|
pp = PoolProxy.new(cp)
|
49
|
-
|
50
|
-
ch.send(:
|
64
|
+
|
65
|
+
ch.send(:owner_to_pool)[connection_specification_name] = pp
|
51
66
|
end
|
52
67
|
|
53
68
|
def initialize_clusters!
|
54
69
|
turntable_config[:clusters].each do |name, spec|
|
55
|
-
self.turntable_clusters[name] ||= Cluster.new(spec
|
70
|
+
self.turntable_clusters[name] ||= Cluster.new(spec)
|
56
71
|
end
|
57
72
|
end
|
58
73
|
|
@@ -67,9 +82,7 @@ module ActiveRecord::Turntable
|
|
67
82
|
end
|
68
83
|
|
69
84
|
def clear_all_connections!
|
70
|
-
turntable_connections.values.each
|
71
|
-
pool.disconnect!
|
72
|
-
end
|
85
|
+
turntable_connections.values.each(&:disconnect!)
|
73
86
|
end
|
74
87
|
|
75
88
|
def sequencer(sequence_name, *args)
|
@@ -1,8 +1,7 @@
|
|
1
|
-
require
|
1
|
+
require "active_support/core_ext/hash/indifferent_access"
|
2
2
|
|
3
3
|
module ActiveRecord::Turntable
|
4
4
|
class Cluster
|
5
|
-
|
6
5
|
DEFAULT_CONFIG = {
|
7
6
|
"shards" => [],
|
8
7
|
"algorithm" => "range",
|
@@ -15,19 +14,19 @@ module ActiveRecord::Turntable
|
|
15
14
|
|
16
15
|
# setup sequencer
|
17
16
|
seq = (@options[:seq] || @config[:seq])
|
18
|
-
if seq
|
19
|
-
if seq.values.size > 0 && seq.values.first[
|
17
|
+
if seq
|
18
|
+
if seq.values.size > 0 && seq.values.first[:seq_type] == "mysql"
|
20
19
|
@seq_shard = SeqShard.new(seq.values.first)
|
21
20
|
end
|
22
21
|
end
|
23
22
|
|
24
23
|
# setup shards
|
25
24
|
@config[:shards].each do |spec|
|
26
|
-
@shards[spec[
|
25
|
+
@shards[spec[:connection]] ||= Shard.new(spec)
|
27
26
|
end
|
28
27
|
|
29
28
|
# setup algorithm
|
30
|
-
alg_name = "ActiveRecord::Turntable::Algorithm::#{@config[
|
29
|
+
alg_name = "ActiveRecord::Turntable::Algorithm::#{@config[:algorithm].camelize}Algorithm"
|
31
30
|
@algorithm = alg_name.constantize.new(@config)
|
32
31
|
end
|
33
32
|
|
@@ -35,15 +34,13 @@ module ActiveRecord::Turntable
|
|
35
34
|
@seq_shard
|
36
35
|
end
|
37
36
|
|
38
|
-
|
39
|
-
@shards
|
40
|
-
end
|
37
|
+
attr_reader :shards
|
41
38
|
|
42
39
|
def shard_for(key)
|
43
40
|
@shards[@algorithm.calculate(key)]
|
44
41
|
rescue
|
45
42
|
raise ActiveRecord::Turntable::CannotSpecifyShardError,
|
46
|
-
|
43
|
+
"cannot select_shard for key:#{key}"
|
47
44
|
end
|
48
45
|
|
49
46
|
def select_shard(key)
|
@@ -65,7 +62,7 @@ module ActiveRecord::Turntable
|
|
65
62
|
end
|
66
63
|
else
|
67
64
|
shard.connection.transaction(options) do
|
68
|
-
|
65
|
+
yield
|
69
66
|
end
|
70
67
|
end
|
71
68
|
end
|
@@ -82,12 +79,12 @@ module ActiveRecord::Turntable
|
|
82
79
|
shards[shard_or_object]
|
83
80
|
else
|
84
81
|
raise ActiveRecord::Turntable::TurntableError,
|
85
|
-
|
82
|
+
"transaction cannot call to object: #{shard_or_object}"
|
86
83
|
end
|
87
84
|
end
|
88
85
|
|
89
86
|
def weighted_shards(key = nil)
|
90
|
-
Hash[@algorithm.calculate_used_shards_with_weight(key).map do |k,v|
|
87
|
+
Hash[@algorithm.calculate_used_shards_with_weight(key).map do |k, v|
|
91
88
|
[@shards[k], v]
|
92
89
|
end]
|
93
90
|
end
|
@@ -4,14 +4,14 @@ module ActiveRecord::Turntable
|
|
4
4
|
|
5
5
|
included do
|
6
6
|
ActiveSupport.on_load(:turntable_config_loaded) do
|
7
|
-
turntable_clusters.each do |name,
|
7
|
+
turntable_clusters.each do |name, _cluster|
|
8
8
|
turntable_define_cluster_methods(name)
|
9
9
|
end
|
10
10
|
end
|
11
11
|
end
|
12
12
|
|
13
13
|
module ClassMethods
|
14
|
-
def force_transaction_all_shards!(options={}, &block)
|
14
|
+
def force_transaction_all_shards!(options = {}, &block)
|
15
15
|
force_connect_all_shards!
|
16
16
|
shards = turntable_connections.values
|
17
17
|
shards += [ActiveRecord::Base.connection_pool]
|
@@ -31,9 +31,9 @@ module ActiveRecord::Turntable
|
|
31
31
|
|
32
32
|
def force_connect_all_shards!
|
33
33
|
conf = configurations[Rails.env]
|
34
|
-
shards =
|
35
|
-
shards = shards.merge(conf[
|
36
|
-
shards = shards.merge(conf[
|
34
|
+
shards = HashWithIndifferentAccess.new
|
35
|
+
shards = shards.merge(conf[:shards]) if conf[:shards]
|
36
|
+
shards = shards.merge(conf[:seq]) if conf[:seq]
|
37
37
|
shards.each do |name, config|
|
38
38
|
turntable_connections[name] ||=
|
39
39
|
ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec_for(config))
|
@@ -44,7 +44,7 @@ module ActiveRecord::Turntable
|
|
44
44
|
shards_weight = self.turntable_cluster.weighted_shards(self.current_sequence)
|
45
45
|
sum = shards_weight.values.inject(&:+)
|
46
46
|
idx = rand(sum)
|
47
|
-
shard, weight = shards_weight.find {|
|
47
|
+
shard, weight = shards_weight.find {|_k, v|
|
48
48
|
(idx -= v) < 0
|
49
49
|
}
|
50
50
|
self.connection.with_recursive_shards(shard.name, *klasses, &block)
|
@@ -1,5 +1,5 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "active_support/lazy_load_hooks"
|
2
|
+
require "active_support/core_ext/hash/indifferent_access"
|
3
3
|
|
4
4
|
module ActiveRecord::Turntable
|
5
5
|
class Config
|
@@ -14,7 +14,7 @@ module ActiveRecord::Turntable
|
|
14
14
|
@config[key]
|
15
15
|
end
|
16
16
|
|
17
|
-
def self.load!(config_file = ActiveRecord::Base.turntable_config_file, env = (defined?(Rails) ? Rails.env :
|
17
|
+
def self.load!(config_file = ActiveRecord::Base.turntable_config_file, env = (defined?(Rails) ? Rails.env : "development"))
|
18
18
|
instance.load!(config_file, env)
|
19
19
|
end
|
20
20
|
|
@@ -10,7 +10,7 @@ module ActiveRecord::Turntable
|
|
10
10
|
def mixable?(method, *args)
|
11
11
|
(method.to_s =~ METHODS_REGEXP &&
|
12
12
|
args.first !~ EXCLUDE_QUERY_REGEXP) ||
|
13
|
-
(method.to_s ==
|
13
|
+
(method.to_s == "execute" && args.first =~ QUERY_REGEXP)
|
14
14
|
end
|
15
15
|
end
|
16
16
|
end
|
@@ -1,10 +1,10 @@
|
|
1
|
-
require
|
1
|
+
require "active_record/turntable/connection_proxy/mixable"
|
2
2
|
module ActiveRecord::Turntable
|
3
3
|
class ConnectionProxy
|
4
4
|
include Mixable
|
5
5
|
|
6
6
|
# for expiring query cache
|
7
|
-
CLEAR_CACHE_METHODS = [:update, :insert, :delete, :exec_insert, :exec_update, :exec_delete, :insert_many]
|
7
|
+
CLEAR_CACHE_METHODS = [:update, :insert, :delete, :exec_insert, :exec_update, :exec_delete, :insert_many].freeze
|
8
8
|
|
9
9
|
attr_reader :klass
|
10
10
|
attr_writer :spec
|
@@ -22,9 +22,9 @@ module ActiveRecord::Turntable
|
|
22
22
|
delegate :shards_transaction, to: :cluster
|
23
23
|
|
24
24
|
delegate :create_table, :rename_table, :drop_table, :add_column, :remove_colomn,
|
25
|
-
|
26
|
-
|
27
|
-
|
25
|
+
:change_column, :change_column_default, :rename_column, :add_index,
|
26
|
+
:remove_index, :initialize_schema_information,
|
27
|
+
:dump_schema_information, :execute_ignore_duplicate, to: :master_connection
|
28
28
|
|
29
29
|
def transaction(options = {}, &block)
|
30
30
|
connection.transaction(options, &block)
|
@@ -38,7 +38,7 @@ module ActiveRecord::Turntable
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def enable_query_cache!
|
41
|
-
klass.turntable_connections.each do |
|
41
|
+
klass.turntable_connections.each do |_k, v|
|
42
42
|
v.connection.enable_query_cache!
|
43
43
|
end
|
44
44
|
end
|
@@ -48,7 +48,7 @@ module ActiveRecord::Turntable
|
|
48
48
|
end
|
49
49
|
|
50
50
|
def clear_query_cache
|
51
|
-
klass.turntable_connections.each do |
|
51
|
+
klass.turntable_connections.each do |_k, v|
|
52
52
|
v.connection.clear_query_cache
|
53
53
|
end
|
54
54
|
end
|
@@ -59,9 +59,11 @@ module ActiveRecord::Turntable
|
|
59
59
|
connection.send(method, *args, &block)
|
60
60
|
elsif mixable?(method, *args)
|
61
61
|
fader = @mixer.build_fader(method, *args, &block)
|
62
|
-
logger.debug {
|
63
|
-
"
|
64
|
-
"
|
62
|
+
logger.debug {
|
63
|
+
"[ActiveRecord::Turntable] Sending method: #{method}, " \
|
64
|
+
"sql: #{args.first}, " \
|
65
|
+
"shards: #{fader.shards_query_hash.keys.map(&:name)}"
|
66
|
+
}
|
65
67
|
fader.execute
|
66
68
|
else
|
67
69
|
connection.send(method, *args, &block)
|
@@ -76,9 +78,7 @@ module ActiveRecord::Turntable
|
|
76
78
|
master.connection.to_sql(arel, binds)
|
77
79
|
end
|
78
80
|
|
79
|
-
|
80
|
-
@cluster
|
81
|
-
end
|
81
|
+
attr_reader :cluster
|
82
82
|
|
83
83
|
def shards
|
84
84
|
@cluster.shards
|
@@ -113,7 +113,7 @@ module ActiveRecord::Turntable
|
|
113
113
|
end
|
114
114
|
|
115
115
|
def current_shard=(shard)
|
116
|
-
logger.debug { "Changing #{klass}'s shard to #{shard.name}"}
|
116
|
+
logger.debug { "Changing #{klass}'s shard to #{shard.name}" }
|
117
117
|
current_shard_entry[object_id] = shard
|
118
118
|
end
|
119
119
|
|
@@ -136,7 +136,8 @@ module ActiveRecord::Turntable
|
|
136
136
|
def with_shard(shard)
|
137
137
|
shard = cluster.to_shard(shard)
|
138
138
|
|
139
|
-
old_shard
|
139
|
+
old_shard = current_shard
|
140
|
+
old_fixed = fixed_shard
|
140
141
|
self.current_shard = shard
|
141
142
|
self.fixed_shard = shard
|
142
143
|
yield
|
@@ -197,7 +198,7 @@ module ActiveRecord::Turntable
|
|
197
198
|
end
|
198
199
|
|
199
200
|
delegate :connected?, :automatic_reconnect, :automatic_reconnect=, :checkout_timeout, :dead_connection_timeout,
|
200
|
-
|
201
|
+
:spec, :connections, :size, :reaper, :table_exists?, to: :connection_pool
|
201
202
|
|
202
203
|
%w(columns columns_hash column_defaults primary_keys).each do |name|
|
203
204
|
define_method(name.to_sym) do
|
@@ -239,12 +240,12 @@ module ActiveRecord::Turntable
|
|
239
240
|
|
240
241
|
private
|
241
242
|
|
242
|
-
|
243
|
-
|
244
|
-
|
243
|
+
def fixed_shard_entry
|
244
|
+
Thread.current[:turntable_fixed_shard] ||= ThreadSafe::Cache.new
|
245
|
+
end
|
245
246
|
|
246
|
-
|
247
|
-
|
248
|
-
|
247
|
+
def current_shard_entry
|
248
|
+
Thread.current[:turntable_current_shard] ||= ThreadSafe::Cache.new
|
249
|
+
end
|
249
250
|
end
|
250
251
|
end
|
@@ -2,15 +2,15 @@ module ActiveRecord::Turntable
|
|
2
2
|
module Helpers
|
3
3
|
module TestHelper
|
4
4
|
# all shards
|
5
|
-
def FabricateAll(name, overrides={}, &block)
|
5
|
+
def FabricateAll(name, overrides = {}, &block)
|
6
6
|
obj = Fabrication::Fabricator.generate(name, {
|
7
|
-
|
8
|
-
|
7
|
+
save: true,
|
8
|
+
}, overrides, &block)
|
9
9
|
|
10
10
|
default_pool = obj.class.connection_pool
|
11
11
|
connection_pools = obj.class.connection_handler.instance_variable_get(:@connection_pools)
|
12
12
|
|
13
|
-
ActiveRecord::Base.turntable_connections.each do |
|
13
|
+
ActiveRecord::Base.turntable_connections.each do |_conn_name, conn|
|
14
14
|
new_obj = obj.dup
|
15
15
|
connection_pools[new_obj.class.name] = conn
|
16
16
|
new_obj.id = obj.id
|
@@ -1,21 +1,27 @@
|
|
1
1
|
module ActiveRecord::Turntable
|
2
2
|
class MasterShard < Shard
|
3
3
|
def initialize(klass)
|
4
|
-
(klass and klass
|
4
|
+
(klass and original_connection_pool(klass)) or
|
5
5
|
raise MasterShardNotConnected, "connection_pool is nil"
|
6
6
|
@klass = klass
|
7
|
-
@name =
|
7
|
+
@name = "master"
|
8
8
|
end
|
9
9
|
|
10
10
|
def connection_pool
|
11
11
|
if ActiveRecord::Base == @klass
|
12
12
|
ActiveRecord::Base.connection_pool
|
13
13
|
else
|
14
|
-
# use
|
15
|
-
|
14
|
+
# use original parent class connection which is turntable disabled
|
15
|
+
original_connection_pool
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def original_connection_pool(klass = @klass)
|
16
22
|
candidate_connection_pool = nil
|
17
|
-
|
18
|
-
if klass == ActiveRecord::Base
|
23
|
+
until candidate_connection_pool
|
24
|
+
if klass == ActiveRecord::Base || !klass.turntable_enabled?
|
19
25
|
candidate_connection_pool = klass.connection_pool
|
20
26
|
else
|
21
27
|
klass = klass.superclass
|
@@ -23,6 +29,5 @@ module ActiveRecord::Turntable
|
|
23
29
|
end
|
24
30
|
candidate_connection_pool
|
25
31
|
end
|
26
|
-
end
|
27
32
|
end
|
28
33
|
end
|
@@ -3,12 +3,11 @@ module ActiveRecord::Turntable::Migration
|
|
3
3
|
|
4
4
|
included do
|
5
5
|
extend ShardDefinition
|
6
|
+
prepend OverrideMethods
|
6
7
|
class_attribute :target_shards, :current_shard
|
7
|
-
|
8
|
-
|
9
|
-
::ActiveRecord::
|
10
|
-
::ActiveRecord::Migration::CommandRecorder.send(:include, CommandRecorder)
|
11
|
-
::ActiveRecord::Migrator.send(:include, Migrator)
|
8
|
+
::ActiveRecord::ConnectionAdapters::AbstractAdapter.include(SchemaStatementsExt)
|
9
|
+
::ActiveRecord::Migration::CommandRecorder.include(CommandRecorder)
|
10
|
+
::ActiveRecord::Migrator.prepend(Migrator)
|
12
11
|
end
|
13
12
|
|
14
13
|
module ShardDefinition
|
@@ -16,13 +15,13 @@ module ActiveRecord::Turntable::Migration
|
|
16
15
|
config = ActiveRecord::Base.turntable_config
|
17
16
|
(self.target_shards ||= []).concat(
|
18
17
|
if cluster_names.first == :all
|
19
|
-
config[
|
20
|
-
cluster_conf[
|
18
|
+
config[:clusters].map do |_name, cluster_conf|
|
19
|
+
cluster_conf[:shards].map { |shard| shard[:connection] }
|
21
20
|
end
|
22
21
|
else
|
23
22
|
cluster_names.map do |cluster_name|
|
24
|
-
config[
|
25
|
-
shard[
|
23
|
+
config[:clusters][cluster_name][:shards].map do |shard|
|
24
|
+
shard[:connection]
|
26
25
|
end
|
27
26
|
end.flatten
|
28
27
|
end
|
@@ -34,32 +33,34 @@ module ActiveRecord::Turntable::Migration
|
|
34
33
|
end
|
35
34
|
end
|
36
35
|
|
37
|
-
|
38
|
-
|
39
|
-
|
36
|
+
module OverrideMethods
|
37
|
+
def announce(message)
|
38
|
+
super("#{message} - Shard: #{current_shard}")
|
39
|
+
end
|
40
40
|
|
41
|
-
|
42
|
-
|
43
|
-
|
41
|
+
def exec_migration(*args)
|
42
|
+
super(*args) if target_shard?(current_shard)
|
43
|
+
end
|
44
44
|
|
45
|
-
|
46
|
-
|
45
|
+
def target_shard?(shard_name)
|
46
|
+
target_shards.blank? or target_shards.include?(shard_name)
|
47
|
+
end
|
47
48
|
end
|
48
49
|
|
49
50
|
module SchemaStatementsExt
|
50
|
-
def create_sequence_for(table_name, options = {
|
51
|
-
options = options.merge(:
|
51
|
+
def create_sequence_for(table_name, options = {})
|
52
|
+
options = options.merge(id: false)
|
52
53
|
|
53
54
|
# TODO: pkname should be pulled from table definitions
|
54
55
|
pkname = "id"
|
55
56
|
sequence_table_name = ActiveRecord::Turntable::Sequencer.sequence_name(table_name, "id")
|
56
57
|
create_table(sequence_table_name, options) do |t|
|
57
|
-
t.integer :id, :
|
58
|
+
t.integer :id, limit: 8
|
58
59
|
end
|
59
60
|
execute "INSERT INTO #{quote_table_name(sequence_table_name)} (`id`) VALUES (0)"
|
60
61
|
end
|
61
62
|
|
62
|
-
def drop_sequence_for(table_name, options = {
|
63
|
+
def drop_sequence_for(table_name, options = {})
|
63
64
|
# TODO: pkname should be pulled from table definitions
|
64
65
|
pkname = "id"
|
65
66
|
sequence_table_name = ActiveRecord::Turntable::Sequencer.sequence_name(table_name, "id")
|
@@ -85,56 +86,49 @@ module ActiveRecord::Turntable::Migration
|
|
85
86
|
|
86
87
|
private
|
87
88
|
|
88
|
-
|
89
|
-
|
90
|
-
|
89
|
+
def invert_create_sequence_for(args)
|
90
|
+
[:drop_sequence_for, args]
|
91
|
+
end
|
91
92
|
|
92
|
-
|
93
|
-
|
94
|
-
|
93
|
+
def invert_rename_sequence_for(args)
|
94
|
+
[:rename_sequence_for, args.reverse]
|
95
|
+
end
|
95
96
|
end
|
96
97
|
|
97
98
|
module Migrator
|
98
99
|
extend ActiveSupport::Concern
|
99
100
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
original_method_alias = "_original_#{method_name}"
|
105
|
-
unless klass.respond_to?(original_method_alias)
|
106
|
-
alias_method original_method_alias, method_name
|
107
|
-
end
|
108
|
-
alias_method_chain method_name, :turntable
|
109
|
-
end
|
110
|
-
}
|
101
|
+
def self.prepended(base)
|
102
|
+
class << base
|
103
|
+
prepend ClassMethods
|
104
|
+
end
|
111
105
|
end
|
112
106
|
|
113
107
|
module ClassMethods
|
114
|
-
def
|
115
|
-
|
108
|
+
def up(migrations_paths, target_version = nil)
|
109
|
+
super
|
116
110
|
|
117
111
|
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration|
|
118
112
|
puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})"
|
119
|
-
|
113
|
+
super(migrations_paths, target_version)
|
120
114
|
end
|
121
115
|
end
|
122
116
|
|
123
|
-
def
|
124
|
-
|
117
|
+
def down(migrations_paths, target_version = nil, &block)
|
118
|
+
super
|
125
119
|
|
126
120
|
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration|
|
127
121
|
puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})"
|
128
|
-
|
122
|
+
super(migrations_paths, target_version, &block)
|
129
123
|
end
|
130
124
|
end
|
131
125
|
|
132
|
-
def
|
133
|
-
|
126
|
+
def run(*args)
|
127
|
+
super
|
134
128
|
|
135
129
|
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration|
|
136
130
|
puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})"
|
137
|
-
|
131
|
+
super(*args)
|
138
132
|
end
|
139
133
|
end
|
140
134
|
end
|