ar-octopus 0.8.2 → 0.8.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +9 -9
- data/.rspec +1 -1
- data/.rubocop.yml +46 -0
- data/.rubocop_todo.yml +52 -0
- data/.ruby-version +1 -1
- data/.travis.yml +3 -9
- data/Appraisals +4 -0
- data/Rakefile +17 -16
- data/ar-octopus.gemspec +22 -16
- data/gemfiles/rails41.gemfile +7 -0
- data/init.rb +1 -1
- data/lib/ar-octopus.rb +1 -1
- data/lib/octopus.rb +38 -37
- data/lib/octopus/abstract_adapter.rb +0 -2
- data/lib/octopus/association.rb +8 -6
- data/lib/octopus/association_shard_tracking.rb +80 -81
- data/lib/octopus/collection_association.rb +7 -5
- data/lib/octopus/collection_proxy.rb +11 -9
- data/lib/octopus/has_and_belongs_to_many_association.rb +5 -3
- data/lib/octopus/load_balancing.rb +3 -2
- data/lib/octopus/load_balancing/round_robin.rb +12 -8
- data/lib/octopus/migration.rb +117 -108
- data/lib/octopus/model.rb +130 -134
- data/lib/octopus/persistence.rb +1 -1
- data/lib/octopus/proxy.rb +345 -339
- data/lib/octopus/railtie.rb +2 -2
- data/lib/octopus/relation_proxy.rb +6 -1
- data/lib/octopus/scope_proxy.rb +38 -36
- data/lib/octopus/shard_tracking.rb +36 -35
- data/lib/octopus/shard_tracking/attribute.rb +12 -14
- data/lib/octopus/shard_tracking/dynamic.rb +7 -3
- data/lib/octopus/singular_association.rb +5 -3
- data/lib/octopus/slave_group.rb +10 -8
- data/lib/octopus/version.rb +1 -1
- data/rails/init.rb +1 -1
- data/sample_app/autotest/discover.rb +2 -2
- data/sample_app/config/application.rb +1 -1
- data/sample_app/config/boot.rb +1 -1
- data/sample_app/config/environments/test.rb +1 -1
- data/sample_app/config/initializers/session_store.rb +1 -1
- data/sample_app/config/initializers/wrap_parameters.rb +1 -1
- data/sample_app/config/routes.rb +1 -1
- data/sample_app/db/migrate/20100720210335_create_sample_users.rb +2 -2
- data/sample_app/db/schema.rb +10 -10
- data/sample_app/db/seeds.rb +3 -3
- data/sample_app/features/step_definitions/seeds_steps.rb +4 -4
- data/sample_app/features/step_definitions/web_steps.rb +3 -4
- data/sample_app/features/support/env.rb +3 -4
- data/sample_app/features/support/paths.rb +4 -4
- data/sample_app/spec/spec_helper.rb +3 -3
- data/spec/migrations/10_create_users_using_replication.rb +3 -3
- data/spec/migrations/11_add_field_in_all_slaves.rb +3 -3
- data/spec/migrations/12_create_users_using_block.rb +7 -7
- data/spec/migrations/13_create_users_using_block_and_using.rb +4 -4
- data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +2 -2
- data/spec/migrations/15_create_user_on_shards_of_default_group_with_versions.rb +2 -2
- data/spec/migrations/1_create_users_on_master.rb +3 -3
- data/spec/migrations/2_create_users_on_canada.rb +3 -3
- data/spec/migrations/3_create_users_on_both_shards.rb +3 -3
- data/spec/migrations/4_create_users_on_shards_of_a_group.rb +3 -3
- data/spec/migrations/5_create_users_on_multiples_groups.rb +2 -2
- data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +3 -3
- data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +3 -3
- data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +3 -3
- data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +4 -4
- data/spec/octopus/association_shard_tracking_spec.rb +413 -417
- data/spec/octopus/collection_proxy_spec.rb +6 -5
- data/spec/octopus/log_subscriber_spec.rb +4 -4
- data/spec/octopus/migration_spec.rb +48 -48
- data/spec/octopus/model_spec.rb +267 -292
- data/spec/octopus/octopus_spec.rb +40 -41
- data/spec/octopus/proxy_spec.rb +124 -124
- data/spec/octopus/relation_proxy_spec.rb +32 -32
- data/spec/octopus/replicated_slave_grouped_spec.rb +23 -23
- data/spec/octopus/replication_spec.rb +61 -66
- data/spec/octopus/scope_proxy_spec.rb +56 -10
- data/spec/octopus/sharded_replicated_slave_grouped_spec.rb +29 -29
- data/spec/octopus/sharded_spec.rb +10 -10
- data/spec/spec_helper.rb +6 -6
- data/spec/support/active_record/connection_adapters/modify_config_adapter.rb +1 -3
- data/spec/support/database_connection.rb +2 -2
- data/spec/support/database_models.rb +16 -17
- data/spec/support/octopus_helper.rb +19 -21
- data/spec/support/query_count.rb +1 -3
- data/spec/support/shared_contexts.rb +3 -3
- data/spec/tasks/octopus.rake_spec.rb +10 -10
- metadata +43 -26
data/lib/octopus/model.rb
CHANGED
@@ -1,183 +1,179 @@
|
|
1
1
|
require 'active_support/deprecation'
|
2
2
|
|
3
|
-
module Octopus
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
3
|
+
module Octopus
|
4
|
+
module Model
|
5
|
+
def self.extended(base)
|
6
|
+
base.send(:include, Octopus::ShardTracking::Attribute)
|
7
|
+
base.send(:include, InstanceMethods)
|
8
|
+
base.extend(ClassMethods)
|
9
|
+
end
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
11
|
+
module SharedMethods
|
12
|
+
def clean_table_name
|
13
|
+
return unless connection_proxy.should_clean_table_name?
|
13
14
|
|
14
|
-
|
15
|
-
|
16
|
-
|
15
|
+
if self != ActiveRecord::Base && self.respond_to?(:reset_table_name) && !custom_octopus_table_name
|
16
|
+
reset_table_name
|
17
|
+
end
|
17
18
|
|
18
|
-
|
19
|
-
|
20
|
-
|
19
|
+
reset_column_information
|
20
|
+
instance_variable_set(:@quoted_table_name, nil)
|
21
|
+
end
|
21
22
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
23
|
+
def using(shard)
|
24
|
+
if Octopus.enabled?
|
25
|
+
clean_table_name
|
26
|
+
Octopus::ScopeProxy.new(shard, self)
|
27
|
+
else
|
28
|
+
self
|
29
|
+
end
|
28
30
|
end
|
29
31
|
end
|
30
|
-
end
|
31
32
|
|
32
|
-
|
33
|
-
|
33
|
+
module InstanceMethods
|
34
|
+
include SharedMethods
|
34
35
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
36
|
+
def self.included(base)
|
37
|
+
base.send(:alias_method, :equality_without_octopus, :==)
|
38
|
+
base.send(:alias_method, :==, :equality_with_octopus)
|
39
|
+
base.send(:alias_method, :eql?, :==)
|
40
|
+
base.send(:alias_method_chain, :perform_validations, :octopus)
|
41
|
+
end
|
41
42
|
|
42
|
-
|
43
|
-
|
43
|
+
def set_current_shard
|
44
|
+
return unless Octopus.enabled?
|
44
45
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
46
|
+
if new_record? || self.class.connection_proxy.block
|
47
|
+
self.current_shard = self.class.connection_proxy.current_shard
|
48
|
+
else
|
49
|
+
self.current_shard = self.class.connection_proxy.last_current_shard || self.class.connection_proxy.current_shard
|
50
|
+
end
|
49
51
|
end
|
50
|
-
end
|
51
52
|
|
52
|
-
|
53
|
-
|
54
|
-
|
53
|
+
def should_set_current_shard?
|
54
|
+
self.respond_to?(:current_shard) && !current_shard.nil?
|
55
|
+
end
|
55
56
|
|
56
|
-
|
57
|
-
|
58
|
-
|
57
|
+
def equality_with_octopus(comparison_object)
|
58
|
+
equality_without_octopus(comparison_object) && comparison_object.current_shard == current_shard
|
59
|
+
end
|
59
60
|
|
60
|
-
|
61
|
-
|
62
|
-
|
61
|
+
def perform_validations_with_octopus(*args)
|
62
|
+
if Octopus.enabled? && should_set_current_shard?
|
63
|
+
Octopus.using(current_shard) do
|
64
|
+
perform_validations_without_octopus(*args)
|
65
|
+
end
|
66
|
+
else
|
63
67
|
perform_validations_without_octopus(*args)
|
64
68
|
end
|
65
|
-
else
|
66
|
-
perform_validations_without_octopus(*args)
|
67
69
|
end
|
68
70
|
end
|
69
|
-
end
|
70
71
|
|
71
|
-
|
72
|
-
|
72
|
+
module ClassMethods
|
73
|
+
include SharedMethods
|
73
74
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
75
|
+
def self.extended(base)
|
76
|
+
base.class_attribute(:replicated)
|
77
|
+
base.class_attribute(:sharded)
|
78
|
+
base.hijack_methods
|
79
|
+
end
|
79
80
|
|
80
|
-
|
81
|
-
|
82
|
-
|
81
|
+
def replicated_model
|
82
|
+
self.replicated = true
|
83
|
+
end
|
83
84
|
|
84
|
-
|
85
|
-
|
86
|
-
|
85
|
+
def sharded_model
|
86
|
+
self.sharded = true
|
87
|
+
end
|
87
88
|
|
88
|
-
|
89
|
-
|
90
|
-
|
89
|
+
def hijack_methods
|
90
|
+
around_save :run_on_shard
|
91
|
+
after_initialize :set_current_shard
|
91
92
|
|
92
|
-
|
93
|
-
|
94
|
-
|
93
|
+
class << self
|
94
|
+
attr_accessor :custom_octopus_connection
|
95
|
+
attr_accessor :custom_octopus_table_name
|
95
96
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
97
|
+
alias_method_chain :connection, :octopus
|
98
|
+
alias_method_chain :connection_pool, :octopus
|
99
|
+
alias_method_chain :clear_all_connections!, :octopus
|
100
|
+
alias_method_chain :clear_active_connections!, :octopus
|
101
|
+
alias_method_chain :connected?, :octopus
|
101
102
|
|
102
|
-
|
103
|
-
alias_method_chain(:set_table_name, :octopus)
|
104
|
-
end
|
103
|
+
alias_method_chain(:set_table_name, :octopus) if Octopus.rails3?
|
105
104
|
|
106
|
-
|
107
|
-
|
108
|
-
|
105
|
+
def table_name=(value = nil)
|
106
|
+
self.custom_octopus_table_name = true
|
107
|
+
super
|
108
|
+
end
|
109
109
|
end
|
110
110
|
end
|
111
|
-
end
|
112
111
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
ActiveRecord::Base.class_variable_set :@@connection_proxy, p
|
119
|
-
p
|
120
|
-
end
|
121
|
-
end
|
112
|
+
def connection_proxy
|
113
|
+
ActiveRecord::Base.class_variable_defined?(:@@connection_proxy) &&
|
114
|
+
ActiveRecord::Base.class_variable_get(:@@connection_proxy) ||
|
115
|
+
ActiveRecord::Base.class_variable_set(:@@connection_proxy, Octopus::Proxy.new)
|
116
|
+
end
|
122
117
|
|
123
|
-
|
124
|
-
|
125
|
-
|
118
|
+
def should_use_normal_connection?
|
119
|
+
!Octopus.enabled? || custom_octopus_connection
|
120
|
+
end
|
126
121
|
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
122
|
+
def connection_with_octopus
|
123
|
+
if should_use_normal_connection?
|
124
|
+
connection_without_octopus
|
125
|
+
else
|
126
|
+
connection_proxy.current_model = self
|
127
|
+
connection_proxy
|
128
|
+
end
|
133
129
|
end
|
134
|
-
end
|
135
130
|
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
131
|
+
def connection_pool_with_octopus
|
132
|
+
if should_use_normal_connection?
|
133
|
+
connection_pool_without_octopus
|
134
|
+
else
|
135
|
+
connection_proxy.connection_pool
|
136
|
+
end
|
141
137
|
end
|
142
|
-
end
|
143
138
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
139
|
+
def clear_active_connections_with_octopus!
|
140
|
+
if should_use_normal_connection?
|
141
|
+
clear_active_connections_without_octopus!
|
142
|
+
else
|
143
|
+
connection_proxy.clear_active_connections!
|
144
|
+
end
|
149
145
|
end
|
150
|
-
end
|
151
146
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
147
|
+
def clear_all_connections_with_octopus!
|
148
|
+
if should_use_normal_connection?
|
149
|
+
clear_all_connections_without_octopus!
|
150
|
+
else
|
151
|
+
connection_proxy.clear_all_connections!
|
152
|
+
end
|
157
153
|
end
|
158
|
-
end
|
159
154
|
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
155
|
+
def connected_with_octopus?
|
156
|
+
if should_use_normal_connection?
|
157
|
+
connected_without_octopus?
|
158
|
+
else
|
159
|
+
connection_proxy.connected?
|
160
|
+
end
|
165
161
|
end
|
166
|
-
end
|
167
162
|
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
163
|
+
def set_table_name_with_octopus(value = nil, &block)
|
164
|
+
self.custom_octopus_table_name = true
|
165
|
+
set_table_name_without_octopus(value, &block)
|
166
|
+
end
|
172
167
|
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
168
|
+
def octopus_establish_connection(spec = ENV['DATABASE_URL'])
|
169
|
+
self.custom_octopus_connection = true if spec
|
170
|
+
establish_connection(spec)
|
171
|
+
end
|
177
172
|
|
178
|
-
|
179
|
-
|
180
|
-
|
173
|
+
def octopus_set_table_name(value = nil)
|
174
|
+
ActiveSupport::Deprecation.warn 'Calling `octopus_set_table_name` is deprecated and will be removed in Octopus 1.0.', caller
|
175
|
+
set_table_name(value)
|
176
|
+
end
|
181
177
|
end
|
182
178
|
end
|
183
179
|
end
|
data/lib/octopus/persistence.rb
CHANGED
data/lib/octopus/proxy.rb
CHANGED
@@ -1,437 +1,443 @@
|
|
1
|
-
require
|
1
|
+
require 'set'
|
2
2
|
require 'octopus/slave_group'
|
3
3
|
require 'octopus/load_balancing/round_robin'
|
4
4
|
|
5
|
-
|
6
|
-
|
5
|
+
module Octopus
|
6
|
+
class Proxy
|
7
|
+
attr_accessor :config, :sharded
|
7
8
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
def initialize(config = Octopus.config)
|
10
|
+
initialize_shards(config)
|
11
|
+
initialize_replication(config) if !config.nil? && config['replicated']
|
12
|
+
end
|
12
13
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
@shards_config.each do |key, value|
|
29
|
-
if value.is_a?(String)
|
30
|
-
value = resolve_string_connection(value).merge(:octopus_shard => key)
|
31
|
-
initialize_adapter(value['adapter'])
|
32
|
-
@shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
33
|
-
elsif value.is_a?(Hash) && value.has_key?("adapter")
|
34
|
-
value.merge!(:octopus_shard => key)
|
35
|
-
initialize_adapter(value['adapter'])
|
36
|
-
@shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
37
|
-
|
38
|
-
slave_group_configs = value.select do |k,v|
|
39
|
-
structurally_slave_group? v
|
40
|
-
end
|
14
|
+
def initialize_shards(config)
|
15
|
+
@shards = HashWithIndifferentAccess.new
|
16
|
+
@shards_slave_groups = HashWithIndifferentAccess.new
|
17
|
+
@slave_groups = HashWithIndifferentAccess.new
|
18
|
+
@groups = {}
|
19
|
+
@adapters = Set.new
|
20
|
+
@config = ActiveRecord::Base.connection_pool_without_octopus.connection.instance_variable_get(:@config)
|
21
|
+
|
22
|
+
unless config.nil?
|
23
|
+
@entire_sharded = config['entire_sharded']
|
24
|
+
@shards_config = config[Octopus.rails_env]
|
25
|
+
end
|
26
|
+
|
27
|
+
@shards_config ||= []
|
41
28
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
29
|
+
@shards_config.each do |key, value|
|
30
|
+
if value.is_a?(String)
|
31
|
+
value = resolve_string_connection(value).merge(:octopus_shard => key)
|
32
|
+
initialize_adapter(value['adapter'])
|
33
|
+
@shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
34
|
+
elsif value.is_a?(Hash) && value.key?('adapter')
|
35
|
+
value.merge!(:octopus_shard => key)
|
36
|
+
initialize_adapter(value['adapter'])
|
37
|
+
@shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
38
|
+
|
39
|
+
slave_group_configs = value.select do |_k, v|
|
40
|
+
structurally_slave_group? v
|
41
|
+
end
|
42
|
+
|
43
|
+
if slave_group_configs.present?
|
44
|
+
slave_groups = HashWithIndifferentAccess.new
|
45
|
+
slave_group_configs.each do |slave_group_name, slave_configs|
|
46
|
+
slaves = HashWithIndifferentAccess.new
|
47
|
+
slave_configs.each do |slave_name, slave_config|
|
48
|
+
@shards[slave_name.to_sym] = connection_pool_for(slave_config, "#{value['adapter']}_connection")
|
49
|
+
slaves[slave_name.to_sym] = slave_name.to_sym
|
50
|
+
end
|
51
|
+
slave_groups[slave_group_name.to_sym] = Octopus::SlaveGroup.new(slaves)
|
49
52
|
end
|
50
|
-
|
53
|
+
@shards_slave_groups[key.to_sym] = slave_groups
|
54
|
+
@sharded = true
|
51
55
|
end
|
52
|
-
|
53
|
-
@
|
54
|
-
end
|
55
|
-
elsif value.is_a?(Hash)
|
56
|
-
@groups[key.to_s] = []
|
56
|
+
elsif value.is_a?(Hash)
|
57
|
+
@groups[key.to_s] = []
|
57
58
|
|
58
|
-
|
59
|
-
|
59
|
+
value.each do |k, v|
|
60
|
+
fail 'You have duplicated shard names!' if @shards.key?(k.to_sym)
|
60
61
|
|
61
|
-
|
62
|
-
|
62
|
+
initialize_adapter(v['adapter'])
|
63
|
+
config_with_octopus_shard = v.merge(:octopus_shard => k)
|
63
64
|
|
64
|
-
|
65
|
-
|
66
|
-
|
65
|
+
@shards[k.to_sym] = connection_pool_for(config_with_octopus_shard, "#{v['adapter']}_connection")
|
66
|
+
@groups[key.to_s] << k.to_sym
|
67
|
+
end
|
67
68
|
|
68
|
-
|
69
|
-
|
70
|
-
|
69
|
+
if structurally_slave_group? value
|
70
|
+
slaves = Hash[@groups[key.to_s].map { |v| [v, v] }]
|
71
|
+
@slave_groups[key.to_sym] = Octopus::SlaveGroup.new(slaves)
|
72
|
+
end
|
71
73
|
end
|
72
74
|
end
|
75
|
+
|
76
|
+
@shards[:master] ||= ActiveRecord::Base.connection_pool_without_octopus
|
73
77
|
end
|
74
78
|
|
75
|
-
|
76
|
-
|
79
|
+
def initialize_replication(config)
|
80
|
+
@replicated = true
|
81
|
+
if config.key?('fully_replicated')
|
82
|
+
@fully_replicated = config['fully_replicated']
|
83
|
+
else
|
84
|
+
@fully_replicated = true
|
85
|
+
end
|
77
86
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
@fully_replicated = config["fully_replicated"]
|
82
|
-
else
|
83
|
-
@fully_replicated = true
|
87
|
+
@slaves_list = @shards.keys.map { |sym| sym.to_s }.sort
|
88
|
+
@slaves_list.delete('master')
|
89
|
+
@slaves_load_balancer = Octopus::LoadBalancing::RoundRobin.new(@slaves_list)
|
84
90
|
end
|
85
91
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
end
|
90
|
-
|
91
|
-
def current_model
|
92
|
-
Thread.current["octopus.current_model"]
|
93
|
-
end
|
92
|
+
def current_model
|
93
|
+
Thread.current['octopus.current_model']
|
94
|
+
end
|
94
95
|
|
95
|
-
|
96
|
-
|
97
|
-
|
96
|
+
def current_model=(model)
|
97
|
+
Thread.current['octopus.current_model'] = model.is_a?(ActiveRecord::Base) ? model.class : model
|
98
|
+
end
|
98
99
|
|
99
|
-
|
100
|
-
|
101
|
-
|
100
|
+
def current_shard
|
101
|
+
Thread.current['octopus.current_shard'] ||= :master
|
102
|
+
end
|
102
103
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
104
|
+
def current_shard=(shard_symbol)
|
105
|
+
self.current_slave_group = nil
|
106
|
+
if shard_symbol.is_a?(Array)
|
107
|
+
shard_symbol.each { |symbol| fail "Nonexistent Shard Name: #{symbol}" if @shards[symbol].nil? }
|
108
|
+
elsif shard_symbol.is_a?(Hash)
|
109
|
+
hash = shard_symbol
|
110
|
+
shard_symbol = hash[:shard]
|
111
|
+
slave_group_symbol = hash[:slave_group]
|
112
|
+
|
113
|
+
if shard_symbol.nil? && slave_group_symbol.nil?
|
114
|
+
fail 'Neither shard or slave group must be specified'
|
115
|
+
end
|
115
116
|
|
116
|
-
|
117
|
-
|
118
|
-
|
117
|
+
if shard_symbol.present?
|
118
|
+
fail "Nonexistent Shard Name: #{shard_symbol}" if @shards[shard_symbol].nil?
|
119
|
+
end
|
119
120
|
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
121
|
+
if slave_group_symbol.present?
|
122
|
+
if (@shards_slave_groups.try(:[], shard_symbol).present? && @shards_slave_groups[shard_symbol][slave_group_symbol].nil?) ||
|
123
|
+
(@shards_slave_groups.try(:[], shard_symbol).nil? && @slave_groups[slave_group_symbol].nil?)
|
124
|
+
fail "Nonexistent Slave Group Name: #{slave_group_symbol} in shards config: #{@shards_config.inspect}"
|
125
|
+
end
|
126
|
+
self.current_slave_group = slave_group_symbol
|
124
127
|
end
|
125
|
-
|
128
|
+
else
|
129
|
+
fail "Nonexistent Shard Name: #{shard_symbol}" if @shards[shard_symbol].nil?
|
126
130
|
end
|
127
|
-
|
128
|
-
|
131
|
+
|
132
|
+
Thread.current['octopus.current_shard'] = shard_symbol
|
129
133
|
end
|
130
134
|
|
131
|
-
|
132
|
-
|
135
|
+
def current_group
|
136
|
+
Thread.current['octopus.current_group']
|
137
|
+
end
|
133
138
|
|
134
|
-
|
135
|
-
|
136
|
-
|
139
|
+
def current_group=(group_symbol)
|
140
|
+
# TODO: Error message should include all groups if given more than one bad name.
|
141
|
+
[group_symbol].flatten.compact.each do |group|
|
142
|
+
fail "Nonexistent Group Name: #{group}" unless has_group?(group)
|
143
|
+
end
|
137
144
|
|
138
|
-
|
139
|
-
# TODO: Error message should include all groups if given more than one bad name.
|
140
|
-
[group_symbol].flatten.compact.each do |group|
|
141
|
-
raise "Nonexistent Group Name: #{group}" unless has_group?(group)
|
145
|
+
Thread.current['octopus.current_group'] = group_symbol
|
142
146
|
end
|
143
147
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
def current_slave_group
|
148
|
-
Thread.current["octopus.current_slave_group"]
|
149
|
-
end
|
148
|
+
def current_slave_group
|
149
|
+
Thread.current['octopus.current_slave_group']
|
150
|
+
end
|
150
151
|
|
151
|
-
|
152
|
-
|
153
|
-
|
152
|
+
def current_slave_group=(slave_group_symbol)
|
153
|
+
Thread.current['octopus.current_slave_group'] = slave_group_symbol
|
154
|
+
end
|
154
155
|
|
155
|
-
|
156
|
-
|
157
|
-
|
156
|
+
def block
|
157
|
+
Thread.current['octopus.block']
|
158
|
+
end
|
158
159
|
|
159
|
-
|
160
|
-
|
161
|
-
|
160
|
+
def block=(block)
|
161
|
+
Thread.current['octopus.block'] = block
|
162
|
+
end
|
162
163
|
|
163
|
-
|
164
|
-
|
165
|
-
|
164
|
+
def last_current_shard
|
165
|
+
Thread.current['octopus.last_current_shard']
|
166
|
+
end
|
166
167
|
|
167
|
-
|
168
|
-
|
169
|
-
|
168
|
+
def last_current_shard=(last_current_shard)
|
169
|
+
Thread.current['octopus.last_current_shard'] = last_current_shard
|
170
|
+
end
|
170
171
|
|
171
|
-
|
172
|
-
|
173
|
-
|
172
|
+
def fully_replicated?
|
173
|
+
@fully_replicated || Thread.current['octopus.fully_replicated']
|
174
|
+
end
|
174
175
|
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
176
|
+
# Public: Whether or not a group exists with the given name converted to a
|
177
|
+
# string.
|
178
|
+
#
|
179
|
+
# Returns a boolean.
|
180
|
+
def has_group?(group)
|
181
|
+
@groups.key?(group.to_s)
|
182
|
+
end
|
182
183
|
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
184
|
+
# Public: Retrieves names of all loaded shards.
|
185
|
+
#
|
186
|
+
# Returns an array of shard names as symbols
|
187
|
+
def shard_names
|
188
|
+
@shards.keys
|
189
|
+
end
|
189
190
|
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
191
|
+
# Public: Retrieves the defined shards for a given group.
|
192
|
+
#
|
193
|
+
# Returns an array of shard names as symbols or nil if the group is not
|
194
|
+
# defined.
|
195
|
+
def shards_for_group(group)
|
196
|
+
@groups.fetch(group.to_s, nil)
|
197
|
+
end
|
197
198
|
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
199
|
+
# Rails 3.1 sets automatic_reconnect to false when it removes
|
200
|
+
# connection pool. Octopus can potentially retain a reference to a closed
|
201
|
+
# connection pool. Previously, that would work since the pool would just
|
202
|
+
# reconnect, but in Rails 3.1 the flag prevents this.
|
203
|
+
def safe_connection(connection_pool)
|
204
|
+
connection_pool.automatic_reconnect ||= true
|
205
|
+
connection_pool.connection
|
206
|
+
end
|
206
207
|
|
207
|
-
|
208
|
-
|
209
|
-
|
208
|
+
def select_connection
|
209
|
+
safe_connection(@shards[shard_name])
|
210
|
+
end
|
210
211
|
|
211
|
-
|
212
|
-
|
213
|
-
|
212
|
+
def shard_name
|
213
|
+
current_shard.is_a?(Array) ? current_shard.first : current_shard
|
214
|
+
end
|
214
215
|
|
215
|
-
|
216
|
-
|
217
|
-
|
216
|
+
def should_clean_table_name?
|
217
|
+
@adapters.size > 1
|
218
|
+
end
|
218
219
|
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
220
|
+
def run_queries_on_shard(shard, &_block)
|
221
|
+
keeping_connection_proxy do
|
222
|
+
using_shard(shard) do
|
223
|
+
yield
|
224
|
+
end
|
223
225
|
end
|
224
226
|
end
|
225
|
-
end
|
226
227
|
|
227
|
-
|
228
|
-
|
229
|
-
|
228
|
+
def send_queries_to_multiple_shards(shards, &block)
|
229
|
+
shards.each do |shard|
|
230
|
+
run_queries_on_shard(shard, &block)
|
231
|
+
end
|
230
232
|
end
|
231
|
-
end
|
232
233
|
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
234
|
+
def clean_connection_proxy
|
235
|
+
self.current_shard = :master
|
236
|
+
self.current_group = nil
|
237
|
+
self.block = false
|
238
|
+
end
|
238
239
|
|
239
|
-
|
240
|
-
|
241
|
-
|
240
|
+
def check_schema_migrations(shard)
|
241
|
+
OctopusModel.using(shard).connection.table_exists?(
|
242
|
+
ActiveRecord::Migrator.schema_migrations_table_name,
|
243
|
+
) || OctopusModel.using(shard).connection.initialize_schema_migrations_table
|
242
244
|
end
|
243
|
-
end
|
244
245
|
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
246
|
+
def transaction(options = {}, &block)
|
247
|
+
replicated = @replicated && (current_model.replicated || fully_replicated?)
|
248
|
+
if !sharded && replicated
|
249
|
+
run_queries_on_shard(:master) do
|
250
|
+
select_connection.transaction(options, &block)
|
251
|
+
end
|
252
|
+
else
|
249
253
|
select_connection.transaction(options, &block)
|
250
254
|
end
|
251
|
-
else
|
252
|
-
select_connection.transaction(options, &block)
|
253
255
|
end
|
254
|
-
end
|
255
256
|
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
257
|
+
def method_missing(method, *args, &block)
|
258
|
+
if should_clean_connection_proxy?(method)
|
259
|
+
conn = select_connection
|
260
|
+
self.last_current_shard = current_shard
|
261
|
+
clean_connection_proxy
|
262
|
+
conn.send(method, *args, &block)
|
263
|
+
elsif should_send_queries_to_shard_slave_group?(method)
|
264
|
+
send_queries_to_shard_slave_group(method, *args, &block)
|
265
|
+
elsif should_send_queries_to_slave_group?(method)
|
266
|
+
send_queries_to_slave_group(method, *args, &block)
|
267
|
+
elsif should_send_queries_to_replicated_databases?(method)
|
268
|
+
send_queries_to_selected_slave(method, *args, &block)
|
269
|
+
else
|
270
|
+
select_connection.send(method, *args, &block)
|
271
|
+
end
|
270
272
|
end
|
271
|
-
end
|
272
273
|
|
273
|
-
|
274
|
-
|
275
|
-
|
274
|
+
def respond_to?(method, include_private = false)
|
275
|
+
super || select_connection.respond_to?(method, include_private)
|
276
|
+
end
|
276
277
|
|
277
|
-
|
278
|
-
|
279
|
-
|
278
|
+
def connection_pool
|
279
|
+
@shards[current_shard]
|
280
|
+
end
|
280
281
|
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
282
|
+
def enable_query_cache!
|
283
|
+
clear_query_cache
|
284
|
+
@shards.each { |_k, v| safe_connection(v).enable_query_cache! }
|
285
|
+
end
|
285
286
|
|
286
|
-
|
287
|
-
|
288
|
-
|
287
|
+
def disable_query_cache!
|
288
|
+
@shards.each { |_k, v| safe_connection(v).disable_query_cache! }
|
289
|
+
end
|
289
290
|
|
290
|
-
|
291
|
-
|
292
|
-
|
291
|
+
def clear_query_cache
|
292
|
+
@shards.each { |_k, v| safe_connection(v).clear_query_cache }
|
293
|
+
end
|
293
294
|
|
294
|
-
|
295
|
-
|
296
|
-
|
295
|
+
def clear_active_connections!
|
296
|
+
@shards.each { |_k, v| v.release_connection }
|
297
|
+
end
|
297
298
|
|
298
|
-
|
299
|
-
|
300
|
-
|
299
|
+
def clear_all_connections!
|
300
|
+
@shards.each { |_k, v| v.disconnect! }
|
301
|
+
end
|
301
302
|
|
302
|
-
|
303
|
-
|
304
|
-
|
303
|
+
def connected?
|
304
|
+
@shards.any? { |_k, v| v.connected? }
|
305
|
+
end
|
305
306
|
|
306
|
-
|
307
|
-
|
308
|
-
|
307
|
+
def should_send_queries_to_shard_slave_group?(method)
|
308
|
+
should_use_slaves_for_method?(method) && @shards_slave_groups.try(:[], current_shard).try(:[], current_slave_group).present?
|
309
|
+
end
|
309
310
|
|
310
|
-
|
311
|
-
|
312
|
-
|
311
|
+
def send_queries_to_shard_slave_group(method, *args, &block)
|
312
|
+
send_queries_to_balancer(@shards_slave_groups[current_shard][current_slave_group], method, *args, &block)
|
313
|
+
end
|
313
314
|
|
314
|
-
|
315
|
-
|
316
|
-
|
315
|
+
def should_send_queries_to_slave_group?(method)
|
316
|
+
should_use_slaves_for_method?(method) && @slave_groups.try(:[], current_slave_group).present?
|
317
|
+
end
|
317
318
|
|
318
|
-
|
319
|
-
|
320
|
-
|
319
|
+
def send_queries_to_slave_group(method, *args, &block)
|
320
|
+
send_queries_to_balancer(@slave_groups[current_slave_group], method, *args, &block)
|
321
|
+
end
|
321
322
|
|
322
|
-
|
323
|
+
protected
|
323
324
|
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
325
|
+
def connection_pool_for(adapter, config)
|
326
|
+
if Octopus.rails4?
|
327
|
+
arg = ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(adapter.dup, config)
|
328
|
+
else
|
329
|
+
arg = ActiveRecord::Base::ConnectionSpecification.new(adapter.dup, config)
|
330
|
+
end
|
331
|
+
|
332
|
+
ActiveRecord::ConnectionAdapters::ConnectionPool.new(arg)
|
329
333
|
end
|
330
334
|
|
331
|
-
|
332
|
-
|
335
|
+
def initialize_adapter(adapter)
|
336
|
+
@adapters << adapter
|
337
|
+
begin
|
338
|
+
require "active_record/connection_adapters/#{adapter}_adapter"
|
339
|
+
rescue LoadError
|
340
|
+
raise "Please install the #{adapter} adapter: `gem install activerecord-#{adapter}-adapter` (#{$ERROR_INFO})"
|
341
|
+
end
|
342
|
+
end
|
333
343
|
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
344
|
+
def resolve_string_connection(spec)
|
345
|
+
if Octopus.rails41?
|
346
|
+
resolver = ActiveRecord::ConnectionAdapters::ConnectionSpecification::Resolver.new({})
|
347
|
+
resolver.spec(spec).config.stringify_keys
|
348
|
+
else
|
349
|
+
if Octopus.rails4?
|
350
|
+
resolver = ActiveRecord::ConnectionAdapters::ConnectionSpecification::Resolver.new(spec, {})
|
351
|
+
else
|
352
|
+
resolver = ActiveRecord::Base::ConnectionSpecification::Resolver.new(spec, {})
|
353
|
+
end
|
354
|
+
resolver.spec.config.stringify_keys
|
355
|
+
end
|
340
356
|
end
|
341
|
-
end
|
342
357
|
|
343
|
-
|
344
|
-
|
345
|
-
resolver = ActiveRecord::ConnectionAdapters::ConnectionSpecification::Resolver.new(spec, {})
|
346
|
-
else
|
347
|
-
resolver = ActiveRecord::Base::ConnectionSpecification::Resolver.new(spec, {})
|
358
|
+
def should_clean_connection_proxy?(method)
|
359
|
+
method.to_s =~ /insert|select|execute/ && !@replicated && !block
|
348
360
|
end
|
349
|
-
resolver.spec.config.stringify_keys
|
350
|
-
end
|
351
361
|
|
352
|
-
|
353
|
-
method
|
354
|
-
|
362
|
+
# Try to use slaves if and only if `replicated: true` is specified in `shards.yml` and no slaves groups are defined
|
363
|
+
def should_send_queries_to_replicated_databases?(method)
|
364
|
+
@replicated && method.to_s =~ /select/ && !block && !slaves_grouped?
|
365
|
+
end
|
355
366
|
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
367
|
+
def send_queries_to_selected_slave(method, *args, &block)
|
368
|
+
if current_model.replicated || fully_replicated?
|
369
|
+
selected_slave = @slaves_load_balancer.next
|
370
|
+
else
|
371
|
+
selected_slave = :master
|
372
|
+
end
|
360
373
|
|
361
|
-
|
362
|
-
if current_model.replicated || fully_replicated?
|
363
|
-
selected_slave = @slaves_load_balancer.next
|
364
|
-
else
|
365
|
-
selected_slave = :master
|
374
|
+
send_queries_to_slave(selected_slave, method, *args, &block)
|
366
375
|
end
|
367
376
|
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
def should_use_slaves_for_method?(method)
|
381
|
-
@replicated && (current_model.replicated || fully_replicated?) && method.to_s =~ /select/
|
382
|
-
end
|
377
|
+
# We should use slaves if and only if its safe to do so.
|
378
|
+
#
|
379
|
+
# We can safely use slaves when:
|
380
|
+
# (1) `replicated: true` is specified in `shards.yml`
|
381
|
+
# (2) The current model is `replicated()`, or `fully_replicated: true` is specified in `shards.yml` which means that
|
382
|
+
# all the model is `replicated()`
|
383
|
+
# (3) It's a SELECT query
|
384
|
+
# while ensuring that we revert `current_shard` from the selected slave to the (shard's) master
|
385
|
+
# not to make queries other than SELECT leak to the slave.
|
386
|
+
def should_use_slaves_for_method?(method)
|
387
|
+
@replicated && (current_model.replicated || fully_replicated?) && method.to_s =~ /select/
|
388
|
+
end
|
383
389
|
|
384
|
-
|
385
|
-
|
386
|
-
|
390
|
+
def slaves_grouped?
|
391
|
+
@slave_groups.present?
|
392
|
+
end
|
387
393
|
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
394
|
+
# Temporarily switch `current_shard` to the next slave in a slave group and send queries to it
|
395
|
+
# while preserving `current_shard`
|
396
|
+
def send_queries_to_balancer(balancer, method, *args, &block)
|
397
|
+
send_queries_to_slave(balancer.next, method, *args, &block)
|
398
|
+
end
|
393
399
|
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
400
|
+
# Temporarily switch `current_shard` to the specified slave and send queries to it
|
401
|
+
# while preserving `current_shard`
|
402
|
+
def send_queries_to_slave(slave, method, *args, &block)
|
403
|
+
using_shard(slave) do
|
404
|
+
select_connection.send(method, *args, &block)
|
405
|
+
end
|
399
406
|
end
|
400
|
-
end
|
401
407
|
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
+
# Temporarily block cleaning connection proxy and run the block
|
409
|
+
#
|
410
|
+
# @see Octopus::Proxy#should_clean_connection?
|
411
|
+
# @see Octopus::Proxy#clean_connection_proxy
|
412
|
+
def keeping_connection_proxy(&_block)
|
413
|
+
last_block = block
|
408
414
|
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
415
|
+
begin
|
416
|
+
self.block = true
|
417
|
+
yield
|
418
|
+
ensure
|
419
|
+
self.block = last_block || false
|
420
|
+
end
|
414
421
|
end
|
415
|
-
end
|
416
422
|
|
417
|
-
|
418
|
-
|
419
|
-
|
423
|
+
# Temporarily switch `current_shard` and run the block
|
424
|
+
def using_shard(shard, &_block)
|
425
|
+
older_shard = current_shard
|
420
426
|
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
427
|
+
begin
|
428
|
+
self.current_shard = shard
|
429
|
+
yield
|
430
|
+
ensure
|
431
|
+
self.current_shard = older_shard
|
432
|
+
end
|
426
433
|
end
|
427
|
-
end
|
428
434
|
|
429
|
-
|
430
|
-
|
431
|
-
|
435
|
+
def structurally_slave?(config)
|
436
|
+
config.is_a?(Hash) && config.key?('adapter')
|
437
|
+
end
|
432
438
|
|
433
|
-
|
434
|
-
|
439
|
+
def structurally_slave_group?(config)
|
440
|
+
config.is_a?(Hash) && config.values.any? { |v| structurally_slave? v }
|
441
|
+
end
|
435
442
|
end
|
436
|
-
|
437
443
|
end
|