ar-octopus 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +9 -9
- data/.rspec +1 -1
- data/.rubocop.yml +46 -0
- data/.rubocop_todo.yml +52 -0
- data/.ruby-version +1 -1
- data/.travis.yml +3 -9
- data/Appraisals +4 -0
- data/Rakefile +17 -16
- data/ar-octopus.gemspec +22 -16
- data/gemfiles/rails41.gemfile +7 -0
- data/init.rb +1 -1
- data/lib/ar-octopus.rb +1 -1
- data/lib/octopus.rb +38 -37
- data/lib/octopus/abstract_adapter.rb +0 -2
- data/lib/octopus/association.rb +8 -6
- data/lib/octopus/association_shard_tracking.rb +80 -81
- data/lib/octopus/collection_association.rb +7 -5
- data/lib/octopus/collection_proxy.rb +11 -9
- data/lib/octopus/has_and_belongs_to_many_association.rb +5 -3
- data/lib/octopus/load_balancing.rb +3 -2
- data/lib/octopus/load_balancing/round_robin.rb +12 -8
- data/lib/octopus/migration.rb +117 -108
- data/lib/octopus/model.rb +130 -134
- data/lib/octopus/persistence.rb +1 -1
- data/lib/octopus/proxy.rb +345 -339
- data/lib/octopus/railtie.rb +2 -2
- data/lib/octopus/relation_proxy.rb +6 -1
- data/lib/octopus/scope_proxy.rb +38 -36
- data/lib/octopus/shard_tracking.rb +36 -35
- data/lib/octopus/shard_tracking/attribute.rb +12 -14
- data/lib/octopus/shard_tracking/dynamic.rb +7 -3
- data/lib/octopus/singular_association.rb +5 -3
- data/lib/octopus/slave_group.rb +10 -8
- data/lib/octopus/version.rb +1 -1
- data/rails/init.rb +1 -1
- data/sample_app/autotest/discover.rb +2 -2
- data/sample_app/config/application.rb +1 -1
- data/sample_app/config/boot.rb +1 -1
- data/sample_app/config/environments/test.rb +1 -1
- data/sample_app/config/initializers/session_store.rb +1 -1
- data/sample_app/config/initializers/wrap_parameters.rb +1 -1
- data/sample_app/config/routes.rb +1 -1
- data/sample_app/db/migrate/20100720210335_create_sample_users.rb +2 -2
- data/sample_app/db/schema.rb +10 -10
- data/sample_app/db/seeds.rb +3 -3
- data/sample_app/features/step_definitions/seeds_steps.rb +4 -4
- data/sample_app/features/step_definitions/web_steps.rb +3 -4
- data/sample_app/features/support/env.rb +3 -4
- data/sample_app/features/support/paths.rb +4 -4
- data/sample_app/spec/spec_helper.rb +3 -3
- data/spec/migrations/10_create_users_using_replication.rb +3 -3
- data/spec/migrations/11_add_field_in_all_slaves.rb +3 -3
- data/spec/migrations/12_create_users_using_block.rb +7 -7
- data/spec/migrations/13_create_users_using_block_and_using.rb +4 -4
- data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +2 -2
- data/spec/migrations/15_create_user_on_shards_of_default_group_with_versions.rb +2 -2
- data/spec/migrations/1_create_users_on_master.rb +3 -3
- data/spec/migrations/2_create_users_on_canada.rb +3 -3
- data/spec/migrations/3_create_users_on_both_shards.rb +3 -3
- data/spec/migrations/4_create_users_on_shards_of_a_group.rb +3 -3
- data/spec/migrations/5_create_users_on_multiples_groups.rb +2 -2
- data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +3 -3
- data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +3 -3
- data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +3 -3
- data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +4 -4
- data/spec/octopus/association_shard_tracking_spec.rb +413 -417
- data/spec/octopus/collection_proxy_spec.rb +6 -5
- data/spec/octopus/log_subscriber_spec.rb +4 -4
- data/spec/octopus/migration_spec.rb +48 -48
- data/spec/octopus/model_spec.rb +267 -292
- data/spec/octopus/octopus_spec.rb +40 -41
- data/spec/octopus/proxy_spec.rb +124 -124
- data/spec/octopus/relation_proxy_spec.rb +32 -32
- data/spec/octopus/replicated_slave_grouped_spec.rb +23 -23
- data/spec/octopus/replication_spec.rb +61 -66
- data/spec/octopus/scope_proxy_spec.rb +56 -10
- data/spec/octopus/sharded_replicated_slave_grouped_spec.rb +29 -29
- data/spec/octopus/sharded_spec.rb +10 -10
- data/spec/spec_helper.rb +6 -6
- data/spec/support/active_record/connection_adapters/modify_config_adapter.rb +1 -3
- data/spec/support/database_connection.rb +2 -2
- data/spec/support/database_models.rb +16 -17
- data/spec/support/octopus_helper.rb +19 -21
- data/spec/support/query_count.rb +1 -3
- data/spec/support/shared_contexts.rb +3 -3
- data/spec/tasks/octopus.rake_spec.rb +10 -10
- metadata +43 -26
data/lib/octopus/model.rb
CHANGED
@@ -1,183 +1,179 @@
|
|
1
1
|
require 'active_support/deprecation'
|
2
2
|
|
3
|
-
module Octopus
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
3
|
+
module Octopus
|
4
|
+
module Model
|
5
|
+
def self.extended(base)
|
6
|
+
base.send(:include, Octopus::ShardTracking::Attribute)
|
7
|
+
base.send(:include, InstanceMethods)
|
8
|
+
base.extend(ClassMethods)
|
9
|
+
end
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
11
|
+
module SharedMethods
|
12
|
+
def clean_table_name
|
13
|
+
return unless connection_proxy.should_clean_table_name?
|
13
14
|
|
14
|
-
|
15
|
-
|
16
|
-
|
15
|
+
if self != ActiveRecord::Base && self.respond_to?(:reset_table_name) && !custom_octopus_table_name
|
16
|
+
reset_table_name
|
17
|
+
end
|
17
18
|
|
18
|
-
|
19
|
-
|
20
|
-
|
19
|
+
reset_column_information
|
20
|
+
instance_variable_set(:@quoted_table_name, nil)
|
21
|
+
end
|
21
22
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
23
|
+
def using(shard)
|
24
|
+
if Octopus.enabled?
|
25
|
+
clean_table_name
|
26
|
+
Octopus::ScopeProxy.new(shard, self)
|
27
|
+
else
|
28
|
+
self
|
29
|
+
end
|
28
30
|
end
|
29
31
|
end
|
30
|
-
end
|
31
32
|
|
32
|
-
|
33
|
-
|
33
|
+
module InstanceMethods
|
34
|
+
include SharedMethods
|
34
35
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
36
|
+
def self.included(base)
|
37
|
+
base.send(:alias_method, :equality_without_octopus, :==)
|
38
|
+
base.send(:alias_method, :==, :equality_with_octopus)
|
39
|
+
base.send(:alias_method, :eql?, :==)
|
40
|
+
base.send(:alias_method_chain, :perform_validations, :octopus)
|
41
|
+
end
|
41
42
|
|
42
|
-
|
43
|
-
|
43
|
+
def set_current_shard
|
44
|
+
return unless Octopus.enabled?
|
44
45
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
46
|
+
if new_record? || self.class.connection_proxy.block
|
47
|
+
self.current_shard = self.class.connection_proxy.current_shard
|
48
|
+
else
|
49
|
+
self.current_shard = self.class.connection_proxy.last_current_shard || self.class.connection_proxy.current_shard
|
50
|
+
end
|
49
51
|
end
|
50
|
-
end
|
51
52
|
|
52
|
-
|
53
|
-
|
54
|
-
|
53
|
+
def should_set_current_shard?
|
54
|
+
self.respond_to?(:current_shard) && !current_shard.nil?
|
55
|
+
end
|
55
56
|
|
56
|
-
|
57
|
-
|
58
|
-
|
57
|
+
def equality_with_octopus(comparison_object)
|
58
|
+
equality_without_octopus(comparison_object) && comparison_object.current_shard == current_shard
|
59
|
+
end
|
59
60
|
|
60
|
-
|
61
|
-
|
62
|
-
|
61
|
+
def perform_validations_with_octopus(*args)
|
62
|
+
if Octopus.enabled? && should_set_current_shard?
|
63
|
+
Octopus.using(current_shard) do
|
64
|
+
perform_validations_without_octopus(*args)
|
65
|
+
end
|
66
|
+
else
|
63
67
|
perform_validations_without_octopus(*args)
|
64
68
|
end
|
65
|
-
else
|
66
|
-
perform_validations_without_octopus(*args)
|
67
69
|
end
|
68
70
|
end
|
69
|
-
end
|
70
71
|
|
71
|
-
|
72
|
-
|
72
|
+
module ClassMethods
|
73
|
+
include SharedMethods
|
73
74
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
75
|
+
def self.extended(base)
|
76
|
+
base.class_attribute(:replicated)
|
77
|
+
base.class_attribute(:sharded)
|
78
|
+
base.hijack_methods
|
79
|
+
end
|
79
80
|
|
80
|
-
|
81
|
-
|
82
|
-
|
81
|
+
def replicated_model
|
82
|
+
self.replicated = true
|
83
|
+
end
|
83
84
|
|
84
|
-
|
85
|
-
|
86
|
-
|
85
|
+
def sharded_model
|
86
|
+
self.sharded = true
|
87
|
+
end
|
87
88
|
|
88
|
-
|
89
|
-
|
90
|
-
|
89
|
+
def hijack_methods
|
90
|
+
around_save :run_on_shard
|
91
|
+
after_initialize :set_current_shard
|
91
92
|
|
92
|
-
|
93
|
-
|
94
|
-
|
93
|
+
class << self
|
94
|
+
attr_accessor :custom_octopus_connection
|
95
|
+
attr_accessor :custom_octopus_table_name
|
95
96
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
97
|
+
alias_method_chain :connection, :octopus
|
98
|
+
alias_method_chain :connection_pool, :octopus
|
99
|
+
alias_method_chain :clear_all_connections!, :octopus
|
100
|
+
alias_method_chain :clear_active_connections!, :octopus
|
101
|
+
alias_method_chain :connected?, :octopus
|
101
102
|
|
102
|
-
|
103
|
-
alias_method_chain(:set_table_name, :octopus)
|
104
|
-
end
|
103
|
+
alias_method_chain(:set_table_name, :octopus) if Octopus.rails3?
|
105
104
|
|
106
|
-
|
107
|
-
|
108
|
-
|
105
|
+
def table_name=(value = nil)
|
106
|
+
self.custom_octopus_table_name = true
|
107
|
+
super
|
108
|
+
end
|
109
109
|
end
|
110
110
|
end
|
111
|
-
end
|
112
111
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
ActiveRecord::Base.class_variable_set :@@connection_proxy, p
|
119
|
-
p
|
120
|
-
end
|
121
|
-
end
|
112
|
+
def connection_proxy
|
113
|
+
ActiveRecord::Base.class_variable_defined?(:@@connection_proxy) &&
|
114
|
+
ActiveRecord::Base.class_variable_get(:@@connection_proxy) ||
|
115
|
+
ActiveRecord::Base.class_variable_set(:@@connection_proxy, Octopus::Proxy.new)
|
116
|
+
end
|
122
117
|
|
123
|
-
|
124
|
-
|
125
|
-
|
118
|
+
def should_use_normal_connection?
|
119
|
+
!Octopus.enabled? || custom_octopus_connection
|
120
|
+
end
|
126
121
|
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
122
|
+
def connection_with_octopus
|
123
|
+
if should_use_normal_connection?
|
124
|
+
connection_without_octopus
|
125
|
+
else
|
126
|
+
connection_proxy.current_model = self
|
127
|
+
connection_proxy
|
128
|
+
end
|
133
129
|
end
|
134
|
-
end
|
135
130
|
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
131
|
+
def connection_pool_with_octopus
|
132
|
+
if should_use_normal_connection?
|
133
|
+
connection_pool_without_octopus
|
134
|
+
else
|
135
|
+
connection_proxy.connection_pool
|
136
|
+
end
|
141
137
|
end
|
142
|
-
end
|
143
138
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
139
|
+
def clear_active_connections_with_octopus!
|
140
|
+
if should_use_normal_connection?
|
141
|
+
clear_active_connections_without_octopus!
|
142
|
+
else
|
143
|
+
connection_proxy.clear_active_connections!
|
144
|
+
end
|
149
145
|
end
|
150
|
-
end
|
151
146
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
147
|
+
def clear_all_connections_with_octopus!
|
148
|
+
if should_use_normal_connection?
|
149
|
+
clear_all_connections_without_octopus!
|
150
|
+
else
|
151
|
+
connection_proxy.clear_all_connections!
|
152
|
+
end
|
157
153
|
end
|
158
|
-
end
|
159
154
|
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
155
|
+
def connected_with_octopus?
|
156
|
+
if should_use_normal_connection?
|
157
|
+
connected_without_octopus?
|
158
|
+
else
|
159
|
+
connection_proxy.connected?
|
160
|
+
end
|
165
161
|
end
|
166
|
-
end
|
167
162
|
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
163
|
+
def set_table_name_with_octopus(value = nil, &block)
|
164
|
+
self.custom_octopus_table_name = true
|
165
|
+
set_table_name_without_octopus(value, &block)
|
166
|
+
end
|
172
167
|
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
168
|
+
def octopus_establish_connection(spec = ENV['DATABASE_URL'])
|
169
|
+
self.custom_octopus_connection = true if spec
|
170
|
+
establish_connection(spec)
|
171
|
+
end
|
177
172
|
|
178
|
-
|
179
|
-
|
180
|
-
|
173
|
+
def octopus_set_table_name(value = nil)
|
174
|
+
ActiveSupport::Deprecation.warn 'Calling `octopus_set_table_name` is deprecated and will be removed in Octopus 1.0.', caller
|
175
|
+
set_table_name(value)
|
176
|
+
end
|
181
177
|
end
|
182
178
|
end
|
183
179
|
end
|
data/lib/octopus/persistence.rb
CHANGED
data/lib/octopus/proxy.rb
CHANGED
@@ -1,437 +1,443 @@
|
|
1
|
-
require
|
1
|
+
require 'set'
|
2
2
|
require 'octopus/slave_group'
|
3
3
|
require 'octopus/load_balancing/round_robin'
|
4
4
|
|
5
|
-
|
6
|
-
|
5
|
+
module Octopus
|
6
|
+
class Proxy
|
7
|
+
attr_accessor :config, :sharded
|
7
8
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
def initialize(config = Octopus.config)
|
10
|
+
initialize_shards(config)
|
11
|
+
initialize_replication(config) if !config.nil? && config['replicated']
|
12
|
+
end
|
12
13
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
@shards_config.each do |key, value|
|
29
|
-
if value.is_a?(String)
|
30
|
-
value = resolve_string_connection(value).merge(:octopus_shard => key)
|
31
|
-
initialize_adapter(value['adapter'])
|
32
|
-
@shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
33
|
-
elsif value.is_a?(Hash) && value.has_key?("adapter")
|
34
|
-
value.merge!(:octopus_shard => key)
|
35
|
-
initialize_adapter(value['adapter'])
|
36
|
-
@shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
37
|
-
|
38
|
-
slave_group_configs = value.select do |k,v|
|
39
|
-
structurally_slave_group? v
|
40
|
-
end
|
14
|
+
def initialize_shards(config)
|
15
|
+
@shards = HashWithIndifferentAccess.new
|
16
|
+
@shards_slave_groups = HashWithIndifferentAccess.new
|
17
|
+
@slave_groups = HashWithIndifferentAccess.new
|
18
|
+
@groups = {}
|
19
|
+
@adapters = Set.new
|
20
|
+
@config = ActiveRecord::Base.connection_pool_without_octopus.connection.instance_variable_get(:@config)
|
21
|
+
|
22
|
+
unless config.nil?
|
23
|
+
@entire_sharded = config['entire_sharded']
|
24
|
+
@shards_config = config[Octopus.rails_env]
|
25
|
+
end
|
26
|
+
|
27
|
+
@shards_config ||= []
|
41
28
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
29
|
+
@shards_config.each do |key, value|
|
30
|
+
if value.is_a?(String)
|
31
|
+
value = resolve_string_connection(value).merge(:octopus_shard => key)
|
32
|
+
initialize_adapter(value['adapter'])
|
33
|
+
@shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
34
|
+
elsif value.is_a?(Hash) && value.key?('adapter')
|
35
|
+
value.merge!(:octopus_shard => key)
|
36
|
+
initialize_adapter(value['adapter'])
|
37
|
+
@shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
38
|
+
|
39
|
+
slave_group_configs = value.select do |_k, v|
|
40
|
+
structurally_slave_group? v
|
41
|
+
end
|
42
|
+
|
43
|
+
if slave_group_configs.present?
|
44
|
+
slave_groups = HashWithIndifferentAccess.new
|
45
|
+
slave_group_configs.each do |slave_group_name, slave_configs|
|
46
|
+
slaves = HashWithIndifferentAccess.new
|
47
|
+
slave_configs.each do |slave_name, slave_config|
|
48
|
+
@shards[slave_name.to_sym] = connection_pool_for(slave_config, "#{value['adapter']}_connection")
|
49
|
+
slaves[slave_name.to_sym] = slave_name.to_sym
|
50
|
+
end
|
51
|
+
slave_groups[slave_group_name.to_sym] = Octopus::SlaveGroup.new(slaves)
|
49
52
|
end
|
50
|
-
|
53
|
+
@shards_slave_groups[key.to_sym] = slave_groups
|
54
|
+
@sharded = true
|
51
55
|
end
|
52
|
-
|
53
|
-
@
|
54
|
-
end
|
55
|
-
elsif value.is_a?(Hash)
|
56
|
-
@groups[key.to_s] = []
|
56
|
+
elsif value.is_a?(Hash)
|
57
|
+
@groups[key.to_s] = []
|
57
58
|
|
58
|
-
|
59
|
-
|
59
|
+
value.each do |k, v|
|
60
|
+
fail 'You have duplicated shard names!' if @shards.key?(k.to_sym)
|
60
61
|
|
61
|
-
|
62
|
-
|
62
|
+
initialize_adapter(v['adapter'])
|
63
|
+
config_with_octopus_shard = v.merge(:octopus_shard => k)
|
63
64
|
|
64
|
-
|
65
|
-
|
66
|
-
|
65
|
+
@shards[k.to_sym] = connection_pool_for(config_with_octopus_shard, "#{v['adapter']}_connection")
|
66
|
+
@groups[key.to_s] << k.to_sym
|
67
|
+
end
|
67
68
|
|
68
|
-
|
69
|
-
|
70
|
-
|
69
|
+
if structurally_slave_group? value
|
70
|
+
slaves = Hash[@groups[key.to_s].map { |v| [v, v] }]
|
71
|
+
@slave_groups[key.to_sym] = Octopus::SlaveGroup.new(slaves)
|
72
|
+
end
|
71
73
|
end
|
72
74
|
end
|
75
|
+
|
76
|
+
@shards[:master] ||= ActiveRecord::Base.connection_pool_without_octopus
|
73
77
|
end
|
74
78
|
|
75
|
-
|
76
|
-
|
79
|
+
def initialize_replication(config)
|
80
|
+
@replicated = true
|
81
|
+
if config.key?('fully_replicated')
|
82
|
+
@fully_replicated = config['fully_replicated']
|
83
|
+
else
|
84
|
+
@fully_replicated = true
|
85
|
+
end
|
77
86
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
@fully_replicated = config["fully_replicated"]
|
82
|
-
else
|
83
|
-
@fully_replicated = true
|
87
|
+
@slaves_list = @shards.keys.map { |sym| sym.to_s }.sort
|
88
|
+
@slaves_list.delete('master')
|
89
|
+
@slaves_load_balancer = Octopus::LoadBalancing::RoundRobin.new(@slaves_list)
|
84
90
|
end
|
85
91
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
end
|
90
|
-
|
91
|
-
def current_model
|
92
|
-
Thread.current["octopus.current_model"]
|
93
|
-
end
|
92
|
+
def current_model
|
93
|
+
Thread.current['octopus.current_model']
|
94
|
+
end
|
94
95
|
|
95
|
-
|
96
|
-
|
97
|
-
|
96
|
+
def current_model=(model)
|
97
|
+
Thread.current['octopus.current_model'] = model.is_a?(ActiveRecord::Base) ? model.class : model
|
98
|
+
end
|
98
99
|
|
99
|
-
|
100
|
-
|
101
|
-
|
100
|
+
def current_shard
|
101
|
+
Thread.current['octopus.current_shard'] ||= :master
|
102
|
+
end
|
102
103
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
104
|
+
def current_shard=(shard_symbol)
|
105
|
+
self.current_slave_group = nil
|
106
|
+
if shard_symbol.is_a?(Array)
|
107
|
+
shard_symbol.each { |symbol| fail "Nonexistent Shard Name: #{symbol}" if @shards[symbol].nil? }
|
108
|
+
elsif shard_symbol.is_a?(Hash)
|
109
|
+
hash = shard_symbol
|
110
|
+
shard_symbol = hash[:shard]
|
111
|
+
slave_group_symbol = hash[:slave_group]
|
112
|
+
|
113
|
+
if shard_symbol.nil? && slave_group_symbol.nil?
|
114
|
+
fail 'Neither shard or slave group must be specified'
|
115
|
+
end
|
115
116
|
|
116
|
-
|
117
|
-
|
118
|
-
|
117
|
+
if shard_symbol.present?
|
118
|
+
fail "Nonexistent Shard Name: #{shard_symbol}" if @shards[shard_symbol].nil?
|
119
|
+
end
|
119
120
|
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
121
|
+
if slave_group_symbol.present?
|
122
|
+
if (@shards_slave_groups.try(:[], shard_symbol).present? && @shards_slave_groups[shard_symbol][slave_group_symbol].nil?) ||
|
123
|
+
(@shards_slave_groups.try(:[], shard_symbol).nil? && @slave_groups[slave_group_symbol].nil?)
|
124
|
+
fail "Nonexistent Slave Group Name: #{slave_group_symbol} in shards config: #{@shards_config.inspect}"
|
125
|
+
end
|
126
|
+
self.current_slave_group = slave_group_symbol
|
124
127
|
end
|
125
|
-
|
128
|
+
else
|
129
|
+
fail "Nonexistent Shard Name: #{shard_symbol}" if @shards[shard_symbol].nil?
|
126
130
|
end
|
127
|
-
|
128
|
-
|
131
|
+
|
132
|
+
Thread.current['octopus.current_shard'] = shard_symbol
|
129
133
|
end
|
130
134
|
|
131
|
-
|
132
|
-
|
135
|
+
def current_group
|
136
|
+
Thread.current['octopus.current_group']
|
137
|
+
end
|
133
138
|
|
134
|
-
|
135
|
-
|
136
|
-
|
139
|
+
def current_group=(group_symbol)
|
140
|
+
# TODO: Error message should include all groups if given more than one bad name.
|
141
|
+
[group_symbol].flatten.compact.each do |group|
|
142
|
+
fail "Nonexistent Group Name: #{group}" unless has_group?(group)
|
143
|
+
end
|
137
144
|
|
138
|
-
|
139
|
-
# TODO: Error message should include all groups if given more than one bad name.
|
140
|
-
[group_symbol].flatten.compact.each do |group|
|
141
|
-
raise "Nonexistent Group Name: #{group}" unless has_group?(group)
|
145
|
+
Thread.current['octopus.current_group'] = group_symbol
|
142
146
|
end
|
143
147
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
def current_slave_group
|
148
|
-
Thread.current["octopus.current_slave_group"]
|
149
|
-
end
|
148
|
+
def current_slave_group
|
149
|
+
Thread.current['octopus.current_slave_group']
|
150
|
+
end
|
150
151
|
|
151
|
-
|
152
|
-
|
153
|
-
|
152
|
+
def current_slave_group=(slave_group_symbol)
|
153
|
+
Thread.current['octopus.current_slave_group'] = slave_group_symbol
|
154
|
+
end
|
154
155
|
|
155
|
-
|
156
|
-
|
157
|
-
|
156
|
+
def block
|
157
|
+
Thread.current['octopus.block']
|
158
|
+
end
|
158
159
|
|
159
|
-
|
160
|
-
|
161
|
-
|
160
|
+
def block=(block)
|
161
|
+
Thread.current['octopus.block'] = block
|
162
|
+
end
|
162
163
|
|
163
|
-
|
164
|
-
|
165
|
-
|
164
|
+
def last_current_shard
|
165
|
+
Thread.current['octopus.last_current_shard']
|
166
|
+
end
|
166
167
|
|
167
|
-
|
168
|
-
|
169
|
-
|
168
|
+
def last_current_shard=(last_current_shard)
|
169
|
+
Thread.current['octopus.last_current_shard'] = last_current_shard
|
170
|
+
end
|
170
171
|
|
171
|
-
|
172
|
-
|
173
|
-
|
172
|
+
def fully_replicated?
|
173
|
+
@fully_replicated || Thread.current['octopus.fully_replicated']
|
174
|
+
end
|
174
175
|
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
176
|
+
# Public: Whether or not a group exists with the given name converted to a
|
177
|
+
# string.
|
178
|
+
#
|
179
|
+
# Returns a boolean.
|
180
|
+
def has_group?(group)
|
181
|
+
@groups.key?(group.to_s)
|
182
|
+
end
|
182
183
|
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
184
|
+
# Public: Retrieves names of all loaded shards.
|
185
|
+
#
|
186
|
+
# Returns an array of shard names as symbols
|
187
|
+
def shard_names
|
188
|
+
@shards.keys
|
189
|
+
end
|
189
190
|
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
191
|
+
# Public: Retrieves the defined shards for a given group.
|
192
|
+
#
|
193
|
+
# Returns an array of shard names as symbols or nil if the group is not
|
194
|
+
# defined.
|
195
|
+
def shards_for_group(group)
|
196
|
+
@groups.fetch(group.to_s, nil)
|
197
|
+
end
|
197
198
|
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
199
|
+
# Rails 3.1 sets automatic_reconnect to false when it removes
|
200
|
+
# connection pool. Octopus can potentially retain a reference to a closed
|
201
|
+
# connection pool. Previously, that would work since the pool would just
|
202
|
+
# reconnect, but in Rails 3.1 the flag prevents this.
|
203
|
+
def safe_connection(connection_pool)
|
204
|
+
connection_pool.automatic_reconnect ||= true
|
205
|
+
connection_pool.connection
|
206
|
+
end
|
206
207
|
|
207
|
-
|
208
|
-
|
209
|
-
|
208
|
+
def select_connection
|
209
|
+
safe_connection(@shards[shard_name])
|
210
|
+
end
|
210
211
|
|
211
|
-
|
212
|
-
|
213
|
-
|
212
|
+
def shard_name
|
213
|
+
current_shard.is_a?(Array) ? current_shard.first : current_shard
|
214
|
+
end
|
214
215
|
|
215
|
-
|
216
|
-
|
217
|
-
|
216
|
+
def should_clean_table_name?
|
217
|
+
@adapters.size > 1
|
218
|
+
end
|
218
219
|
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
220
|
+
def run_queries_on_shard(shard, &_block)
|
221
|
+
keeping_connection_proxy do
|
222
|
+
using_shard(shard) do
|
223
|
+
yield
|
224
|
+
end
|
223
225
|
end
|
224
226
|
end
|
225
|
-
end
|
226
227
|
|
227
|
-
|
228
|
-
|
229
|
-
|
228
|
+
def send_queries_to_multiple_shards(shards, &block)
|
229
|
+
shards.each do |shard|
|
230
|
+
run_queries_on_shard(shard, &block)
|
231
|
+
end
|
230
232
|
end
|
231
|
-
end
|
232
233
|
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
234
|
+
def clean_connection_proxy
|
235
|
+
self.current_shard = :master
|
236
|
+
self.current_group = nil
|
237
|
+
self.block = false
|
238
|
+
end
|
238
239
|
|
239
|
-
|
240
|
-
|
241
|
-
|
240
|
+
def check_schema_migrations(shard)
|
241
|
+
OctopusModel.using(shard).connection.table_exists?(
|
242
|
+
ActiveRecord::Migrator.schema_migrations_table_name,
|
243
|
+
) || OctopusModel.using(shard).connection.initialize_schema_migrations_table
|
242
244
|
end
|
243
|
-
end
|
244
245
|
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
246
|
+
def transaction(options = {}, &block)
|
247
|
+
replicated = @replicated && (current_model.replicated || fully_replicated?)
|
248
|
+
if !sharded && replicated
|
249
|
+
run_queries_on_shard(:master) do
|
250
|
+
select_connection.transaction(options, &block)
|
251
|
+
end
|
252
|
+
else
|
249
253
|
select_connection.transaction(options, &block)
|
250
254
|
end
|
251
|
-
else
|
252
|
-
select_connection.transaction(options, &block)
|
253
255
|
end
|
254
|
-
end
|
255
256
|
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
257
|
+
def method_missing(method, *args, &block)
|
258
|
+
if should_clean_connection_proxy?(method)
|
259
|
+
conn = select_connection
|
260
|
+
self.last_current_shard = current_shard
|
261
|
+
clean_connection_proxy
|
262
|
+
conn.send(method, *args, &block)
|
263
|
+
elsif should_send_queries_to_shard_slave_group?(method)
|
264
|
+
send_queries_to_shard_slave_group(method, *args, &block)
|
265
|
+
elsif should_send_queries_to_slave_group?(method)
|
266
|
+
send_queries_to_slave_group(method, *args, &block)
|
267
|
+
elsif should_send_queries_to_replicated_databases?(method)
|
268
|
+
send_queries_to_selected_slave(method, *args, &block)
|
269
|
+
else
|
270
|
+
select_connection.send(method, *args, &block)
|
271
|
+
end
|
270
272
|
end
|
271
|
-
end
|
272
273
|
|
273
|
-
|
274
|
-
|
275
|
-
|
274
|
+
def respond_to?(method, include_private = false)
|
275
|
+
super || select_connection.respond_to?(method, include_private)
|
276
|
+
end
|
276
277
|
|
277
|
-
|
278
|
-
|
279
|
-
|
278
|
+
def connection_pool
|
279
|
+
@shards[current_shard]
|
280
|
+
end
|
280
281
|
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
282
|
+
def enable_query_cache!
|
283
|
+
clear_query_cache
|
284
|
+
@shards.each { |_k, v| safe_connection(v).enable_query_cache! }
|
285
|
+
end
|
285
286
|
|
286
|
-
|
287
|
-
|
288
|
-
|
287
|
+
def disable_query_cache!
|
288
|
+
@shards.each { |_k, v| safe_connection(v).disable_query_cache! }
|
289
|
+
end
|
289
290
|
|
290
|
-
|
291
|
-
|
292
|
-
|
291
|
+
def clear_query_cache
|
292
|
+
@shards.each { |_k, v| safe_connection(v).clear_query_cache }
|
293
|
+
end
|
293
294
|
|
294
|
-
|
295
|
-
|
296
|
-
|
295
|
+
def clear_active_connections!
|
296
|
+
@shards.each { |_k, v| v.release_connection }
|
297
|
+
end
|
297
298
|
|
298
|
-
|
299
|
-
|
300
|
-
|
299
|
+
def clear_all_connections!
|
300
|
+
@shards.each { |_k, v| v.disconnect! }
|
301
|
+
end
|
301
302
|
|
302
|
-
|
303
|
-
|
304
|
-
|
303
|
+
def connected?
|
304
|
+
@shards.any? { |_k, v| v.connected? }
|
305
|
+
end
|
305
306
|
|
306
|
-
|
307
|
-
|
308
|
-
|
307
|
+
def should_send_queries_to_shard_slave_group?(method)
|
308
|
+
should_use_slaves_for_method?(method) && @shards_slave_groups.try(:[], current_shard).try(:[], current_slave_group).present?
|
309
|
+
end
|
309
310
|
|
310
|
-
|
311
|
-
|
312
|
-
|
311
|
+
def send_queries_to_shard_slave_group(method, *args, &block)
|
312
|
+
send_queries_to_balancer(@shards_slave_groups[current_shard][current_slave_group], method, *args, &block)
|
313
|
+
end
|
313
314
|
|
314
|
-
|
315
|
-
|
316
|
-
|
315
|
+
def should_send_queries_to_slave_group?(method)
|
316
|
+
should_use_slaves_for_method?(method) && @slave_groups.try(:[], current_slave_group).present?
|
317
|
+
end
|
317
318
|
|
318
|
-
|
319
|
-
|
320
|
-
|
319
|
+
def send_queries_to_slave_group(method, *args, &block)
|
320
|
+
send_queries_to_balancer(@slave_groups[current_slave_group], method, *args, &block)
|
321
|
+
end
|
321
322
|
|
322
|
-
|
323
|
+
protected
|
323
324
|
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
325
|
+
def connection_pool_for(adapter, config)
|
326
|
+
if Octopus.rails4?
|
327
|
+
arg = ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(adapter.dup, config)
|
328
|
+
else
|
329
|
+
arg = ActiveRecord::Base::ConnectionSpecification.new(adapter.dup, config)
|
330
|
+
end
|
331
|
+
|
332
|
+
ActiveRecord::ConnectionAdapters::ConnectionPool.new(arg)
|
329
333
|
end
|
330
334
|
|
331
|
-
|
332
|
-
|
335
|
+
def initialize_adapter(adapter)
|
336
|
+
@adapters << adapter
|
337
|
+
begin
|
338
|
+
require "active_record/connection_adapters/#{adapter}_adapter"
|
339
|
+
rescue LoadError
|
340
|
+
raise "Please install the #{adapter} adapter: `gem install activerecord-#{adapter}-adapter` (#{$ERROR_INFO})"
|
341
|
+
end
|
342
|
+
end
|
333
343
|
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
344
|
+
def resolve_string_connection(spec)
|
345
|
+
if Octopus.rails41?
|
346
|
+
resolver = ActiveRecord::ConnectionAdapters::ConnectionSpecification::Resolver.new({})
|
347
|
+
resolver.spec(spec).config.stringify_keys
|
348
|
+
else
|
349
|
+
if Octopus.rails4?
|
350
|
+
resolver = ActiveRecord::ConnectionAdapters::ConnectionSpecification::Resolver.new(spec, {})
|
351
|
+
else
|
352
|
+
resolver = ActiveRecord::Base::ConnectionSpecification::Resolver.new(spec, {})
|
353
|
+
end
|
354
|
+
resolver.spec.config.stringify_keys
|
355
|
+
end
|
340
356
|
end
|
341
|
-
end
|
342
357
|
|
343
|
-
|
344
|
-
|
345
|
-
resolver = ActiveRecord::ConnectionAdapters::ConnectionSpecification::Resolver.new(spec, {})
|
346
|
-
else
|
347
|
-
resolver = ActiveRecord::Base::ConnectionSpecification::Resolver.new(spec, {})
|
358
|
+
def should_clean_connection_proxy?(method)
|
359
|
+
method.to_s =~ /insert|select|execute/ && !@replicated && !block
|
348
360
|
end
|
349
|
-
resolver.spec.config.stringify_keys
|
350
|
-
end
|
351
361
|
|
352
|
-
|
353
|
-
method
|
354
|
-
|
362
|
+
# Try to use slaves if and only if `replicated: true` is specified in `shards.yml` and no slaves groups are defined
|
363
|
+
def should_send_queries_to_replicated_databases?(method)
|
364
|
+
@replicated && method.to_s =~ /select/ && !block && !slaves_grouped?
|
365
|
+
end
|
355
366
|
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
367
|
+
def send_queries_to_selected_slave(method, *args, &block)
|
368
|
+
if current_model.replicated || fully_replicated?
|
369
|
+
selected_slave = @slaves_load_balancer.next
|
370
|
+
else
|
371
|
+
selected_slave = :master
|
372
|
+
end
|
360
373
|
|
361
|
-
|
362
|
-
if current_model.replicated || fully_replicated?
|
363
|
-
selected_slave = @slaves_load_balancer.next
|
364
|
-
else
|
365
|
-
selected_slave = :master
|
374
|
+
send_queries_to_slave(selected_slave, method, *args, &block)
|
366
375
|
end
|
367
376
|
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
def should_use_slaves_for_method?(method)
|
381
|
-
@replicated && (current_model.replicated || fully_replicated?) && method.to_s =~ /select/
|
382
|
-
end
|
377
|
+
# We should use slaves if and only if its safe to do so.
|
378
|
+
#
|
379
|
+
# We can safely use slaves when:
|
380
|
+
# (1) `replicated: true` is specified in `shards.yml`
|
381
|
+
# (2) The current model is `replicated()`, or `fully_replicated: true` is specified in `shards.yml` which means that
|
382
|
+
# all the model is `replicated()`
|
383
|
+
# (3) It's a SELECT query
|
384
|
+
# while ensuring that we revert `current_shard` from the selected slave to the (shard's) master
|
385
|
+
# not to make queries other than SELECT leak to the slave.
|
386
|
+
def should_use_slaves_for_method?(method)
|
387
|
+
@replicated && (current_model.replicated || fully_replicated?) && method.to_s =~ /select/
|
388
|
+
end
|
383
389
|
|
384
|
-
|
385
|
-
|
386
|
-
|
390
|
+
def slaves_grouped?
|
391
|
+
@slave_groups.present?
|
392
|
+
end
|
387
393
|
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
394
|
+
# Temporarily switch `current_shard` to the next slave in a slave group and send queries to it
|
395
|
+
# while preserving `current_shard`
|
396
|
+
def send_queries_to_balancer(balancer, method, *args, &block)
|
397
|
+
send_queries_to_slave(balancer.next, method, *args, &block)
|
398
|
+
end
|
393
399
|
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
400
|
+
# Temporarily switch `current_shard` to the specified slave and send queries to it
|
401
|
+
# while preserving `current_shard`
|
402
|
+
def send_queries_to_slave(slave, method, *args, &block)
|
403
|
+
using_shard(slave) do
|
404
|
+
select_connection.send(method, *args, &block)
|
405
|
+
end
|
399
406
|
end
|
400
|
-
end
|
401
407
|
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
+
# Temporarily block cleaning connection proxy and run the block
|
409
|
+
#
|
410
|
+
# @see Octopus::Proxy#should_clean_connection?
|
411
|
+
# @see Octopus::Proxy#clean_connection_proxy
|
412
|
+
def keeping_connection_proxy(&_block)
|
413
|
+
last_block = block
|
408
414
|
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
415
|
+
begin
|
416
|
+
self.block = true
|
417
|
+
yield
|
418
|
+
ensure
|
419
|
+
self.block = last_block || false
|
420
|
+
end
|
414
421
|
end
|
415
|
-
end
|
416
422
|
|
417
|
-
|
418
|
-
|
419
|
-
|
423
|
+
# Temporarily switch `current_shard` and run the block
|
424
|
+
def using_shard(shard, &_block)
|
425
|
+
older_shard = current_shard
|
420
426
|
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
427
|
+
begin
|
428
|
+
self.current_shard = shard
|
429
|
+
yield
|
430
|
+
ensure
|
431
|
+
self.current_shard = older_shard
|
432
|
+
end
|
426
433
|
end
|
427
|
-
end
|
428
434
|
|
429
|
-
|
430
|
-
|
431
|
-
|
435
|
+
def structurally_slave?(config)
|
436
|
+
config.is_a?(Hash) && config.key?('adapter')
|
437
|
+
end
|
432
438
|
|
433
|
-
|
434
|
-
|
439
|
+
def structurally_slave_group?(config)
|
440
|
+
config.is_a?(Hash) && config.values.any? { |v| structurally_slave? v }
|
441
|
+
end
|
435
442
|
end
|
436
|
-
|
437
443
|
end
|