ar-octopus 0.9.0 → 0.9.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +10 -2
- data/Appraisals +4 -0
- data/README.mkdn +1 -1
- data/gemfiles/rails51.gemfile +7 -0
- data/lib/octopus.rb +11 -6
- data/lib/octopus/collection_association.rb +9 -3
- data/lib/octopus/finder_methods.rb +1 -1
- data/lib/octopus/load_balancing/round_robin.rb +1 -0
- data/lib/octopus/log_subscriber.rb +6 -2
- data/lib/octopus/migration.rb +25 -9
- data/lib/octopus/model.rb +17 -25
- data/lib/octopus/proxy.rb +78 -279
- data/lib/octopus/proxy_config.rb +252 -0
- data/lib/octopus/relation_proxy.rb +7 -1
- data/lib/octopus/scope_proxy.rb +1 -1
- data/lib/octopus/shard_tracking.rb +2 -1
- data/lib/octopus/version.rb +1 -1
- data/spec/migrations/10_create_users_using_replication.rb +1 -1
- data/spec/migrations/11_add_field_in_all_slaves.rb +1 -1
- data/spec/migrations/12_create_users_using_block.rb +1 -1
- data/spec/migrations/13_create_users_using_block_and_using.rb +1 -1
- data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +1 -1
- data/spec/migrations/15_create_user_on_shards_of_default_group_with_versions.rb +1 -1
- data/spec/migrations/1_create_users_on_master.rb +1 -1
- data/spec/migrations/2_create_users_on_canada.rb +1 -1
- data/spec/migrations/3_create_users_on_both_shards.rb +1 -1
- data/spec/migrations/4_create_users_on_shards_of_a_group.rb +1 -1
- data/spec/migrations/5_create_users_on_multiples_groups.rb +1 -1
- data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +1 -1
- data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +1 -1
- data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +1 -1
- data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +1 -1
- data/spec/octopus/association_shard_tracking_spec.rb +15 -15
- data/spec/octopus/load_balancing/round_robin_spec.rb +15 -0
- data/spec/octopus/log_subscriber_spec.rb +1 -1
- data/spec/octopus/model_spec.rb +34 -5
- data/spec/octopus/octopus_spec.rb +1 -1
- data/spec/octopus/proxy_spec.rb +16 -38
- data/spec/octopus/relation_proxy_spec.rb +4 -0
- data/spec/octopus/replication_spec.rb +5 -1
- data/spec/spec_helper.rb +2 -0
- data/spec/support/octopus_helper.rb +1 -2
- data/spec/tasks/octopus.rake_spec.rb +2 -4
- metadata +7 -6
- data/.ruby-version +0 -1
- data/init.rb +0 -1
- data/rails/init.rb +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b11a47088e651be24f08e79859ff992113aafa0d
|
4
|
+
data.tar.gz: 9a33b695f7736f5ea3fecf905442eb26743a1c1d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 773b9c6d05403653ab26fc992daddfd9ac62d17e965ab0e980f8f052c450c8bde69951471993b03d56bac32c1015bcc42e7395876b91cc2df30a9a3892199778
|
7
|
+
data.tar.gz: f185670274cafadefa2c14390a32cd2e8dbeae2ca59059b0c07cd4f2f93d215c5eefa0ff29783281cc6ff761bc918f3f984dc6ef19b6eb84554dc9056b5370a1
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -4,14 +4,22 @@ env:
|
|
4
4
|
before_script:
|
5
5
|
- "bundle exec rake db:prepare"
|
6
6
|
rvm:
|
7
|
-
- 2.2.
|
8
|
-
- 2.3.
|
7
|
+
- 2.2.7
|
8
|
+
- 2.3.4
|
9
|
+
- 2.4.1
|
9
10
|
gemfile:
|
10
11
|
- gemfiles/rails4.gemfile
|
11
12
|
- gemfiles/rails41.gemfile
|
12
13
|
- gemfiles/rails42.gemfile
|
13
14
|
- gemfiles/rails5.gemfile
|
15
|
+
- gemfiles/rails51.gemfile
|
14
16
|
notifications:
|
15
17
|
recipients:
|
16
18
|
- gabriel.sobrinho@gmail.com
|
17
19
|
- thiago.pradi@gmail.com
|
20
|
+
matrix:
|
21
|
+
exclude:
|
22
|
+
- rvm: 2.4.1
|
23
|
+
gemfile: gemfiles/rails4.gemfile
|
24
|
+
- rvm: 2.4.1
|
25
|
+
gemfile: gemfiles/rails41.gemfile
|
data/Appraisals
CHANGED
data/README.mkdn
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
Octopus is a better way to do Database Sharding in ActiveRecord. Sharding allows multiple databases in the same rails application. While there are several projects that implement Sharding (e.g. DbCharmer, DataFabric, MultiDb), each project has its own limitations. The main goal of octopus project is to provide a better way of doing Database Sharding.
|
6
6
|
|
7
7
|
## Feature list:
|
8
|
-
The api is designed to be simple as possible. Octopus focuses on the end user, giving the power of multiple databases but with reliable code and flexibility. Octopus is compatible with Rails
|
8
|
+
The api is designed to be simple as possible. Octopus focuses on the end user, giving the power of multiple databases but with reliable code and flexibility. Octopus is compatible with Rails 4 and Rails 5.
|
9
9
|
|
10
10
|
Octopus supports:
|
11
11
|
|
data/lib/octopus.rb
CHANGED
@@ -102,8 +102,12 @@ module Octopus
|
|
102
102
|
rails4? && ActiveRecord::VERSION::MINOR == 2
|
103
103
|
end
|
104
104
|
|
105
|
-
def self.
|
106
|
-
ActiveRecord::VERSION::MAJOR == 5
|
105
|
+
def self.rails50?
|
106
|
+
ActiveRecord::VERSION::MAJOR == 5 && ActiveRecord::VERSION::MINOR == 0
|
107
|
+
end
|
108
|
+
|
109
|
+
def self.rails51?
|
110
|
+
ActiveRecord::VERSION::MAJOR == 5 && ActiveRecord::VERSION::MINOR == 1
|
107
111
|
end
|
108
112
|
|
109
113
|
attr_writer :logger
|
@@ -152,11 +156,11 @@ module Octopus
|
|
152
156
|
end
|
153
157
|
|
154
158
|
def self.fully_replicated(&_block)
|
155
|
-
old_fully_replicated = Thread.current[Octopus::
|
156
|
-
Thread.current[Octopus::
|
159
|
+
old_fully_replicated = Thread.current[Octopus::ProxyConfig::FULLY_REPLICATED_KEY]
|
160
|
+
Thread.current[Octopus::ProxyConfig::FULLY_REPLICATED_KEY] = true
|
157
161
|
yield
|
158
162
|
ensure
|
159
|
-
Thread.current[Octopus::
|
163
|
+
Thread.current[Octopus::ProxyConfig::FULLY_REPLICATED_KEY] = old_fully_replicated
|
160
164
|
end
|
161
165
|
end
|
162
166
|
|
@@ -170,7 +174,7 @@ require 'octopus/model'
|
|
170
174
|
require 'octopus/migration'
|
171
175
|
require 'octopus/association'
|
172
176
|
require 'octopus/collection_association'
|
173
|
-
require 'octopus/has_and_belongs_to_many_association' unless Octopus.rails41? || Octopus.
|
177
|
+
require 'octopus/has_and_belongs_to_many_association' unless Octopus.rails41? || Octopus.rails50? || Octopus.rails51?
|
174
178
|
require 'octopus/association_shard_tracking'
|
175
179
|
require 'octopus/persistence'
|
176
180
|
require 'octopus/log_subscriber'
|
@@ -180,6 +184,7 @@ require 'octopus/finder_methods'
|
|
180
184
|
|
181
185
|
require 'octopus/railtie' if defined?(::Rails::Railtie)
|
182
186
|
|
187
|
+
require 'octopus/proxy_config'
|
183
188
|
require 'octopus/proxy'
|
184
189
|
require 'octopus/collection_proxy'
|
185
190
|
require 'octopus/relation_proxy'
|
@@ -1,9 +1,15 @@
|
|
1
1
|
module Octopus
|
2
2
|
module CollectionAssociation
|
3
3
|
def self.included(base)
|
4
|
-
|
5
|
-
|
6
|
-
|
4
|
+
if Octopus.rails51?
|
5
|
+
base.sharded_methods :reader, :writer, :ids_reader, :ids_writer, :create, :create!,
|
6
|
+
:build, :include?,
|
7
|
+
:load_target, :reload, :size, :select
|
8
|
+
else
|
9
|
+
base.sharded_methods :reader, :writer, :ids_reader, :ids_writer, :create, :create!,
|
10
|
+
:build, :any?, :count, :empty?, :first, :include?, :last, :length,
|
11
|
+
:load_target, :many?, :reload, :size, :select, :uniq
|
12
|
+
end
|
7
13
|
end
|
8
14
|
end
|
9
15
|
end
|
@@ -3,8 +3,12 @@ module Octopus
|
|
3
3
|
module LogSubscriber
|
4
4
|
def self.included(base)
|
5
5
|
base.send(:attr_accessor, :octopus_shard)
|
6
|
-
|
7
|
-
base.
|
6
|
+
|
7
|
+
base.send :alias_method, :sql_without_octopus_shard, :sql
|
8
|
+
base.send :alias_method, :sql, :sql_with_octopus_shard
|
9
|
+
|
10
|
+
base.send :alias_method, :debug_without_octopus_shard, :debug
|
11
|
+
base.send :alias_method, :debug, :debug_with_octopus_shard
|
8
12
|
end
|
9
13
|
|
10
14
|
def sql_with_octopus_shard(event)
|
data/lib/octopus/migration.rb
CHANGED
@@ -19,7 +19,9 @@ module Octopus
|
|
19
19
|
def self.included(base)
|
20
20
|
base.extend(ClassMethods)
|
21
21
|
|
22
|
-
base.
|
22
|
+
base.send :alias_method, :announce_without_octopus, :announce
|
23
|
+
base.send :alias_method, :announce, :announce_with_octopus
|
24
|
+
|
23
25
|
base.class_attribute :current_shard, :current_group, :current_group_specified, :instance_reader => false, :instance_writer => false
|
24
26
|
end
|
25
27
|
|
@@ -64,16 +66,28 @@ module Octopus
|
|
64
66
|
|
65
67
|
base.class_eval do
|
66
68
|
class << self
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
69
|
+
alias_method :migrate_without_octopus, :migrate
|
70
|
+
alias_method :migrate, :migrate_with_octopus
|
71
|
+
|
72
|
+
alias_method :up_without_octopus, :up
|
73
|
+
alias_method :up, :up_with_octopus
|
74
|
+
|
75
|
+
alias_method :down_without_octopus, :down
|
76
|
+
alias_method :down, :down_with_octopus
|
77
|
+
|
78
|
+
alias_method :run_without_octopus, :run
|
79
|
+
alias_method :run, :run_with_octopus
|
71
80
|
end
|
72
81
|
end
|
73
82
|
|
74
|
-
base.
|
75
|
-
base.
|
76
|
-
|
83
|
+
base.send :alias_method, :run_without_octopus, :run
|
84
|
+
base.send :alias_method, :run, :run_with_octopus
|
85
|
+
|
86
|
+
base.send :alias_method, :migrate_without_octopus, :migrate
|
87
|
+
base.send :alias_method, :migrate, :migrate_with_octopus
|
88
|
+
|
89
|
+
base.send :alias_method, :migrations_without_octopus, :migrations
|
90
|
+
base.send :alias_method, :migrations, :migrations_with_octopus
|
77
91
|
end
|
78
92
|
|
79
93
|
def run_with_octopus(&block)
|
@@ -151,7 +165,9 @@ end
|
|
151
165
|
module Octopus
|
152
166
|
module UnknownMigrationVersionError
|
153
167
|
def self.included(base)
|
154
|
-
base.
|
168
|
+
base.send :alias_method, :initialize_without_octopus, :initialize
|
169
|
+
base.send :alias_method, :initialize, :initialize_with_octopus
|
170
|
+
|
155
171
|
base.send(:attr_accessor, :version)
|
156
172
|
end
|
157
173
|
|
data/lib/octopus/model.rb
CHANGED
@@ -9,17 +9,6 @@ module Octopus
|
|
9
9
|
end
|
10
10
|
|
11
11
|
module SharedMethods
|
12
|
-
def clean_table_name
|
13
|
-
return unless connection_proxy.should_clean_table_name?
|
14
|
-
|
15
|
-
if self != ActiveRecord::Base && self.respond_to?(:reset_table_name) && !custom_octopus_table_name
|
16
|
-
reset_table_name
|
17
|
-
end
|
18
|
-
|
19
|
-
reset_column_information
|
20
|
-
instance_variable_set(:@quoted_table_name, nil)
|
21
|
-
end
|
22
|
-
|
23
12
|
def using(shard)
|
24
13
|
if block_given?
|
25
14
|
raise Octopus::Exception, <<-EOF
|
@@ -30,7 +19,6 @@ If you are trying to scope everything to a specific shard, use Octopus.using ins
|
|
30
19
|
end
|
31
20
|
|
32
21
|
if Octopus.enabled?
|
33
|
-
clean_table_name
|
34
22
|
Octopus::ScopeProxy.new(shard, self)
|
35
23
|
else
|
36
24
|
self
|
@@ -45,18 +33,13 @@ If you are trying to scope everything to a specific shard, use Octopus.using ins
|
|
45
33
|
base.send(:alias_method, :equality_without_octopus, :==)
|
46
34
|
base.send(:alias_method, :==, :equality_with_octopus)
|
47
35
|
base.send(:alias_method, :eql?, :==)
|
48
|
-
base.send(:
|
36
|
+
base.send(:alias_method, :perform_validations_without_octopus, :perform_validations)
|
37
|
+
base.send(:alias_method, :perform_validations, :perform_validations_with_octopus)
|
49
38
|
end
|
50
39
|
|
51
40
|
def set_current_shard
|
52
41
|
return unless Octopus.enabled?
|
53
|
-
|
54
|
-
if new_record? || self.class.connection_proxy.block
|
55
|
-
shard = self.class.connection_proxy.current_shard
|
56
|
-
else
|
57
|
-
shard = self.class.connection_proxy.last_current_shard || self.class.connection_proxy.current_shard
|
58
|
-
end
|
59
|
-
|
42
|
+
shard = self.class.connection_proxy.current_shard
|
60
43
|
self.current_shard = shard if self.class.allowed_shard?(shard)
|
61
44
|
end
|
62
45
|
|
@@ -111,11 +94,20 @@ If you are trying to scope everything to a specific shard, use Octopus.using ins
|
|
111
94
|
class << self
|
112
95
|
attr_accessor :custom_octopus_table_name
|
113
96
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
97
|
+
alias_method :connection_without_octopus, :connection
|
98
|
+
alias_method :connection, :connection_with_octopus
|
99
|
+
|
100
|
+
alias_method :connection_pool_without_octopus, :connection_pool
|
101
|
+
alias_method :connection_pool, :connection_pool_with_octopus
|
102
|
+
|
103
|
+
alias_method :clear_all_connections_without_octopus!, :clear_all_connections!
|
104
|
+
alias_method :clear_all_connections!, :clear_all_connections_with_octopus!
|
105
|
+
|
106
|
+
alias_method :clear_active_connections_without_octopus!, :clear_active_connections!
|
107
|
+
alias_method :clear_active_connections!, :clear_active_connections_with_octopus!
|
108
|
+
|
109
|
+
alias_method :connected_without_octopus?, :connected?
|
110
|
+
alias_method :connected?, :connected_with_octopus?
|
119
111
|
|
120
112
|
def table_name=(value = nil)
|
121
113
|
self.custom_octopus_table_name = true
|
data/lib/octopus/proxy.rb
CHANGED
@@ -4,216 +4,63 @@ require 'octopus/load_balancing/round_robin'
|
|
4
4
|
|
5
5
|
module Octopus
|
6
6
|
class Proxy
|
7
|
-
attr_accessor :
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
7
|
+
attr_accessor :proxy_config
|
8
|
+
|
9
|
+
delegate :current_model, :current_model=,
|
10
|
+
:current_shard, :current_shard=,
|
11
|
+
:current_group, :current_group=,
|
12
|
+
:current_slave_group, :current_slave_group=,
|
13
|
+
:current_load_balance_options, :current_load_balance_options=,
|
14
|
+
:block, :block=, :fully_replicated?, :has_group?,
|
15
|
+
:shard_names, :shards_for_group, :shards, :sharded, :slaves_list,
|
16
|
+
:shards_slave_groups, :slave_groups, :replicated, :slaves_load_balancer,
|
17
|
+
:config, :initialize_shards, :shard_name, to: :proxy_config, prefix: false
|
17
18
|
|
18
19
|
def initialize(config = Octopus.config)
|
19
|
-
|
20
|
-
initialize_replication(config) if !config.nil? && config['replicated']
|
21
|
-
end
|
22
|
-
|
23
|
-
def initialize_shards(config)
|
24
|
-
@shards = HashWithIndifferentAccess.new
|
25
|
-
@shards_slave_groups = HashWithIndifferentAccess.new
|
26
|
-
@slave_groups = HashWithIndifferentAccess.new
|
27
|
-
@groups = {}
|
28
|
-
@adapters = Set.new
|
29
|
-
@config = ActiveRecord::Base.connection_pool_without_octopus.spec.config
|
30
|
-
|
31
|
-
unless config.nil?
|
32
|
-
@entire_sharded = config['entire_sharded']
|
33
|
-
@shards_config = config[Octopus.rails_env]
|
34
|
-
end
|
35
|
-
|
36
|
-
@shards_config ||= []
|
37
|
-
|
38
|
-
@shards_config.each do |key, value|
|
39
|
-
if value.is_a?(String)
|
40
|
-
value = resolve_string_connection(value).merge(:octopus_shard => key)
|
41
|
-
initialize_adapter(value['adapter'])
|
42
|
-
@shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
43
|
-
elsif value.is_a?(Hash) && value.key?('adapter')
|
44
|
-
value.merge!(:octopus_shard => key)
|
45
|
-
initialize_adapter(value['adapter'])
|
46
|
-
@shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
47
|
-
|
48
|
-
slave_group_configs = value.select do |_k, v|
|
49
|
-
structurally_slave_group? v
|
50
|
-
end
|
51
|
-
|
52
|
-
if slave_group_configs.present?
|
53
|
-
slave_groups = HashWithIndifferentAccess.new
|
54
|
-
slave_group_configs.each do |slave_group_name, slave_configs|
|
55
|
-
slaves = HashWithIndifferentAccess.new
|
56
|
-
slave_configs.each do |slave_name, slave_config|
|
57
|
-
@shards[slave_name.to_sym] = connection_pool_for(slave_config, "#{value['adapter']}_connection")
|
58
|
-
slaves[slave_name.to_sym] = slave_name.to_sym
|
59
|
-
end
|
60
|
-
slave_groups[slave_group_name.to_sym] = Octopus::SlaveGroup.new(slaves)
|
61
|
-
end
|
62
|
-
@shards_slave_groups[key.to_sym] = slave_groups
|
63
|
-
@sharded = true
|
64
|
-
end
|
65
|
-
elsif value.is_a?(Hash)
|
66
|
-
@groups[key.to_s] = []
|
67
|
-
|
68
|
-
value.each do |k, v|
|
69
|
-
fail 'You have duplicated shard names!' if @shards.key?(k.to_sym)
|
70
|
-
|
71
|
-
initialize_adapter(v['adapter'])
|
72
|
-
config_with_octopus_shard = v.merge(:octopus_shard => k)
|
73
|
-
|
74
|
-
@shards[k.to_sym] = connection_pool_for(config_with_octopus_shard, "#{v['adapter']}_connection")
|
75
|
-
@groups[key.to_s] << k.to_sym
|
76
|
-
end
|
77
|
-
|
78
|
-
if structurally_slave_group? value
|
79
|
-
slaves = Hash[@groups[key.to_s].map { |v| [v, v] }]
|
80
|
-
@slave_groups[key.to_sym] = Octopus::SlaveGroup.new(slaves)
|
81
|
-
end
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
@shards[:master] ||= ActiveRecord::Base.connection_pool_without_octopus if Octopus.master_shard == :master
|
86
|
-
end
|
87
|
-
|
88
|
-
def initialize_replication(config)
|
89
|
-
@replicated = true
|
90
|
-
if config.key?('fully_replicated')
|
91
|
-
@fully_replicated = config['fully_replicated']
|
92
|
-
else
|
93
|
-
@fully_replicated = true
|
94
|
-
end
|
95
|
-
|
96
|
-
@slaves_list = @shards.keys.map(&:to_s).sort
|
97
|
-
@slaves_list.delete('master')
|
98
|
-
@slaves_load_balancer = Octopus.load_balancer.new(@slaves_list)
|
99
|
-
end
|
100
|
-
|
101
|
-
def current_model
|
102
|
-
Thread.current[CURRENT_MODEL_KEY]
|
103
|
-
end
|
104
|
-
|
105
|
-
def current_model=(model)
|
106
|
-
Thread.current[CURRENT_MODEL_KEY] = model.is_a?(ActiveRecord::Base) ? model.class : model
|
20
|
+
self.proxy_config = Octopus::ProxyConfig.new(config)
|
107
21
|
end
|
108
22
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
load_balance_options = hash[:load_balance_options]
|
122
|
-
|
123
|
-
if shard_symbol.nil? && slave_group_symbol.nil?
|
124
|
-
fail 'Neither shard or slave group must be specified'
|
125
|
-
end
|
23
|
+
# Rails Connection Methods - Those methods are overriden to add custom behavior that helps
|
24
|
+
# Octopus introduce Sharding / Replication.
|
25
|
+
delegate :adapter_name, :add_transaction_record, :case_sensitive_modifier,
|
26
|
+
:type_cast, :to_sql, :quote, :quote_column_name, :quote_table_name,
|
27
|
+
:quote_table_name_for_assignment, :supports_migrations?, :table_alias_for,
|
28
|
+
:table_exists?, :in_clause_length, :supports_ddl_transactions?,
|
29
|
+
:sanitize_limit, :prefetch_primary_key?, :current_database, :initialize_schema_migrations_table,
|
30
|
+
:combine_bind_parameters, :empty_insert_statement_value, :assume_migrated_upto_version,
|
31
|
+
:schema_cache, :substitute_at, :internal_string_options_for_primary_key, :lookup_cast_type_from_column,
|
32
|
+
:supports_advisory_locks?, :get_advisory_lock, :initialize_internal_metadata_table,
|
33
|
+
:release_advisory_lock, :prepare_binds_for_database, :cacheable_query, :column_name_for_operation,
|
34
|
+
:prepared_statements, :transaction_state, :create_table, to: :select_connection
|
126
35
|
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
if slave_group_symbol.present?
|
132
|
-
if (@shards_slave_groups.try(:[], shard_symbol).present? && @shards_slave_groups[shard_symbol][slave_group_symbol].nil?) ||
|
133
|
-
(@shards_slave_groups.try(:[], shard_symbol).nil? && @slave_groups[slave_group_symbol].nil?)
|
134
|
-
fail "Nonexistent Slave Group Name: #{slave_group_symbol} in shards config: #{@shards_config.inspect}"
|
135
|
-
end
|
136
|
-
end
|
137
|
-
self.current_slave_group = slave_group_symbol
|
138
|
-
self.current_load_balance_options = load_balance_options
|
139
|
-
else
|
140
|
-
fail "Nonexistent Shard Name: #{shard_symbol}" if @shards[shard_symbol].nil?
|
141
|
-
end
|
142
|
-
|
143
|
-
Thread.current[CURRENT_SHARD_KEY] = shard_symbol
|
36
|
+
def execute(sql, name = nil)
|
37
|
+
conn = select_connection
|
38
|
+
clean_connection_proxy
|
39
|
+
conn.execute(sql, name)
|
144
40
|
end
|
145
41
|
|
146
|
-
def
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
def current_group=(group_symbol)
|
151
|
-
# TODO: Error message should include all groups if given more than one bad name.
|
152
|
-
[group_symbol].flatten.compact.each do |group|
|
153
|
-
fail "Nonexistent Group Name: #{group}" unless has_group?(group)
|
154
|
-
end
|
155
|
-
|
156
|
-
Thread.current[CURRENT_GROUP_KEY] = group_symbol
|
42
|
+
def insert(arel, name = nil, pk = nil, id_value = nil, sequence_name = nil, binds = [])
|
43
|
+
conn = select_connection
|
44
|
+
clean_connection_proxy
|
45
|
+
conn.insert(arel, name, pk, id_value, sequence_name, binds)
|
157
46
|
end
|
158
47
|
|
159
|
-
def
|
160
|
-
|
48
|
+
def update(arel, name = nil, binds = [])
|
49
|
+
conn = select_connection
|
50
|
+
clean_connection_proxy
|
51
|
+
conn.update(arel, name, binds)
|
161
52
|
end
|
162
53
|
|
163
|
-
def
|
164
|
-
|
165
|
-
Thread.current[CURRENT_LOAD_BALANCE_OPTIONS_KEY] = nil if slave_group_symbol.nil?
|
54
|
+
def delete(*args, &block)
|
55
|
+
legacy_method_missing_logic('delete', *args, &block)
|
166
56
|
end
|
167
57
|
|
168
|
-
def
|
169
|
-
|
58
|
+
def select_all(*args, &block)
|
59
|
+
legacy_method_missing_logic('select_all', *args, &block)
|
170
60
|
end
|
171
61
|
|
172
|
-
def
|
173
|
-
|
174
|
-
end
|
175
|
-
|
176
|
-
def block
|
177
|
-
Thread.current[BLOCK_KEY]
|
178
|
-
end
|
179
|
-
|
180
|
-
def block=(block)
|
181
|
-
Thread.current[BLOCK_KEY] = block
|
182
|
-
end
|
183
|
-
|
184
|
-
def last_current_shard
|
185
|
-
Thread.current[LAST_CURRENT_SHARD_KEY]
|
186
|
-
end
|
187
|
-
|
188
|
-
def last_current_shard=(last_current_shard)
|
189
|
-
Thread.current[LAST_CURRENT_SHARD_KEY] = last_current_shard
|
190
|
-
end
|
191
|
-
|
192
|
-
def fully_replicated?
|
193
|
-
@fully_replicated || Thread.current[FULLY_REPLICATED_KEY]
|
194
|
-
end
|
195
|
-
|
196
|
-
# Public: Whether or not a group exists with the given name converted to a
|
197
|
-
# string.
|
198
|
-
#
|
199
|
-
# Returns a boolean.
|
200
|
-
def has_group?(group)
|
201
|
-
@groups.key?(group.to_s)
|
202
|
-
end
|
203
|
-
|
204
|
-
# Public: Retrieves names of all loaded shards.
|
205
|
-
#
|
206
|
-
# Returns an array of shard names as symbols
|
207
|
-
def shard_names
|
208
|
-
@shards.keys
|
209
|
-
end
|
210
|
-
|
211
|
-
# Public: Retrieves the defined shards for a given group.
|
212
|
-
#
|
213
|
-
# Returns an array of shard names as symbols or nil if the group is not
|
214
|
-
# defined.
|
215
|
-
def shards_for_group(group)
|
216
|
-
@groups.fetch(group.to_s, nil)
|
62
|
+
def select_value(*args, &block)
|
63
|
+
legacy_method_missing_logic('select_value', *args, &block)
|
217
64
|
end
|
218
65
|
|
219
66
|
# Rails 3.1 sets automatic_reconnect to false when it removes
|
@@ -222,22 +69,14 @@ module Octopus
|
|
222
69
|
# reconnect, but in Rails 3.1 the flag prevents this.
|
223
70
|
def safe_connection(connection_pool)
|
224
71
|
connection_pool.automatic_reconnect ||= true
|
225
|
-
if !connection_pool.connected? &&
|
72
|
+
if !connection_pool.connected? && shards[Octopus.master_shard].connection.query_cache_enabled
|
226
73
|
connection_pool.connection.enable_query_cache!
|
227
74
|
end
|
228
75
|
connection_pool.connection
|
229
76
|
end
|
230
77
|
|
231
78
|
def select_connection
|
232
|
-
safe_connection(
|
233
|
-
end
|
234
|
-
|
235
|
-
def shard_name
|
236
|
-
current_shard.is_a?(Array) ? current_shard.first : current_shard
|
237
|
-
end
|
238
|
-
|
239
|
-
def should_clean_table_name?
|
240
|
-
@adapters.size > 1
|
79
|
+
safe_connection(shards[shard_name])
|
241
80
|
end
|
242
81
|
|
243
82
|
def run_queries_on_shard(shard, &_block)
|
@@ -261,7 +100,7 @@ module Octopus
|
|
261
100
|
end
|
262
101
|
|
263
102
|
def send_queries_to_all_shards(&block)
|
264
|
-
send_queries_to_multiple_shards(shard_names.uniq { |shard_name|
|
103
|
+
send_queries_to_multiple_shards(shard_names.uniq { |shard_name| shards[shard_name] }, &block)
|
265
104
|
end
|
266
105
|
|
267
106
|
def clean_connection_proxy
|
@@ -288,20 +127,7 @@ module Octopus
|
|
288
127
|
end
|
289
128
|
|
290
129
|
def method_missing(method, *args, &block)
|
291
|
-
|
292
|
-
conn = select_connection
|
293
|
-
self.last_current_shard = current_shard
|
294
|
-
clean_connection_proxy
|
295
|
-
conn.send(method, *args, &block)
|
296
|
-
elsif should_send_queries_to_shard_slave_group?(method)
|
297
|
-
send_queries_to_shard_slave_group(method, *args, &block)
|
298
|
-
elsif should_send_queries_to_slave_group?(method)
|
299
|
-
send_queries_to_slave_group(method, *args, &block)
|
300
|
-
elsif should_send_queries_to_replicated_databases?(method)
|
301
|
-
send_queries_to_selected_slave(method, *args, &block)
|
302
|
-
else
|
303
|
-
select_connection.send(method, *args, &block)
|
304
|
-
end
|
130
|
+
legacy_method_missing_logic(method, *args, &block)
|
305
131
|
end
|
306
132
|
|
307
133
|
def respond_to?(method, include_private = false)
|
@@ -309,7 +135,7 @@ module Octopus
|
|
309
135
|
end
|
310
136
|
|
311
137
|
def connection_pool
|
312
|
-
|
138
|
+
shards[current_shard]
|
313
139
|
end
|
314
140
|
|
315
141
|
def enable_query_cache!
|
@@ -334,30 +160,49 @@ module Octopus
|
|
334
160
|
end
|
335
161
|
|
336
162
|
def connected?
|
337
|
-
|
163
|
+
shards.any? { |_k, v| v.connected? }
|
338
164
|
end
|
339
165
|
|
340
166
|
def should_send_queries_to_shard_slave_group?(method)
|
341
|
-
should_use_slaves_for_method?(method) &&
|
167
|
+
should_use_slaves_for_method?(method) && shards_slave_groups.try(:[], current_shard).try(:[], current_slave_group).present?
|
342
168
|
end
|
343
169
|
|
344
170
|
def send_queries_to_shard_slave_group(method, *args, &block)
|
345
|
-
send_queries_to_balancer(
|
171
|
+
send_queries_to_balancer(shards_slave_groups[current_shard][current_slave_group], method, *args, &block)
|
346
172
|
end
|
347
173
|
|
348
174
|
def should_send_queries_to_slave_group?(method)
|
349
|
-
should_use_slaves_for_method?(method) &&
|
175
|
+
should_use_slaves_for_method?(method) && slave_groups.try(:[], current_slave_group).present?
|
350
176
|
end
|
351
177
|
|
352
178
|
def send_queries_to_slave_group(method, *args, &block)
|
353
|
-
send_queries_to_balancer(
|
179
|
+
send_queries_to_balancer(slave_groups[current_slave_group], method, *args, &block)
|
354
180
|
end
|
355
181
|
|
356
182
|
protected
|
357
183
|
|
184
|
+
# @thiagopradi - This legacy method missing logic will be keep for a while for compatibility
|
185
|
+
# and will be removed when Octopus 1.0 will be released.
|
186
|
+
# We are planning to migrate to a much stable logic for the Proxy that doesn't require method missing.
|
187
|
+
def legacy_method_missing_logic(method, *args, &block)
|
188
|
+
if should_clean_connection_proxy?(method)
|
189
|
+
conn = select_connection
|
190
|
+
clean_connection_proxy
|
191
|
+
conn.send(method, *args, &block)
|
192
|
+
elsif should_send_queries_to_shard_slave_group?(method)
|
193
|
+
send_queries_to_shard_slave_group(method, *args, &block)
|
194
|
+
elsif should_send_queries_to_slave_group?(method)
|
195
|
+
send_queries_to_slave_group(method, *args, &block)
|
196
|
+
elsif should_send_queries_to_replicated_databases?(method)
|
197
|
+
send_queries_to_selected_slave(method, *args, &block)
|
198
|
+
else
|
199
|
+
select_connection.send(method, *args, &block)
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
358
203
|
# Ensure that a single failing slave doesn't take down the entire application
|
359
204
|
def with_each_healthy_shard
|
360
|
-
|
205
|
+
shards.each do |shard_name, v|
|
361
206
|
begin
|
362
207
|
yield(v)
|
363
208
|
rescue => e
|
@@ -369,17 +214,10 @@ module Octopus
|
|
369
214
|
end
|
370
215
|
end
|
371
216
|
|
372
|
-
|
373
|
-
if conn_handler.respond_to?(:connection_pool_list)
|
374
|
-
# Rails 4+
|
375
|
-
ar_pools = conn_handler.connection_pool_list
|
376
|
-
else
|
377
|
-
# Rails 3.2
|
378
|
-
ar_pools = conn_handler.connection_pools.values
|
379
|
-
end
|
217
|
+
ar_pools = ActiveRecord::Base.connection_handler.connection_pool_list
|
380
218
|
|
381
219
|
ar_pools.each do |pool|
|
382
|
-
next if pool ==
|
220
|
+
next if pool == shards[:master] # Already handled this
|
383
221
|
|
384
222
|
begin
|
385
223
|
yield(pool)
|
@@ -393,52 +231,22 @@ module Octopus
|
|
393
231
|
end
|
394
232
|
end
|
395
233
|
|
396
|
-
def connection_pool_for(adapter, config)
|
397
|
-
if Octopus.rails4?
|
398
|
-
arg = ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(adapter.dup, config)
|
399
|
-
else
|
400
|
-
name = adapter["octopus_shard"]
|
401
|
-
arg = ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(name, adapter.dup, config)
|
402
|
-
end
|
403
|
-
|
404
|
-
ActiveRecord::ConnectionAdapters::ConnectionPool.new(arg)
|
405
|
-
end
|
406
|
-
|
407
|
-
def initialize_adapter(adapter)
|
408
|
-
@adapters << adapter
|
409
|
-
begin
|
410
|
-
require "active_record/connection_adapters/#{adapter}_adapter"
|
411
|
-
rescue LoadError
|
412
|
-
raise "Please install the #{adapter} adapter: `gem install activerecord-#{adapter}-adapter` (#{$ERROR_INFO})"
|
413
|
-
end
|
414
|
-
end
|
415
|
-
|
416
|
-
def resolve_string_connection(spec)
|
417
|
-
if Octopus.rails41? || Octopus.rails5?
|
418
|
-
resolver = ActiveRecord::ConnectionAdapters::ConnectionSpecification::Resolver.new({})
|
419
|
-
HashWithIndifferentAccess.new(resolver.spec(spec).config)
|
420
|
-
else
|
421
|
-
resolver = ActiveRecord::ConnectionAdapters::ConnectionSpecification::Resolver.new(spec, {})
|
422
|
-
HashWithIndifferentAccess.new(resolver.spec.config)
|
423
|
-
end
|
424
|
-
end
|
425
|
-
|
426
234
|
def should_clean_connection_proxy?(method)
|
427
235
|
method.to_s =~ /insert|select|execute/ && !current_model_replicated? && (!block || block != current_shard)
|
428
236
|
end
|
429
237
|
|
430
238
|
# Try to use slaves if and only if `replicated: true` is specified in `shards.yml` and no slaves groups are defined
|
431
239
|
def should_send_queries_to_replicated_databases?(method)
|
432
|
-
|
240
|
+
replicated && method.to_s =~ /select/ && !block && !slaves_grouped?
|
433
241
|
end
|
434
242
|
|
435
243
|
def current_model_replicated?
|
436
|
-
|
244
|
+
replicated && (current_model.try(:replicated) || fully_replicated?)
|
437
245
|
end
|
438
246
|
|
439
247
|
def send_queries_to_selected_slave(method, *args, &block)
|
440
248
|
if current_model.replicated || fully_replicated?
|
441
|
-
selected_slave =
|
249
|
+
selected_slave = slaves_load_balancer.next current_load_balance_options
|
442
250
|
else
|
443
251
|
selected_slave = Octopus.master_shard
|
444
252
|
end
|
@@ -460,7 +268,7 @@ module Octopus
|
|
460
268
|
end
|
461
269
|
|
462
270
|
def slaves_grouped?
|
463
|
-
|
271
|
+
slave_groups.present?
|
464
272
|
end
|
465
273
|
|
466
274
|
# Temporarily switch `current_shard` to the next slave in a slave group and send queries to it
|
@@ -498,7 +306,6 @@ module Octopus
|
|
498
306
|
older_slave_group = current_slave_group
|
499
307
|
older_load_balance_options = current_load_balance_options
|
500
308
|
|
501
|
-
|
502
309
|
begin
|
503
310
|
unless current_model && !current_model.allowed_shard?(shard)
|
504
311
|
self.current_shard = shard
|
@@ -522,13 +329,5 @@ module Octopus
|
|
522
329
|
self.current_group = older_group
|
523
330
|
end
|
524
331
|
end
|
525
|
-
|
526
|
-
def structurally_slave?(config)
|
527
|
-
config.is_a?(Hash) && config.key?('adapter')
|
528
|
-
end
|
529
|
-
|
530
|
-
def structurally_slave_group?(config)
|
531
|
-
config.is_a?(Hash) && config.values.any? { |v| structurally_slave? v }
|
532
|
-
end
|
533
332
|
end
|
534
333
|
end
|