ar-octopus-master 0.9.2.master → 0.9.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +12 -0
- data/.rspec +2 -0
- data/.rubocop.yml +46 -0
- data/.rubocop_todo.yml +56 -0
- data/.travis.yml +25 -0
- data/Appraisals +20 -0
- data/Gemfile +4 -0
- data/README.mkdn +242 -0
- data/Rakefile +172 -0
- data/TODO.txt +7 -0
- data/ar-octopus.gemspec +39 -0
- data/gemfiles/rails4.gemfile +7 -0
- data/gemfiles/rails41.gemfile +7 -0
- data/gemfiles/rails42.gemfile +7 -0
- data/gemfiles/rails5.gemfile +7 -0
- data/gemfiles/rails51.gemfile +7 -0
- data/lib/ar-octopus.rb +1 -0
- data/lib/octopus.rb +205 -0
- data/lib/octopus/abstract_adapter.rb +33 -0
- data/lib/octopus/association.rb +14 -0
- data/lib/octopus/association_shard_tracking.rb +74 -0
- data/lib/octopus/collection_association.rb +17 -0
- data/lib/octopus/collection_proxy.rb +16 -0
- data/lib/octopus/exception.rb +4 -0
- data/lib/octopus/finder_methods.rb +8 -0
- data/lib/octopus/has_and_belongs_to_many_association.rb +9 -0
- data/lib/octopus/load_balancing.rb +4 -0
- data/lib/octopus/load_balancing/round_robin.rb +20 -0
- data/lib/octopus/log_subscriber.rb +26 -0
- data/lib/octopus/migration.rb +195 -0
- data/lib/octopus/model.rb +223 -0
- data/lib/octopus/persistence.rb +45 -0
- data/lib/octopus/proxy.rb +346 -0
- data/lib/octopus/proxy_config.rb +252 -0
- data/lib/octopus/query_cache_for_shards.rb +25 -0
- data/lib/octopus/railtie.rb +11 -0
- data/lib/octopus/relation_proxy.rb +58 -0
- data/lib/octopus/result_patch.rb +19 -0
- data/lib/octopus/scope_proxy.rb +68 -0
- data/lib/octopus/shard_tracking.rb +46 -0
- data/lib/octopus/shard_tracking/attribute.rb +22 -0
- data/lib/octopus/shard_tracking/dynamic.rb +11 -0
- data/lib/octopus/singular_association.rb +9 -0
- data/lib/octopus/slave_group.rb +13 -0
- data/lib/octopus/version.rb +3 -0
- data/lib/tasks/octopus.rake +16 -0
- data/sample_app/.gitignore +4 -0
- data/sample_app/.rspec +1 -0
- data/sample_app/Gemfile +20 -0
- data/sample_app/README +3 -0
- data/sample_app/README.rdoc +261 -0
- data/sample_app/Rakefile +7 -0
- data/sample_app/app/assets/images/rails.png +0 -0
- data/sample_app/app/assets/javascripts/application.js +15 -0
- data/sample_app/app/assets/stylesheets/application.css +13 -0
- data/sample_app/app/controllers/application_controller.rb +4 -0
- data/sample_app/app/helpers/application_helper.rb +2 -0
- data/sample_app/app/mailers/.gitkeep +0 -0
- data/sample_app/app/models/.gitkeep +0 -0
- data/sample_app/app/models/item.rb +3 -0
- data/sample_app/app/models/user.rb +3 -0
- data/sample_app/app/views/layouts/application.html.erb +14 -0
- data/sample_app/autotest/discover.rb +2 -0
- data/sample_app/config.ru +4 -0
- data/sample_app/config/application.rb +62 -0
- data/sample_app/config/boot.rb +6 -0
- data/sample_app/config/cucumber.yml +8 -0
- data/sample_app/config/database.yml +28 -0
- data/sample_app/config/environment.rb +5 -0
- data/sample_app/config/environments/development.rb +37 -0
- data/sample_app/config/environments/production.rb +67 -0
- data/sample_app/config/environments/test.rb +37 -0
- data/sample_app/config/initializers/backtrace_silencers.rb +7 -0
- data/sample_app/config/initializers/inflections.rb +15 -0
- data/sample_app/config/initializers/mime_types.rb +5 -0
- data/sample_app/config/initializers/secret_token.rb +7 -0
- data/sample_app/config/initializers/session_store.rb +8 -0
- data/sample_app/config/initializers/wrap_parameters.rb +14 -0
- data/sample_app/config/locales/en.yml +5 -0
- data/sample_app/config/routes.rb +58 -0
- data/sample_app/config/shards.yml +28 -0
- data/sample_app/db/migrate/20100720172715_create_users.rb +15 -0
- data/sample_app/db/migrate/20100720172730_create_items.rb +16 -0
- data/sample_app/db/migrate/20100720210335_create_sample_users.rb +11 -0
- data/sample_app/db/schema.rb +29 -0
- data/sample_app/db/seeds.rb +16 -0
- data/sample_app/doc/README_FOR_APP +2 -0
- data/sample_app/features/migrate.feature +45 -0
- data/sample_app/features/seed.feature +15 -0
- data/sample_app/features/step_definitions/seeds_steps.rb +13 -0
- data/sample_app/features/step_definitions/web_steps.rb +218 -0
- data/sample_app/features/support/database.rb +13 -0
- data/sample_app/features/support/env.rb +57 -0
- data/sample_app/features/support/paths.rb +33 -0
- data/sample_app/lib/assets/.gitkeep +0 -0
- data/sample_app/lib/tasks/.gitkeep +0 -0
- data/sample_app/lib/tasks/cucumber.rake +64 -0
- data/sample_app/log/.gitkeep +0 -0
- data/sample_app/public/404.html +26 -0
- data/sample_app/public/422.html +26 -0
- data/sample_app/public/500.html +26 -0
- data/sample_app/public/favicon.ico +0 -0
- data/sample_app/public/images/rails.png +0 -0
- data/sample_app/public/index.html +279 -0
- data/sample_app/public/javascripts/application.js +2 -0
- data/sample_app/public/javascripts/controls.js +965 -0
- data/sample_app/public/javascripts/dragdrop.js +974 -0
- data/sample_app/public/javascripts/effects.js +1123 -0
- data/sample_app/public/javascripts/prototype.js +4874 -0
- data/sample_app/public/javascripts/rails.js +118 -0
- data/sample_app/public/robots.txt +5 -0
- data/sample_app/public/stylesheets/.gitkeep +0 -0
- data/sample_app/script/cucumber +10 -0
- data/sample_app/script/rails +6 -0
- data/sample_app/spec/models/item_spec.rb +5 -0
- data/sample_app/spec/models/user_spec.rb +5 -0
- data/sample_app/spec/spec_helper.rb +27 -0
- data/sample_app/vendor/assets/javascripts/.gitkeep +0 -0
- data/sample_app/vendor/assets/stylesheets/.gitkeep +0 -0
- data/sample_app/vendor/plugins/.gitkeep +0 -0
- data/spec/config/shards.yml +229 -0
- data/spec/migrations/10_create_users_using_replication.rb +9 -0
- data/spec/migrations/11_add_field_in_all_slaves.rb +11 -0
- data/spec/migrations/12_create_users_using_block.rb +23 -0
- data/spec/migrations/13_create_users_using_block_and_using.rb +15 -0
- data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +11 -0
- data/spec/migrations/15_create_user_on_shards_of_default_group_with_versions.rb +9 -0
- data/spec/migrations/1_create_users_on_master.rb +9 -0
- data/spec/migrations/2_create_users_on_canada.rb +11 -0
- data/spec/migrations/3_create_users_on_both_shards.rb +11 -0
- data/spec/migrations/4_create_users_on_shards_of_a_group.rb +11 -0
- data/spec/migrations/5_create_users_on_multiples_groups.rb +11 -0
- data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +11 -0
- data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +11 -0
- data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +11 -0
- data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +11 -0
- data/spec/octopus/association_shard_tracking_spec.rb +1036 -0
- data/spec/octopus/collection_proxy_spec.rb +16 -0
- data/spec/octopus/load_balancing/round_robin_spec.rb +15 -0
- data/spec/octopus/log_subscriber_spec.rb +19 -0
- data/spec/octopus/migration_spec.rb +134 -0
- data/spec/octopus/model_spec.rb +754 -0
- data/spec/octopus/octopus_spec.rb +123 -0
- data/spec/octopus/proxy_spec.rb +303 -0
- data/spec/octopus/query_cache_for_shards_spec.rb +17 -0
- data/spec/octopus/relation_proxy_spec.rb +124 -0
- data/spec/octopus/replicated_slave_grouped_spec.rb +91 -0
- data/spec/octopus/replication_spec.rb +196 -0
- data/spec/octopus/scope_proxy_spec.rb +97 -0
- data/spec/octopus/sharded_replicated_slave_grouped_spec.rb +55 -0
- data/spec/octopus/sharded_spec.rb +33 -0
- data/spec/spec_helper.rb +18 -0
- data/spec/support/active_record/connection_adapters/modify_config_adapter.rb +15 -0
- data/spec/support/database_connection.rb +4 -0
- data/spec/support/database_models.rb +118 -0
- data/spec/support/octopus_helper.rb +54 -0
- data/spec/support/query_count.rb +17 -0
- data/spec/support/shared_contexts.rb +18 -0
- data/spec/tasks/octopus.rake_spec.rb +32 -0
- metadata +203 -5
@@ -0,0 +1,45 @@
|
|
1
|
+
module Octopus
|
2
|
+
module Persistence
|
3
|
+
def update_attribute(*args)
|
4
|
+
run_on_shard { super }
|
5
|
+
end
|
6
|
+
|
7
|
+
def update_attributes(*args)
|
8
|
+
run_on_shard { super }
|
9
|
+
end
|
10
|
+
|
11
|
+
def update_attributes!(*args)
|
12
|
+
run_on_shard { super }
|
13
|
+
end
|
14
|
+
|
15
|
+
def reload(*args)
|
16
|
+
run_on_shard { super }
|
17
|
+
end
|
18
|
+
|
19
|
+
def delete
|
20
|
+
run_on_shard { super }
|
21
|
+
end
|
22
|
+
|
23
|
+
def destroy
|
24
|
+
run_on_shard { super }
|
25
|
+
end
|
26
|
+
|
27
|
+
def touch(*args)
|
28
|
+
run_on_shard { super }
|
29
|
+
end
|
30
|
+
|
31
|
+
def update_column(*args)
|
32
|
+
run_on_shard { super }
|
33
|
+
end
|
34
|
+
|
35
|
+
def increment!(*args)
|
36
|
+
run_on_shard { super }
|
37
|
+
end
|
38
|
+
|
39
|
+
def decrement!(*args)
|
40
|
+
run_on_shard { super }
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
ActiveRecord::Base.send(:include, Octopus::Persistence)
|
@@ -0,0 +1,346 @@
|
|
1
|
+
require 'set'
|
2
|
+
require 'octopus/slave_group'
|
3
|
+
require 'octopus/load_balancing/round_robin'
|
4
|
+
|
5
|
+
module Octopus
|
6
|
+
class Proxy
|
7
|
+
attr_accessor :proxy_config
|
8
|
+
|
9
|
+
delegate :current_model, :current_model=,
|
10
|
+
:current_shard, :current_shard=,
|
11
|
+
:current_group, :current_group=,
|
12
|
+
:current_slave_group, :current_slave_group=,
|
13
|
+
:current_load_balance_options, :current_load_balance_options=,
|
14
|
+
:block, :block=, :fully_replicated?, :has_group?,
|
15
|
+
:shard_names, :shards_for_group, :shards, :sharded, :slaves_list,
|
16
|
+
:shards_slave_groups, :slave_groups, :replicated, :slaves_load_balancer,
|
17
|
+
:config, :initialize_shards, :shard_name, to: :proxy_config, prefix: false
|
18
|
+
|
19
|
+
def initialize(config = Octopus.config)
|
20
|
+
self.proxy_config = Octopus::ProxyConfig.new(config)
|
21
|
+
end
|
22
|
+
|
23
|
+
# Rails Connection Methods - Those methods are overriden to add custom behavior that helps
|
24
|
+
# Octopus introduce Sharding / Replication.
|
25
|
+
delegate :adapter_name, :add_transaction_record, :case_sensitive_modifier,
|
26
|
+
:type_cast, :to_sql, :quote, :quote_column_name, :quote_table_name,
|
27
|
+
:quote_table_name_for_assignment, :supports_migrations?, :table_alias_for,
|
28
|
+
:table_exists?, :in_clause_length, :supports_ddl_transactions?,
|
29
|
+
:sanitize_limit, :prefetch_primary_key?, :current_database, :initialize_schema_migrations_table,
|
30
|
+
:combine_bind_parameters, :empty_insert_statement_value, :assume_migrated_upto_version,
|
31
|
+
:schema_cache, :substitute_at, :internal_string_options_for_primary_key, :lookup_cast_type_from_column,
|
32
|
+
:supports_advisory_locks?, :get_advisory_lock, :initialize_internal_metadata_table,
|
33
|
+
:release_advisory_lock, :prepare_binds_for_database, :cacheable_query, :column_name_for_operation,
|
34
|
+
:prepared_statements, :transaction_state, :create_table, to: :select_connection
|
35
|
+
|
36
|
+
def execute(sql, name = nil)
|
37
|
+
conn = select_connection
|
38
|
+
clean_connection_proxy if should_clean_connection_proxy?('execute')
|
39
|
+
conn.execute(sql, name)
|
40
|
+
end
|
41
|
+
|
42
|
+
def insert(arel, name = nil, pk = nil, id_value = nil, sequence_name = nil, binds = [])
|
43
|
+
conn = select_connection
|
44
|
+
clean_connection_proxy if should_clean_connection_proxy?('insert')
|
45
|
+
conn.insert(arel, name, pk, id_value, sequence_name, binds)
|
46
|
+
end
|
47
|
+
|
48
|
+
def update(arel, name = nil, binds = [])
|
49
|
+
conn = select_connection
|
50
|
+
# Call the legacy should_clean_connection_proxy? method here, emulating an insert.
|
51
|
+
clean_connection_proxy if should_clean_connection_proxy?('insert')
|
52
|
+
conn.update(arel, name, binds)
|
53
|
+
end
|
54
|
+
|
55
|
+
def delete(*args, &block)
|
56
|
+
legacy_method_missing_logic('delete', *args, &block)
|
57
|
+
end
|
58
|
+
|
59
|
+
def select_all(*args, &block)
|
60
|
+
legacy_method_missing_logic('select_all', *args, &block)
|
61
|
+
end
|
62
|
+
|
63
|
+
def select_value(*args, &block)
|
64
|
+
legacy_method_missing_logic('select_value', *args, &block)
|
65
|
+
end
|
66
|
+
|
67
|
+
# Rails 3.1 sets automatic_reconnect to false when it removes
|
68
|
+
# connection pool. Octopus can potentially retain a reference to a closed
|
69
|
+
# connection pool. Previously, that would work since the pool would just
|
70
|
+
# reconnect, but in Rails 3.1 the flag prevents this.
|
71
|
+
def safe_connection(connection_pool)
|
72
|
+
connection_pool.automatic_reconnect ||= true
|
73
|
+
if !connection_pool.connected? && shards[Octopus.master_shard].connection.query_cache_enabled
|
74
|
+
connection_pool.connection.enable_query_cache!
|
75
|
+
end
|
76
|
+
connection_pool.connection
|
77
|
+
end
|
78
|
+
|
79
|
+
def select_connection
|
80
|
+
safe_connection(shards[shard_name])
|
81
|
+
end
|
82
|
+
|
83
|
+
def run_queries_on_shard(shard, &_block)
|
84
|
+
keeping_connection_proxy(shard) do
|
85
|
+
using_shard(shard) do
|
86
|
+
yield
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def send_queries_to_multiple_shards(shards, &block)
|
92
|
+
shards.map do |shard|
|
93
|
+
run_queries_on_shard(shard, &block)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def send_queries_to_group(group, &block)
|
98
|
+
using_group(group) do
|
99
|
+
send_queries_to_multiple_shards(shards_for_group(group), &block)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def send_queries_to_all_shards(&block)
|
104
|
+
send_queries_to_multiple_shards(shard_names.uniq { |shard_name| shards[shard_name] }, &block)
|
105
|
+
end
|
106
|
+
|
107
|
+
def clean_connection_proxy
|
108
|
+
self.current_shard = Octopus.master_shard
|
109
|
+
self.current_model = nil
|
110
|
+
self.current_group = nil
|
111
|
+
self.block = nil
|
112
|
+
end
|
113
|
+
|
114
|
+
def check_schema_migrations(shard)
|
115
|
+
OctopusModel.using(shard).connection.table_exists?(
|
116
|
+
ActiveRecord::Migrator.schema_migrations_table_name,
|
117
|
+
) || OctopusModel.using(shard).connection.initialize_schema_migrations_table
|
118
|
+
end
|
119
|
+
|
120
|
+
def transaction(options = {}, &block)
|
121
|
+
if !sharded && current_model_replicated?
|
122
|
+
run_queries_on_shard(Octopus.master_shard) do
|
123
|
+
select_connection.transaction(options, &block)
|
124
|
+
end
|
125
|
+
else
|
126
|
+
select_connection.transaction(options, &block)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def method_missing(method, *args, &block)
|
131
|
+
legacy_method_missing_logic(method, *args, &block)
|
132
|
+
end
|
133
|
+
|
134
|
+
def respond_to?(method, include_private = false)
|
135
|
+
super || select_connection.respond_to?(method, include_private)
|
136
|
+
end
|
137
|
+
|
138
|
+
def connection_pool
|
139
|
+
shards[current_shard]
|
140
|
+
end
|
141
|
+
|
142
|
+
if Octopus.rails4?
|
143
|
+
def enable_query_cache!
|
144
|
+
clear_query_cache
|
145
|
+
with_each_healthy_shard { |v| v.connected? && safe_connection(v).enable_query_cache! }
|
146
|
+
end
|
147
|
+
|
148
|
+
def disable_query_cache!
|
149
|
+
with_each_healthy_shard { |v| v.connected? && safe_connection(v).disable_query_cache! }
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
def clear_query_cache
|
154
|
+
with_each_healthy_shard { |v| v.connected? && safe_connection(v).clear_query_cache }
|
155
|
+
end
|
156
|
+
|
157
|
+
def clear_active_connections!
|
158
|
+
with_each_healthy_shard(&:release_connection)
|
159
|
+
end
|
160
|
+
|
161
|
+
def clear_all_connections!
|
162
|
+
with_each_healthy_shard(&:disconnect!)
|
163
|
+
end
|
164
|
+
|
165
|
+
def connected?
|
166
|
+
shards.any? { |_k, v| v.connected? }
|
167
|
+
end
|
168
|
+
|
169
|
+
def should_send_queries_to_shard_slave_group?(method)
|
170
|
+
should_use_slaves_for_method?(method) && shards_slave_groups.try(:[], current_shard).try(:[], current_slave_group).present?
|
171
|
+
end
|
172
|
+
|
173
|
+
def send_queries_to_shard_slave_group(method, *args, &block)
|
174
|
+
send_queries_to_balancer(shards_slave_groups[current_shard][current_slave_group], method, *args, &block)
|
175
|
+
end
|
176
|
+
|
177
|
+
def should_send_queries_to_slave_group?(method)
|
178
|
+
should_use_slaves_for_method?(method) && slave_groups.try(:[], current_slave_group).present?
|
179
|
+
end
|
180
|
+
|
181
|
+
def send_queries_to_slave_group(method, *args, &block)
|
182
|
+
send_queries_to_balancer(slave_groups[current_slave_group], method, *args, &block)
|
183
|
+
end
|
184
|
+
|
185
|
+
def current_model_replicated?
|
186
|
+
replicated && (current_model.try(:replicated) || fully_replicated?)
|
187
|
+
end
|
188
|
+
|
189
|
+
protected
|
190
|
+
|
191
|
+
# @thiagopradi - This legacy method missing logic will be keep for a while for compatibility
|
192
|
+
# and will be removed when Octopus 1.0 will be released.
|
193
|
+
# We are planning to migrate to a much stable logic for the Proxy that doesn't require method missing.
|
194
|
+
def legacy_method_missing_logic(method, *args, &block)
|
195
|
+
if should_clean_connection_proxy?(method)
|
196
|
+
conn = select_connection
|
197
|
+
clean_connection_proxy
|
198
|
+
conn.send(method, *args, &block)
|
199
|
+
elsif should_send_queries_to_shard_slave_group?(method)
|
200
|
+
send_queries_to_shard_slave_group(method, *args, &block)
|
201
|
+
elsif should_send_queries_to_slave_group?(method)
|
202
|
+
send_queries_to_slave_group(method, *args, &block)
|
203
|
+
elsif should_send_queries_to_replicated_databases?(method)
|
204
|
+
send_queries_to_selected_slave(method, *args, &block)
|
205
|
+
else
|
206
|
+
val = select_connection.send(method, *args, &block)
|
207
|
+
|
208
|
+
if val.instance_of? ActiveRecord::Result
|
209
|
+
val.current_shard = shard_name
|
210
|
+
end
|
211
|
+
|
212
|
+
val
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
# Ensure that a single failing slave doesn't take down the entire application
|
217
|
+
def with_each_healthy_shard
|
218
|
+
shards.each do |shard_name, v|
|
219
|
+
begin
|
220
|
+
yield(v)
|
221
|
+
rescue => e
|
222
|
+
if Octopus.robust_environment?
|
223
|
+
Octopus.logger.error "Error on shard #{shard_name}: #{e.message}"
|
224
|
+
else
|
225
|
+
raise
|
226
|
+
end
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
230
|
+
ar_pools = ActiveRecord::Base.connection_handler.connection_pool_list
|
231
|
+
|
232
|
+
ar_pools.each do |pool|
|
233
|
+
next if pool == shards[:master] # Already handled this
|
234
|
+
|
235
|
+
begin
|
236
|
+
yield(pool)
|
237
|
+
rescue => e
|
238
|
+
if Octopus.robust_environment?
|
239
|
+
Octopus.logger.error "Error on pool (spec: #{pool.spec}): #{e.message}"
|
240
|
+
else
|
241
|
+
raise
|
242
|
+
end
|
243
|
+
end
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
def should_clean_connection_proxy?(method)
|
248
|
+
method.to_s =~ /insert|select|execute/ && !current_model_replicated? && (!block || block != current_shard)
|
249
|
+
end
|
250
|
+
|
251
|
+
# Try to use slaves if and only if `replicated: true` is specified in `shards.yml` and no slaves groups are defined
|
252
|
+
def should_send_queries_to_replicated_databases?(method)
|
253
|
+
replicated && method.to_s =~ /select/ && !block && !slaves_grouped?
|
254
|
+
end
|
255
|
+
|
256
|
+
def send_queries_to_selected_slave(method, *args, &block)
|
257
|
+
if current_model.replicated || fully_replicated?
|
258
|
+
selected_slave = slaves_load_balancer.next current_load_balance_options
|
259
|
+
else
|
260
|
+
selected_slave = Octopus.master_shard
|
261
|
+
end
|
262
|
+
|
263
|
+
send_queries_to_slave(selected_slave, method, *args, &block)
|
264
|
+
end
|
265
|
+
|
266
|
+
# We should use slaves if and only if its safe to do so.
|
267
|
+
#
|
268
|
+
# We can safely use slaves when:
|
269
|
+
# (1) `replicated: true` is specified in `shards.yml`
|
270
|
+
# (2) The current model is `replicated()`, or `fully_replicated: true` is specified in `shards.yml` which means that
|
271
|
+
# all the model is `replicated()`
|
272
|
+
# (3) It's a SELECT query
|
273
|
+
# while ensuring that we revert `current_shard` from the selected slave to the (shard's) master
|
274
|
+
# not to make queries other than SELECT leak to the slave.
|
275
|
+
def should_use_slaves_for_method?(method)
|
276
|
+
current_model_replicated? && method.to_s =~ /select/
|
277
|
+
end
|
278
|
+
|
279
|
+
def slaves_grouped?
|
280
|
+
slave_groups.present?
|
281
|
+
end
|
282
|
+
|
283
|
+
# Temporarily switch `current_shard` to the next slave in a slave group and send queries to it
|
284
|
+
# while preserving `current_shard`
|
285
|
+
def send_queries_to_balancer(balancer, method, *args, &block)
|
286
|
+
send_queries_to_slave(balancer.next(current_load_balance_options), method, *args, &block)
|
287
|
+
end
|
288
|
+
|
289
|
+
# Temporarily switch `current_shard` to the specified slave and send queries to it
|
290
|
+
# while preserving `current_shard`
|
291
|
+
def send_queries_to_slave(slave, method, *args, &block)
|
292
|
+
using_shard(slave) do
|
293
|
+
val = select_connection.send(method, *args, &block)
|
294
|
+
if val.instance_of? ActiveRecord::Result
|
295
|
+
val.current_shard = slave
|
296
|
+
end
|
297
|
+
val
|
298
|
+
end
|
299
|
+
end
|
300
|
+
|
301
|
+
# Temporarily block cleaning connection proxy and run the block
|
302
|
+
#
|
303
|
+
# @see Octopus::Proxy#should_clean_connection?
|
304
|
+
# @see Octopus::Proxy#clean_connection_proxy
|
305
|
+
def keeping_connection_proxy(shard, &_block)
|
306
|
+
last_block = block
|
307
|
+
|
308
|
+
begin
|
309
|
+
self.block = shard
|
310
|
+
yield
|
311
|
+
ensure
|
312
|
+
self.block = last_block || nil
|
313
|
+
end
|
314
|
+
end
|
315
|
+
|
316
|
+
# Temporarily switch `current_shard` and run the block
|
317
|
+
def using_shard(shard, &_block)
|
318
|
+
older_shard = current_shard
|
319
|
+
older_slave_group = current_slave_group
|
320
|
+
older_load_balance_options = current_load_balance_options
|
321
|
+
|
322
|
+
begin
|
323
|
+
unless current_model && !current_model.allowed_shard?(shard)
|
324
|
+
self.current_shard = shard
|
325
|
+
end
|
326
|
+
yield
|
327
|
+
ensure
|
328
|
+
self.current_shard = older_shard
|
329
|
+
self.current_slave_group = older_slave_group
|
330
|
+
self.current_load_balance_options = older_load_balance_options
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
# Temporarily switch `current_group` and run the block
|
335
|
+
def using_group(group, &_block)
|
336
|
+
older_group = current_group
|
337
|
+
|
338
|
+
begin
|
339
|
+
self.current_group = group
|
340
|
+
yield
|
341
|
+
ensure
|
342
|
+
self.current_group = older_group
|
343
|
+
end
|
344
|
+
end
|
345
|
+
end
|
346
|
+
end
|
@@ -0,0 +1,252 @@
|
|
1
|
+
module Octopus
|
2
|
+
class ProxyConfig
|
3
|
+
CURRENT_MODEL_KEY = 'octopus.current_model'.freeze
|
4
|
+
CURRENT_SHARD_KEY = 'octopus.current_shard'.freeze
|
5
|
+
CURRENT_GROUP_KEY = 'octopus.current_group'.freeze
|
6
|
+
CURRENT_SLAVE_GROUP_KEY = 'octopus.current_slave_group'.freeze
|
7
|
+
CURRENT_LOAD_BALANCE_OPTIONS_KEY = 'octopus.current_load_balance_options'.freeze
|
8
|
+
BLOCK_KEY = 'octopus.block'.freeze
|
9
|
+
FULLY_REPLICATED_KEY = 'octopus.fully_replicated'.freeze
|
10
|
+
|
11
|
+
attr_accessor :config, :sharded, :shards, :shards_slave_groups, :slave_groups,
|
12
|
+
:adapters, :replicated, :slaves_load_balancer, :slaves_list, :shards_slave_groups,
|
13
|
+
:slave_groups, :groups, :entire_sharded, :shards_config
|
14
|
+
|
15
|
+
def initialize(config)
|
16
|
+
initialize_shards(config)
|
17
|
+
initialize_replication(config) if !config.nil? && config['replicated']
|
18
|
+
end
|
19
|
+
|
20
|
+
def current_model
|
21
|
+
Thread.current[CURRENT_MODEL_KEY]
|
22
|
+
end
|
23
|
+
|
24
|
+
def current_model=(model)
|
25
|
+
Thread.current[CURRENT_MODEL_KEY] = model.is_a?(ActiveRecord::Base) ? model.class : model
|
26
|
+
end
|
27
|
+
|
28
|
+
def current_shard
|
29
|
+
Thread.current[CURRENT_SHARD_KEY] ||= Octopus.master_shard
|
30
|
+
end
|
31
|
+
|
32
|
+
def current_shard=(shard_symbol)
|
33
|
+
if shard_symbol.is_a?(Array)
|
34
|
+
self.current_slave_group = nil
|
35
|
+
shard_symbol.each { |symbol| fail "Nonexistent Shard Name: #{symbol}" if shards[symbol].nil? }
|
36
|
+
elsif shard_symbol.is_a?(Hash)
|
37
|
+
hash = shard_symbol
|
38
|
+
shard_symbol = hash[:shard]
|
39
|
+
slave_group_symbol = hash[:slave_group]
|
40
|
+
load_balance_options = hash[:load_balance_options]
|
41
|
+
|
42
|
+
if shard_symbol.nil? && slave_group_symbol.nil?
|
43
|
+
fail 'Neither shard or slave group must be specified'
|
44
|
+
end
|
45
|
+
|
46
|
+
if shard_symbol.present?
|
47
|
+
fail "Nonexistent Shard Name: #{shard_symbol}" if shards[shard_symbol].nil?
|
48
|
+
end
|
49
|
+
|
50
|
+
if slave_group_symbol.present?
|
51
|
+
if (shards_slave_groups.try(:[], shard_symbol).present? && shards_slave_groups[shard_symbol][slave_group_symbol].nil?) ||
|
52
|
+
(shards_slave_groups.try(:[], shard_symbol).nil? && @slave_groups[slave_group_symbol].nil?)
|
53
|
+
fail "Nonexistent Slave Group Name: #{slave_group_symbol} in shards config: #{shards_config.inspect}"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
self.current_slave_group = slave_group_symbol
|
57
|
+
self.current_load_balance_options = load_balance_options
|
58
|
+
else
|
59
|
+
fail "Nonexistent Shard Name: #{shard_symbol}" if shards[shard_symbol].nil?
|
60
|
+
end
|
61
|
+
|
62
|
+
Thread.current[CURRENT_SHARD_KEY] = shard_symbol
|
63
|
+
end
|
64
|
+
|
65
|
+
def current_group
|
66
|
+
Thread.current[CURRENT_GROUP_KEY]
|
67
|
+
end
|
68
|
+
|
69
|
+
def current_group=(group_symbol)
|
70
|
+
# TODO: Error message should include all groups if given more than one bad name.
|
71
|
+
[group_symbol].flatten.compact.each do |group|
|
72
|
+
fail "Nonexistent Group Name: #{group}" unless has_group?(group)
|
73
|
+
end
|
74
|
+
|
75
|
+
Thread.current[CURRENT_GROUP_KEY] = group_symbol
|
76
|
+
end
|
77
|
+
|
78
|
+
def current_slave_group
|
79
|
+
Thread.current[CURRENT_SLAVE_GROUP_KEY]
|
80
|
+
end
|
81
|
+
|
82
|
+
def current_slave_group=(slave_group_symbol)
|
83
|
+
Thread.current[CURRENT_SLAVE_GROUP_KEY] = slave_group_symbol
|
84
|
+
Thread.current[CURRENT_LOAD_BALANCE_OPTIONS_KEY] = nil if slave_group_symbol.nil?
|
85
|
+
end
|
86
|
+
|
87
|
+
def current_load_balance_options
|
88
|
+
Thread.current[CURRENT_LOAD_BALANCE_OPTIONS_KEY]
|
89
|
+
end
|
90
|
+
|
91
|
+
def current_load_balance_options=(options)
|
92
|
+
Thread.current[CURRENT_LOAD_BALANCE_OPTIONS_KEY] = options
|
93
|
+
end
|
94
|
+
|
95
|
+
def block
|
96
|
+
Thread.current[BLOCK_KEY]
|
97
|
+
end
|
98
|
+
|
99
|
+
def block=(block)
|
100
|
+
Thread.current[BLOCK_KEY] = block
|
101
|
+
end
|
102
|
+
|
103
|
+
def fully_replicated?
|
104
|
+
@fully_replicated || Thread.current[FULLY_REPLICATED_KEY]
|
105
|
+
end
|
106
|
+
|
107
|
+
# Public: Whether or not a group exists with the given name converted to a
|
108
|
+
# string.
|
109
|
+
#
|
110
|
+
# Returns a boolean.
|
111
|
+
def has_group?(group)
|
112
|
+
@groups.key?(group.to_s)
|
113
|
+
end
|
114
|
+
|
115
|
+
# Public: Retrieves names of all loaded shards.
|
116
|
+
#
|
117
|
+
# Returns an array of shard names as symbols
|
118
|
+
def shard_names
|
119
|
+
shards.keys
|
120
|
+
end
|
121
|
+
|
122
|
+
def shard_name
|
123
|
+
current_shard.is_a?(Array) ? current_shard.first : current_shard
|
124
|
+
end
|
125
|
+
|
126
|
+
# Public: Retrieves the defined shards for a given group.
|
127
|
+
#
|
128
|
+
# Returns an array of shard names as symbols or nil if the group is not
|
129
|
+
# defined.
|
130
|
+
def shards_for_group(group)
|
131
|
+
@groups.fetch(group.to_s, nil)
|
132
|
+
end
|
133
|
+
|
134
|
+
def initialize_shards(config)
|
135
|
+
self.config = config
|
136
|
+
|
137
|
+
self.shards = HashWithIndifferentAccess.new
|
138
|
+
self.shards_slave_groups = HashWithIndifferentAccess.new
|
139
|
+
self.slave_groups = HashWithIndifferentAccess.new
|
140
|
+
self.groups = {}
|
141
|
+
self.config = ActiveRecord::Base.connection_pool_without_octopus.spec.config
|
142
|
+
|
143
|
+
unless config.nil?
|
144
|
+
self.entire_sharded = config['entire_sharded']
|
145
|
+
self.shards_config = config[Octopus.rails_env]
|
146
|
+
end
|
147
|
+
|
148
|
+
self.shards_config ||= []
|
149
|
+
|
150
|
+
shards_config.each do |key, value|
|
151
|
+
if value.is_a?(String)
|
152
|
+
value = resolve_string_connection(value).merge(:octopus_shard => key)
|
153
|
+
initialize_adapter(value['adapter'])
|
154
|
+
shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
155
|
+
elsif value.is_a?(Hash) && value.key?('adapter')
|
156
|
+
value.merge!(:octopus_shard => key)
|
157
|
+
initialize_adapter(value['adapter'])
|
158
|
+
shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
|
159
|
+
|
160
|
+
slave_group_configs = value.select do |_k, v|
|
161
|
+
structurally_slave_group? v
|
162
|
+
end
|
163
|
+
|
164
|
+
if slave_group_configs.present?
|
165
|
+
slave_groups = HashWithIndifferentAccess.new
|
166
|
+
slave_group_configs.each do |slave_group_name, slave_configs|
|
167
|
+
slaves = HashWithIndifferentAccess.new
|
168
|
+
slave_configs.each do |slave_name, slave_config|
|
169
|
+
shards[slave_name.to_sym] = connection_pool_for(slave_config, "#{value['adapter']}_connection")
|
170
|
+
slaves[slave_name.to_sym] = slave_name.to_sym
|
171
|
+
end
|
172
|
+
slave_groups[slave_group_name.to_sym] = Octopus::SlaveGroup.new(slaves)
|
173
|
+
end
|
174
|
+
@shards_slave_groups[key.to_sym] = slave_groups
|
175
|
+
@sharded = true
|
176
|
+
end
|
177
|
+
elsif value.is_a?(Hash)
|
178
|
+
@groups[key.to_s] = []
|
179
|
+
|
180
|
+
value.each do |k, v|
|
181
|
+
fail 'You have duplicated shard names!' if shards.key?(k.to_sym)
|
182
|
+
|
183
|
+
initialize_adapter(v['adapter'])
|
184
|
+
config_with_octopus_shard = v.merge(:octopus_shard => k)
|
185
|
+
|
186
|
+
shards[k.to_sym] = connection_pool_for(config_with_octopus_shard, "#{v['adapter']}_connection")
|
187
|
+
@groups[key.to_s] << k.to_sym
|
188
|
+
end
|
189
|
+
|
190
|
+
if structurally_slave_group? value
|
191
|
+
slaves = Hash[@groups[key.to_s].map { |v| [v, v] }]
|
192
|
+
@slave_groups[key.to_sym] = Octopus::SlaveGroup.new(slaves)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
shards[:master] ||= ActiveRecord::Base.connection_pool_without_octopus if Octopus.master_shard == :master
|
198
|
+
end
|
199
|
+
|
200
|
+
def initialize_replication(config)
|
201
|
+
@replicated = true
|
202
|
+
if config.key?('fully_replicated')
|
203
|
+
@fully_replicated = config['fully_replicated']
|
204
|
+
else
|
205
|
+
@fully_replicated = true
|
206
|
+
end
|
207
|
+
|
208
|
+
@slaves_list = shards.keys.map(&:to_s).sort
|
209
|
+
@slaves_list.delete('master')
|
210
|
+
@slaves_load_balancer = Octopus.load_balancer.new(@slaves_list)
|
211
|
+
end
|
212
|
+
|
213
|
+
private
|
214
|
+
|
215
|
+
def connection_pool_for(config, adapter)
|
216
|
+
if Octopus.rails4?
|
217
|
+
spec = ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(config.dup, adapter )
|
218
|
+
else
|
219
|
+
name = adapter["octopus_shard"]
|
220
|
+
spec = ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(name, config.dup, adapter)
|
221
|
+
end
|
222
|
+
|
223
|
+
ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec)
|
224
|
+
end
|
225
|
+
|
226
|
+
def resolve_string_connection(spec)
|
227
|
+
if Octopus.rails41? || Octopus.rails50? || Octopus.rails51?
|
228
|
+
resolver = ActiveRecord::ConnectionAdapters::ConnectionSpecification::Resolver.new({})
|
229
|
+
HashWithIndifferentAccess.new(resolver.spec(spec).config)
|
230
|
+
else
|
231
|
+
resolver = ActiveRecord::ConnectionAdapters::ConnectionSpecification::Resolver.new(spec, {})
|
232
|
+
HashWithIndifferentAccess.new(resolver.spec.config)
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
def structurally_slave?(config)
|
237
|
+
config.is_a?(Hash) && config.key?('adapter')
|
238
|
+
end
|
239
|
+
|
240
|
+
def structurally_slave_group?(config)
|
241
|
+
config.is_a?(Hash) && config.values.any? { |v| structurally_slave? v }
|
242
|
+
end
|
243
|
+
|
244
|
+
def initialize_adapter(adapter)
|
245
|
+
begin
|
246
|
+
require "active_record/connection_adapters/#{adapter}_adapter"
|
247
|
+
rescue LoadError
|
248
|
+
raise "Please install the #{adapter} adapter: `gem install activerecord-#{adapter}-adapter` (#{$ERROR_INFO})"
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|