activerecord-turntable 3.0.0.alpha3 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitmodules +3 -0
- data/.rubocop.yml +18 -0
- data/.rubocop_todo.yml +153 -0
- data/.travis.yml +20 -4
- data/CHANGELOG.md +32 -0
- data/Guardfile +2 -2
- data/README.md +68 -14
- data/Rakefile +42 -0
- data/activerecord-turntable.gemspec +13 -3
- data/gemfiles/rails_edge.gemfile +8 -0
- data/lib/active_record/turntable/active_record_ext/abstract_adapter.rb +3 -1
- data/lib/active_record/turntable/active_record_ext/activerecord_import_ext.rb +5 -7
- data/lib/active_record/turntable/active_record_ext/acts_as_archive_extension.rb +2 -2
- data/lib/active_record/turntable/active_record_ext/association.rb +3 -3
- data/lib/active_record/turntable/active_record_ext/clever_load.rb +2 -2
- data/lib/active_record/turntable/active_record_ext/database_tasks.rb +10 -8
- data/lib/active_record/turntable/active_record_ext/fixtures.rb +15 -13
- data/lib/active_record/turntable/active_record_ext/log_subscriber.rb +6 -0
- data/lib/active_record/turntable/active_record_ext/persistence.rb +25 -23
- data/lib/active_record/turntable/active_record_ext/schema_dumper.rb +8 -75
- data/lib/active_record/turntable/algorithm/range_algorithm.rb +6 -7
- data/lib/active_record/turntable/algorithm/range_bsearch_algorithm.rb +6 -7
- data/lib/active_record/turntable/base.rb +2 -17
- data/lib/active_record/turntable/cluster_helper_methods.rb +7 -4
- data/lib/active_record/turntable/connection_proxy.rb +4 -2
- data/lib/active_record/turntable/migration.rb +3 -5
- data/lib/active_record/turntable/mixer.rb +20 -19
- data/lib/active_record/turntable/pool_proxy.rb +20 -14
- data/lib/active_record/turntable/query_cache.rb +1 -1
- data/lib/active_record/turntable/railties/databases.rake +12 -12
- data/lib/active_record/turntable/seq_shard.rb +1 -1
- data/lib/active_record/turntable/sequencer/barrage.rb +3 -2
- data/lib/active_record/turntable/sequencer.rb +33 -29
- data/lib/active_record/turntable/shard.rb +8 -8
- data/lib/active_record/turntable/sharding_condition.rb +14 -14
- data/lib/active_record/turntable/sql_tree_patch.rb +7 -3
- data/lib/active_record/turntable/util.rb +4 -2
- data/lib/active_record/turntable/version.rb +1 -1
- data/lib/active_record/turntable.rb +6 -5
- data/lib/activerecord-turntable.rb +1 -0
- metadata +120 -101
- data/lib/active_record/turntable/helpers/test_helper.rb +0 -25
- data/lib/active_record/turntable/helpers.rb +0 -9
- data/spec/active_record/turntable/active_record_ext/association_preloader_spec.rb +0 -78
- data/spec/active_record/turntable/active_record_ext/association_spec.rb +0 -81
- data/spec/active_record/turntable/active_record_ext/clever_load_spec.rb +0 -72
- data/spec/active_record/turntable/active_record_ext/fixture_set_spec.rb +0 -27
- data/spec/active_record/turntable/active_record_ext/locking_optimistic_spec.rb +0 -28
- data/spec/active_record/turntable/active_record_ext/migration_spec.rb +0 -38
- data/spec/active_record/turntable/active_record_ext/persistence_spec.rb +0 -211
- data/spec/active_record/turntable/active_record_ext/sequencer_spec.rb +0 -22
- data/spec/active_record/turntable/active_record_ext/test_fixtures_spec.rb +0 -34
- data/spec/active_record/turntable/algorithm/modulo_algorithm_spec.rb +0 -34
- data/spec/active_record/turntable/algorithm/range_algorithm_spec.rb +0 -34
- data/spec/active_record/turntable/algorithm/range_bsearch_algorithm_spec.rb +0 -34
- data/spec/active_record/turntable/algorithm_spec.rb +0 -100
- data/spec/active_record/turntable/base_spec.rb +0 -13
- data/spec/active_record/turntable/cluster_spec.rb +0 -48
- data/spec/active_record/turntable/config_spec.rb +0 -17
- data/spec/active_record/turntable/connection_proxy_spec.rb +0 -252
- data/spec/active_record/turntable/finder_spec.rb +0 -40
- data/spec/active_record/turntable/mixer/fader_spec.rb +0 -4
- data/spec/active_record/turntable/mixer_spec.rb +0 -112
- data/spec/active_record/turntable/query_cache_spec.rb +0 -28
- data/spec/active_record/turntable/sequencer/api_spec.rb +0 -38
- data/spec/active_record/turntable/sequencer/barrage_spec.rb +0 -22
- data/spec/active_record/turntable/sequencer/mysql_spec.rb +0 -22
- data/spec/active_record/turntable/shard_spec.rb +0 -21
- data/spec/active_record/turntable/sql_tree_patch_spec.rb +0 -34
- data/spec/active_record/turntable/transaction_spec.rb +0 -35
- data/spec/active_record/turntable_spec.rb +0 -30
- data/spec/config/database.yml +0 -35
- data/spec/config/turntable.yml +0 -56
- data/spec/fabricators/.gitkeep +0 -0
- data/spec/fabricators/turntable_fabricator.rb +0 -12
- data/spec/fixtures/cards.yml +0 -11
- data/spec/migrations/.gitkeep +0 -0
- data/spec/migrations/001_create_users.rb +0 -17
- data/spec/migrations/002_create_user_statuses.rb +0 -16
- data/spec/migrations/003_create_cards.rb +0 -14
- data/spec/migrations/004_create_cards_users.rb +0 -15
- data/spec/models/card.rb +0 -3
- data/spec/models/cards_user.rb +0 -10
- data/spec/models/cards_users_histories.rb +0 -7
- data/spec/models/events_users_history.rb +0 -7
- data/spec/models/user.rb +0 -7
- data/spec/models/user_status.rb +0 -6
- data/spec/spec_helper.rb +0 -38
- data/spec/support/matchers/be_saved_to.rb +0 -6
- data/spec/support/turntable_helper.rb +0 -30
@@ -18,7 +18,7 @@ module ActiveRecord
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def create_current_turntable_cluster(environment = env)
|
21
|
-
each_current_turntable_cluster_configuration(
|
21
|
+
each_current_turntable_cluster_configuration(environment) { |_name, configuration|
|
22
22
|
puts "[turntable] *** executing to database: #{configuration['database']}"
|
23
23
|
create configuration
|
24
24
|
}
|
@@ -26,26 +26,27 @@ module ActiveRecord
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def drop_current_turntable_cluster(environment = env)
|
29
|
-
each_current_turntable_cluster_configuration(
|
29
|
+
each_current_turntable_cluster_configuration(environment) { |_name, configuration|
|
30
30
|
puts "[turntable] *** executing to database: #{configuration['database']}"
|
31
31
|
drop configuration
|
32
32
|
}
|
33
33
|
end
|
34
34
|
|
35
|
-
def each_current_turntable_cluster_connected(
|
36
|
-
|
35
|
+
def each_current_turntable_cluster_connected(environment)
|
36
|
+
old_connection_pool = ActiveRecord::Base.connection_pool
|
37
|
+
each_current_turntable_cluster_configuration(environment) do |name, configuration|
|
37
38
|
ActiveRecord::Base.clear_active_connections!
|
38
39
|
ActiveRecord::Base.establish_connection(configuration)
|
39
40
|
ActiveRecord::Migration.current_shard = name
|
40
41
|
yield(name, configuration)
|
41
42
|
end
|
42
43
|
ActiveRecord::Base.clear_active_connections!
|
43
|
-
ActiveRecord::Base.establish_connection
|
44
|
+
ActiveRecord::Base.establish_connection old_connection_pool.spec.config
|
44
45
|
end
|
45
46
|
|
46
|
-
def each_current_turntable_cluster_configuration(
|
47
|
+
def each_current_turntable_cluster_configuration(environment)
|
47
48
|
environments = [environment]
|
48
|
-
environments << "test" if
|
49
|
+
environments << "test" if environment == "development"
|
49
50
|
|
50
51
|
current_turntable_cluster_configurations(*environments).each do |name, configuration|
|
51
52
|
yield(name, configuration) unless configuration["database"].blank?
|
@@ -70,8 +71,9 @@ module ActiveRecord
|
|
70
71
|
configurations = []
|
71
72
|
environments.each do |environ|
|
72
73
|
config = ActiveRecord::Base.configurations[environ]
|
74
|
+
next unless config
|
73
75
|
%w(shards seq).each do |name|
|
74
|
-
configurations += config[name].to_a
|
76
|
+
configurations += config[name].to_a if config.has_key?(name)
|
75
77
|
end
|
76
78
|
end
|
77
79
|
configurations
|
@@ -7,6 +7,7 @@ module ActiveRecord
|
|
7
7
|
class FixtureSet
|
8
8
|
extend ActiveRecord::Turntable::Util
|
9
9
|
|
10
|
+
# rubocop:disable Style/MultilineMethodCallBraceLayout
|
10
11
|
def self.create_fixtures(fixtures_directory, fixture_set_names, class_names = {}, config = ActiveRecord::Base)
|
11
12
|
fixture_set_names = Array(fixture_set_names).map(&:to_s)
|
12
13
|
class_names = ClassCache.new class_names, config
|
@@ -24,7 +25,7 @@ module ActiveRecord
|
|
24
25
|
|
25
26
|
fixture_sets = files_to_read.map do |fs_name|
|
26
27
|
klass = class_names[fs_name]
|
27
|
-
conn = klass
|
28
|
+
conn = klass ? klass.connection : connection
|
28
29
|
fixtures_map[fs_name] = new( # ActiveRecord::FixtureSet.new
|
29
30
|
conn,
|
30
31
|
fs_name,
|
@@ -35,12 +36,16 @@ module ActiveRecord
|
|
35
36
|
update_all_loaded_fixtures fixtures_map
|
36
37
|
|
37
38
|
ActiveRecord::Base.force_transaction_all_shards!(requires_new: true) do
|
39
|
+
deleted_tables = Hash.new { |h, k| h[k] = Set.new }
|
38
40
|
fixture_sets.each do |fs|
|
39
41
|
conn = fs.model_class.respond_to?(:connection) ? fs.model_class.connection : connection
|
40
42
|
table_rows = fs.table_rows
|
41
43
|
|
42
44
|
table_rows.each_key do |table|
|
43
|
-
conn.
|
45
|
+
unless deleted_tables[conn].include? table
|
46
|
+
conn.delete "DELETE FROM #{conn.quote_table_name(table)}", "Fixture Delete"
|
47
|
+
end
|
48
|
+
deleted_tables[conn] << table
|
44
49
|
end
|
45
50
|
|
46
51
|
table_rows.each do |fixture_set_name, rows|
|
@@ -48,11 +53,9 @@ module ActiveRecord
|
|
48
53
|
conn.insert_fixture(row, fixture_set_name)
|
49
54
|
end
|
50
55
|
end
|
51
|
-
end
|
52
56
|
|
53
|
-
|
54
|
-
|
55
|
-
fixture_sets.each do |fs|
|
57
|
+
# Cap primary key sequences to max(pk).
|
58
|
+
if connection.respond_to?(:reset_pk_sequence!)
|
56
59
|
connection.reset_pk_sequence!(fs.table_name)
|
57
60
|
end
|
58
61
|
end
|
@@ -63,16 +66,14 @@ module ActiveRecord
|
|
63
66
|
end
|
64
67
|
cached_fixtures(connection, fixture_set_names)
|
65
68
|
end
|
69
|
+
# rubocop:enable Style/MultilineMethodCallLayout
|
66
70
|
end
|
67
71
|
|
68
72
|
module TestFixtures
|
69
|
-
|
70
|
-
|
73
|
+
# rubocop:disable Style/ClassVars, Style/RedundantException
|
71
74
|
def setup_fixtures(config = ActiveRecord::Base)
|
72
|
-
return if ActiveRecord::Base.configurations.blank?
|
73
|
-
|
74
75
|
if pre_loaded_fixtures && !use_transactional_fixtures
|
75
|
-
raise "pre_loaded_fixtures requires use_transactional_fixtures"
|
76
|
+
raise RuntimeError, "pre_loaded_fixtures requires use_transactional_fixtures"
|
76
77
|
end
|
77
78
|
|
78
79
|
@fixture_cache = {}
|
@@ -94,14 +95,15 @@ module ActiveRecord
|
|
94
95
|
end
|
95
96
|
# Load fixtures for every test.
|
96
97
|
else
|
97
|
-
ActiveRecord::
|
98
|
+
ActiveRecord::FixtureSet.reset_cache
|
98
99
|
@@already_loaded_fixtures[self.class] = nil
|
99
100
|
@loaded_fixtures = load_fixtures(config)
|
100
101
|
end
|
101
102
|
|
102
103
|
# Instantiate fixtures for every test if requested.
|
103
|
-
instantiate_fixtures
|
104
|
+
instantiate_fixtures if use_instantiated_fixtures
|
104
105
|
end
|
106
|
+
# rubocop:enable Style/ClassVars, Style/RedundantException
|
105
107
|
|
106
108
|
def enlist_fixture_connections
|
107
109
|
ActiveRecord::Base.connection_handler.connection_pool_list.map(&:connection) +
|
@@ -6,6 +6,12 @@ module ActiveRecord::Turntable
|
|
6
6
|
# @note prepend to add shard name logging
|
7
7
|
def sql(event)
|
8
8
|
payload = event.payload
|
9
|
+
|
10
|
+
if self.class::IGNORE_PAYLOAD_NAMES.include?(payload[:name])
|
11
|
+
self.class.runtime += event.duration
|
12
|
+
return
|
13
|
+
end
|
14
|
+
|
9
15
|
if payload[:turntable_shard_name]
|
10
16
|
payload[:name] = "#{payload[:name]} [Shard: #{payload[:turntable_shard_name]}]"
|
11
17
|
end
|
@@ -27,8 +27,9 @@ module ActiveRecord::Turntable
|
|
27
27
|
end
|
28
28
|
|
29
29
|
# @note Override to add sharding scope on `touch`
|
30
|
+
# rubocop:disable Style/UnlessElse
|
30
31
|
def touch(*names, time: nil)
|
31
|
-
raise ActiveRecordError, "cannot touch on a new record object" unless persisted?
|
32
|
+
raise ActiveRecord::ActiveRecordError, "cannot touch on a new record object" unless persisted?
|
32
33
|
|
33
34
|
time ||= current_time_from_proper_timezone
|
34
35
|
attributes = timestamp_attributes_for_update_in_model
|
@@ -68,11 +69,12 @@ module ActiveRecord::Turntable
|
|
68
69
|
true
|
69
70
|
end
|
70
71
|
end
|
72
|
+
# rubocop:enable Style/UnlessElse
|
71
73
|
|
72
74
|
# @note Override to add sharding scope on `update_columns`
|
73
75
|
def update_columns(attributes)
|
74
|
-
raise ActiveRecordError, "cannot update a new record" if new_record?
|
75
|
-
raise ActiveRecordError, "cannot update a destroyed record" if destroyed?
|
76
|
+
raise ActiveRecord::ActiveRecordError, "cannot update a new record" if new_record?
|
77
|
+
raise ActiveRecord::ActiveRecordError, "cannot update a destroyed record" if destroyed?
|
76
78
|
|
77
79
|
attributes.each_key do |key|
|
78
80
|
verify_readonly_attribute(key.to_s)
|
@@ -95,30 +97,30 @@ module ActiveRecord::Turntable
|
|
95
97
|
|
96
98
|
private
|
97
99
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
100
|
+
# @note Override to add sharding scope on destroying
|
101
|
+
def relation_for_destroy
|
102
|
+
klass = self.class
|
103
|
+
relation = klass.unscoped.where(klass.primary_key => id)
|
102
104
|
|
103
|
-
|
104
|
-
|
105
|
+
if klass.turntable_enabled? && klass.primary_key != klass.turntable_shard_key.to_s
|
106
|
+
relation = relation.where(klass.turntable_shard_key => self[klass.turntable_shard_key])
|
107
|
+
end
|
108
|
+
relation
|
105
109
|
end
|
106
|
-
relation
|
107
|
-
end
|
108
110
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
111
|
+
# @note Override to add sharding scope on updating
|
112
|
+
def _update_record(attribute_names = self.attribute_names)
|
113
|
+
klass = self.class
|
114
|
+
attributes_values = arel_attributes_with_values_for_update(attribute_names)
|
115
|
+
if attributes_values.empty?
|
116
|
+
0
|
117
|
+
else
|
118
|
+
scope = if klass.turntable_enabled? && (klass.primary_key != klass.turntable_shard_key.to_s)
|
119
|
+
klass.unscoped.where(klass.turntable_shard_key => self.send(turntable_shard_key))
|
120
|
+
end
|
121
|
+
klass.unscoped._update_record attributes_values, id, id_was, scope
|
122
|
+
end
|
120
123
|
end
|
121
|
-
end
|
122
124
|
end
|
123
125
|
end
|
124
126
|
end
|
@@ -2,93 +2,26 @@
|
|
2
2
|
module ActiveRecord::Turntable
|
3
3
|
module ActiveRecordExt
|
4
4
|
module SchemaDumper
|
5
|
+
SEQUENCE_TABLE_REGEXP = /\A(.*)_id_seq\z/
|
5
6
|
|
6
7
|
private
|
7
8
|
|
8
9
|
# @note Override to dump database sequencer method
|
9
10
|
def table(table, stream)
|
10
|
-
|
11
|
+
unless matchdata = table.match(SEQUENCE_TABLE_REGEXP)
|
12
|
+
return super
|
13
|
+
end
|
14
|
+
|
11
15
|
begin
|
12
16
|
tbl = StringIO.new
|
13
17
|
|
14
|
-
|
15
|
-
if @connection.respond_to?(:primary_keys)
|
16
|
-
pk = @connection.primary_keys(table)
|
17
|
-
pk = pk.first unless pk.size > 1
|
18
|
-
else
|
19
|
-
pk = @connection.primary_key(table)
|
20
|
-
end
|
21
|
-
|
22
|
-
if table =~ /\A(.*)_id_seq\z/
|
23
|
-
tbl.print " create_sequence_for #{remove_prefix_and_suffix(Regexp.last_match(1)).inspect}"
|
24
|
-
else
|
25
|
-
tbl.print " create_table #{remove_prefix_and_suffix(table).inspect}"
|
26
|
-
end
|
27
|
-
|
28
|
-
case pk
|
29
|
-
when String
|
30
|
-
tbl.print ", primary_key: #{pk.inspect}" unless pk == 'id'
|
31
|
-
pkcol = columns.detect { |c| c.name == pk }
|
32
|
-
pkcolspec = @connection.column_spec_for_primary_key(pkcol)
|
33
|
-
if pkcolspec.present?
|
34
|
-
pkcolspec.each do |key, value|
|
35
|
-
tbl.print ", #{key}: #{value}"
|
36
|
-
end
|
37
|
-
end
|
38
|
-
when Array
|
39
|
-
tbl.print ", primary_key: #{pk.inspect}"
|
40
|
-
else
|
41
|
-
tbl.print ", id: false"
|
42
|
-
end
|
18
|
+
tbl.print " create_sequence_for #{remove_prefix_and_suffix(matchdata[1]).inspect}"
|
43
19
|
tbl.print ", force: :cascade"
|
44
20
|
|
45
21
|
table_options = @connection.table_options(table)
|
46
|
-
|
47
|
-
|
48
|
-
if comment = @connection.table_comment(table).presence
|
49
|
-
tbl.print ", comment: #{comment.inspect}"
|
50
|
-
end
|
51
|
-
|
52
|
-
tbl.puts " do |t|"
|
53
|
-
|
54
|
-
# then dump all non-primary key columns
|
55
|
-
column_specs = columns.map do |column|
|
56
|
-
raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" unless @connection.valid_type?(column.type)
|
57
|
-
next if column.name == pk
|
58
|
-
@connection.column_spec(column)
|
59
|
-
end.compact
|
60
|
-
|
61
|
-
# find all migration keys used in this table
|
62
|
-
keys = @connection.migration_keys
|
63
|
-
|
64
|
-
# figure out the lengths for each column based on above keys
|
65
|
-
lengths = keys.map { |key|
|
66
|
-
column_specs.map { |spec|
|
67
|
-
spec[key] ? spec[key].length + 2 : 0
|
68
|
-
}.max
|
69
|
-
}
|
70
|
-
|
71
|
-
# the string we're going to sprintf our values against, with standardized column widths
|
72
|
-
format_string = lengths.map{ |len| "%-#{len}s" }
|
73
|
-
|
74
|
-
# find the max length for the 'type' column, which is special
|
75
|
-
type_length = column_specs.map{ |column| column[:type].length }.max
|
76
|
-
|
77
|
-
# add column type definition to our format string
|
78
|
-
format_string.unshift " t.%-#{type_length}s "
|
79
|
-
|
80
|
-
format_string *= ''
|
81
|
-
|
82
|
-
column_specs.each do |colspec|
|
83
|
-
values = keys.zip(lengths).map{ |key, len| colspec.key?(key) ? colspec[key] + ", " : " " * len }
|
84
|
-
values.unshift colspec[:type]
|
85
|
-
tbl.print((format_string % values).gsub(/,\s*$/, ''))
|
86
|
-
tbl.puts
|
22
|
+
if table_options.present?
|
23
|
+
tbl.print ", #{format_options(table_options)}"
|
87
24
|
end
|
88
|
-
|
89
|
-
indexes_in_create(table, tbl)
|
90
|
-
|
91
|
-
tbl.puts " end"
|
92
25
|
tbl.puts
|
93
26
|
|
94
27
|
tbl.rewind
|
@@ -18,17 +18,16 @@ module ActiveRecord::Turntable::Algorithm
|
|
18
18
|
|
19
19
|
# { connection_name => weight, ... }
|
20
20
|
def calculate_used_shards_with_weight(sequence_value)
|
21
|
-
|
22
|
-
|
23
|
-
shards = @config[:shards][0..idx]
|
21
|
+
current_shard_idx = calculate_idx(sequence_value)
|
22
|
+
shards = @config[:shards][0..current_shard_idx]
|
24
23
|
weighted_hash = Hash.new { |h, k| h[k] = 0 }
|
25
24
|
prev_max = 0
|
26
25
|
shards.each_with_index do |h, idx|
|
27
26
|
weighted_hash[h[:connection]] += if idx < shards.size - 1
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
27
|
+
h[:less_than] - prev_max - 1
|
28
|
+
else
|
29
|
+
sequence_value - prev_max
|
30
|
+
end
|
32
31
|
prev_max = h[:less_than] - 1
|
33
32
|
end
|
34
33
|
weighted_hash
|
@@ -22,17 +22,16 @@ module ActiveRecord::Turntable::Algorithm
|
|
22
22
|
|
23
23
|
# { connection_name => weight, ... }
|
24
24
|
def calculate_used_shards_with_weight(sequence_value)
|
25
|
-
|
26
|
-
|
27
|
-
shards = @config[:shards][0..idx]
|
25
|
+
current_shard_idx = calculate_idx(sequence_value)
|
26
|
+
shards = @config[:shards][0..current_shard_idx]
|
28
27
|
weighted_hash = Hash.new { |h, k| h[k] = 0 }
|
29
28
|
prev_max = 0
|
30
29
|
shards.each_with_index do |h, idx|
|
31
30
|
weighted_hash[h[:connection]] += if idx < shards.size - 1
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
31
|
+
h[:less_than] - prev_max - 1
|
32
|
+
else
|
33
|
+
sequence_value - prev_max
|
34
|
+
end
|
36
35
|
prev_max = h[:less_than] - 1
|
37
36
|
end
|
38
37
|
weighted_hash
|
@@ -22,21 +22,6 @@ module ActiveRecord::Turntable
|
|
22
22
|
include ClusterHelperMethods
|
23
23
|
end
|
24
24
|
|
25
|
-
module ConnectionExtension
|
26
|
-
module ClassMethods
|
27
|
-
def connection_specification_name
|
28
|
-
return super unless turntable_enabled?
|
29
|
-
self.connection_specification_name = "turntable_pool_proxy::#{name}"
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def self.prepended(base)
|
34
|
-
class << base
|
35
|
-
prepend ClassMethods
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
25
|
module ClassMethods
|
41
26
|
# @param [Symbol] cluster_name cluster name for this class
|
42
27
|
# @param [Symbol] shard_key_name shard key attribute name
|
@@ -52,8 +37,7 @@ module ActiveRecord::Turntable
|
|
52
37
|
self.turntable_clusters[cluster_name] ||= Cluster.new(
|
53
38
|
turntable_config[:clusters][cluster_name],
|
54
39
|
options
|
55
|
-
|
56
|
-
prepend ConnectionExtension
|
40
|
+
)
|
57
41
|
turntable_replace_connection_pool
|
58
42
|
end
|
59
43
|
|
@@ -62,6 +46,7 @@ module ActiveRecord::Turntable
|
|
62
46
|
cp = ConnectionProxy.new(self, turntable_cluster)
|
63
47
|
pp = PoolProxy.new(cp)
|
64
48
|
|
49
|
+
self.connection_specification_name = "turntable_pool_proxy::#{name}"
|
65
50
|
ch.send(:owner_to_pool)[connection_specification_name] = pp
|
66
51
|
end
|
67
52
|
|
@@ -30,7 +30,9 @@ module ActiveRecord::Turntable
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def force_connect_all_shards!
|
33
|
-
conf = configurations[
|
33
|
+
conf = configurations[::ActiveRecord::ConnectionHandling::DEFAULT_ENV.call.to_sym]
|
34
|
+
return unless conf
|
35
|
+
|
34
36
|
shards = HashWithIndifferentAccess.new
|
35
37
|
shards = shards.merge(conf[:shards]) if conf[:shards]
|
36
38
|
shards = shards.merge(conf[:seq]) if conf[:seq]
|
@@ -44,20 +46,21 @@ module ActiveRecord::Turntable
|
|
44
46
|
shards_weight = self.turntable_cluster.weighted_shards(self.current_sequence)
|
45
47
|
sum = shards_weight.values.inject(&:+)
|
46
48
|
idx = rand(sum)
|
47
|
-
shard,
|
49
|
+
shard, _weight = shards_weight.find { |_k, v|
|
48
50
|
(idx -= v) < 0
|
49
51
|
}
|
52
|
+
shard ||= shards_weight.keys.first
|
50
53
|
self.connection.with_recursive_shards(shard.name, *klasses, &block)
|
51
54
|
end
|
52
55
|
|
53
56
|
def all_cluster_transaction(options = {})
|
54
57
|
clusters = turntable_clusters.values
|
55
|
-
recursive_cluster_transaction(clusters) { yield }
|
58
|
+
recursive_cluster_transaction(clusters, options) { yield }
|
56
59
|
end
|
57
60
|
|
58
61
|
def recursive_cluster_transaction(clusters, options = {}, &block)
|
59
62
|
current_cluster = clusters.shift
|
60
|
-
current_cluster.shards_transaction do
|
63
|
+
current_cluster.shards_transaction([], options) do
|
61
64
|
if clusters.present?
|
62
65
|
recursive_cluster_transaction(clusters, options, &block)
|
63
66
|
else
|
@@ -53,6 +53,7 @@ module ActiveRecord::Turntable
|
|
53
53
|
end
|
54
54
|
end
|
55
55
|
|
56
|
+
# rubocop:disable Style/MethodMissing
|
56
57
|
def method_missing(method, *args, &block)
|
57
58
|
clear_query_cache_if_needed(method)
|
58
59
|
if shard_fixed?
|
@@ -69,6 +70,7 @@ module ActiveRecord::Turntable
|
|
69
70
|
connection.send(method, *args, &block)
|
70
71
|
end
|
71
72
|
end
|
73
|
+
# rubocop:enable Style/MethodMissing
|
72
74
|
|
73
75
|
def respond_to_missing?(method, include_private = false)
|
74
76
|
connection.send(:respond_to?, method, include_private)
|
@@ -198,7 +200,7 @@ module ActiveRecord::Turntable
|
|
198
200
|
end
|
199
201
|
|
200
202
|
delegate :connected?, :automatic_reconnect, :automatic_reconnect=, :checkout_timeout, :dead_connection_timeout,
|
201
|
-
:spec, :connections, :size, :reaper,
|
203
|
+
:spec, :connections, :size, :reaper, to: :connection_pool
|
202
204
|
|
203
205
|
%w(columns columns_hash column_defaults primary_keys).each do |name|
|
204
206
|
define_method(name.to_sym) do
|
@@ -206,7 +208,7 @@ module ActiveRecord::Turntable
|
|
206
208
|
end
|
207
209
|
end
|
208
210
|
|
209
|
-
%w(
|
211
|
+
%w(data_source_exists?).each do |name|
|
210
212
|
define_method(name.to_sym) do |*args|
|
211
213
|
master.connection_pool.with_connection do |c|
|
212
214
|
c.schema_cache.send(name.to_sym, *args)
|
@@ -52,7 +52,6 @@ module ActiveRecord::Turntable::Migration
|
|
52
52
|
options = options.merge(id: false)
|
53
53
|
|
54
54
|
# TODO: pkname should be pulled from table definitions
|
55
|
-
pkname = "id"
|
56
55
|
sequence_table_name = ActiveRecord::Turntable::Sequencer.sequence_name(table_name, "id")
|
57
56
|
create_table(sequence_table_name, options) do |t|
|
58
57
|
t.integer :id, limit: 8
|
@@ -62,7 +61,6 @@ module ActiveRecord::Turntable::Migration
|
|
62
61
|
|
63
62
|
def drop_sequence_for(table_name, options = {})
|
64
63
|
# TODO: pkname should be pulled from table definitions
|
65
|
-
pkname = "id"
|
66
64
|
sequence_table_name = ActiveRecord::Turntable::Sequencer.sequence_name(table_name, "id")
|
67
65
|
drop_table(sequence_table_name)
|
68
66
|
end
|
@@ -108,7 +106,7 @@ module ActiveRecord::Turntable::Migration
|
|
108
106
|
def up(migrations_paths, target_version = nil)
|
109
107
|
super
|
110
108
|
|
111
|
-
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration|
|
109
|
+
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected(current_environment) do |name, configuration|
|
112
110
|
puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})"
|
113
111
|
super(migrations_paths, target_version)
|
114
112
|
end
|
@@ -117,7 +115,7 @@ module ActiveRecord::Turntable::Migration
|
|
117
115
|
def down(migrations_paths, target_version = nil, &block)
|
118
116
|
super
|
119
117
|
|
120
|
-
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration|
|
118
|
+
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected(current_environment) do |name, configuration|
|
121
119
|
puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})"
|
122
120
|
super(migrations_paths, target_version, &block)
|
123
121
|
end
|
@@ -126,7 +124,7 @@ module ActiveRecord::Turntable::Migration
|
|
126
124
|
def run(*args)
|
127
125
|
super
|
128
126
|
|
129
|
-
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration|
|
127
|
+
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected(current_environment) do |name, configuration|
|
130
128
|
puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})"
|
131
129
|
super(*args)
|
132
130
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# rubocop:disable Style/CaseEquality
|
1
2
|
require "active_support/core_ext/object/try"
|
2
3
|
require "active_record/turntable/sql_tree_patch"
|
3
4
|
|
@@ -11,7 +12,7 @@ module ActiveRecord::Turntable
|
|
11
12
|
|
12
13
|
delegate :logger, to: ActiveRecord::Base
|
13
14
|
|
14
|
-
NOT_USED_FOR_SHARDING_OPERATORS_REGEXP = /\A(NOT IN|IS|IS NOT|BETWEEN|LIKE
|
15
|
+
NOT_USED_FOR_SHARDING_OPERATORS_REGEXP = /\A(NOT IN|IS|IS NOT|BETWEEN|LIKE|!=|<<|>>|<>|>=|<=|[*+%|&><-])\z/
|
15
16
|
|
16
17
|
def initialize(proxy)
|
17
18
|
@proxy = proxy
|
@@ -60,7 +61,7 @@ module ActiveRecord::Turntable
|
|
60
61
|
lkeys = find_shard_keys(tree.lhs, table_name, shard_key)
|
61
62
|
rkeys = find_shard_keys(tree.rhs, table_name, shard_key)
|
62
63
|
if lkeys.present? && rkeys.present?
|
63
|
-
lkeys + rkeys
|
64
|
+
(lkeys + rkeys).uniq
|
64
65
|
else
|
65
66
|
[]
|
66
67
|
end
|
@@ -68,22 +69,22 @@ module ActiveRecord::Turntable
|
|
68
69
|
lkeys = find_shard_keys(tree.lhs, table_name, shard_key)
|
69
70
|
rkeys = find_shard_keys(tree.rhs, table_name, shard_key)
|
70
71
|
if lkeys.present? || rkeys.present?
|
71
|
-
lkeys + rkeys
|
72
|
+
(lkeys + rkeys).uniq
|
72
73
|
else
|
73
74
|
[]
|
74
75
|
end
|
75
76
|
when "IN", "=", "=="
|
76
77
|
field = tree.lhs.respond_to?(:table) ? tree.lhs : nil
|
77
78
|
if tree.rhs.is_a?(SQLTree::Node::SubQuery)
|
78
|
-
if field.try(:table) == table_name
|
79
|
+
if (field.try(:table) == table_name) && (field.name == shard_key)
|
79
80
|
find_shard_keys(tree.rhs.where, table_name, shard_key)
|
80
81
|
else
|
81
82
|
[]
|
82
83
|
end
|
83
84
|
else
|
84
85
|
values = Array(tree.rhs)
|
85
|
-
if field.try(:table) == table_name
|
86
|
-
|
86
|
+
if (field.try(:table) == table_name) && (field.name == shard_key) &&
|
87
|
+
!tree.rhs.is_a?(SQLTree::Node::SubQuery)
|
87
88
|
values.map(&:value).compact
|
88
89
|
else
|
89
90
|
[]
|
@@ -113,14 +114,15 @@ module ActiveRecord::Turntable
|
|
113
114
|
end
|
114
115
|
|
115
116
|
def bind_sql(sql, binds)
|
117
|
+
binds = binds ? binds.dup : []
|
116
118
|
# TODO: substitution value should be determined by adapter
|
117
|
-
query = sql.is_a?(String) ? sql : @proxy.to_sql(sql, binds
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
119
|
+
query = sql.is_a?(String) ? sql : @proxy.to_sql(sql, binds)
|
120
|
+
if query.include?("\0") && binds.is_a?(Array) && binds[0].is_a?(Array) && binds[0][0].is_a?(ActiveRecord::ConnectionAdapters::Column)
|
121
|
+
binds = binds.dup
|
122
|
+
query.gsub("\0") { @proxy.master.connection.quote(*binds.shift.reverse) }
|
123
|
+
else
|
124
|
+
query
|
125
|
+
end
|
124
126
|
end
|
125
127
|
|
126
128
|
def build_select_fader(tree, method, query, *args, &block)
|
@@ -133,6 +135,7 @@ module ActiveRecord::Turntable
|
|
133
135
|
@proxy.klass.table_name,
|
134
136
|
@proxy.klass.turntable_shard_key.to_s)
|
135
137
|
end
|
138
|
+
shard_keys.uniq!
|
136
139
|
|
137
140
|
if shard_keys.size == 1 # shard
|
138
141
|
return Fader::SpecifiedShard.new(@proxy,
|
@@ -142,8 +145,7 @@ module ActiveRecord::Turntable
|
|
142
145
|
tree.select.first.to_sql == '1 AS "one"' # for `SELECT 1 AS one` (AR::Base.exists?)
|
143
146
|
return Fader::SelectShardsMergeResult.new(@proxy,
|
144
147
|
build_shards_with_same_query(@proxy.shards.values, query),
|
145
|
-
method, query, *args, &block
|
146
|
-
)
|
148
|
+
method, query, *args, &block)
|
147
149
|
elsif tree.group_by || tree.order_by || tree.limit.try(:value).to_i > 0
|
148
150
|
raise CannotSpecifyShardError, "cannot specify shard for query: #{tree.to_sql}"
|
149
151
|
elsif shard_keys.present?
|
@@ -155,8 +157,7 @@ module ActiveRecord::Turntable
|
|
155
157
|
else
|
156
158
|
return Fader::SelectShardsMergeResult.new(@proxy,
|
157
159
|
Hash[shard_keys.map { |k| [@proxy.cluster.shard_for(k), query] }],
|
158
|
-
method, query, *args, &block
|
159
|
-
)
|
160
|
+
method, query, *args, &block)
|
160
161
|
end
|
161
162
|
else # scan all shards
|
162
163
|
if SQLTree::Node::SelectDeclaration === tree.select.first &&
|
@@ -177,8 +178,7 @@ module ActiveRecord::Turntable
|
|
177
178
|
end
|
178
179
|
return Fader::SelectShardsMergeResult.new(@proxy,
|
179
180
|
build_shards_with_same_query(@proxy.shards.values, query),
|
180
|
-
method, query, *args, &block
|
181
|
-
)
|
181
|
+
method, query, *args, &block)
|
182
182
|
else
|
183
183
|
raise CannotSpecifyShardError, "cannot specify shard for query: #{tree.to_sql}"
|
184
184
|
end
|
@@ -231,3 +231,4 @@ module ActiveRecord::Turntable
|
|
231
231
|
end
|
232
232
|
end
|
233
233
|
end
|
234
|
+
# rubocop:enable Style/CaseEquality
|