activerecord-turntable 3.0.0.alpha3 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitmodules +3 -0
- data/.rubocop.yml +18 -0
- data/.rubocop_todo.yml +153 -0
- data/.travis.yml +20 -4
- data/CHANGELOG.md +32 -0
- data/Guardfile +2 -2
- data/README.md +68 -14
- data/Rakefile +42 -0
- data/activerecord-turntable.gemspec +13 -3
- data/gemfiles/rails_edge.gemfile +8 -0
- data/lib/active_record/turntable/active_record_ext/abstract_adapter.rb +3 -1
- data/lib/active_record/turntable/active_record_ext/activerecord_import_ext.rb +5 -7
- data/lib/active_record/turntable/active_record_ext/acts_as_archive_extension.rb +2 -2
- data/lib/active_record/turntable/active_record_ext/association.rb +3 -3
- data/lib/active_record/turntable/active_record_ext/clever_load.rb +2 -2
- data/lib/active_record/turntable/active_record_ext/database_tasks.rb +10 -8
- data/lib/active_record/turntable/active_record_ext/fixtures.rb +15 -13
- data/lib/active_record/turntable/active_record_ext/log_subscriber.rb +6 -0
- data/lib/active_record/turntable/active_record_ext/persistence.rb +25 -23
- data/lib/active_record/turntable/active_record_ext/schema_dumper.rb +8 -75
- data/lib/active_record/turntable/algorithm/range_algorithm.rb +6 -7
- data/lib/active_record/turntable/algorithm/range_bsearch_algorithm.rb +6 -7
- data/lib/active_record/turntable/base.rb +2 -17
- data/lib/active_record/turntable/cluster_helper_methods.rb +7 -4
- data/lib/active_record/turntable/connection_proxy.rb +4 -2
- data/lib/active_record/turntable/migration.rb +3 -5
- data/lib/active_record/turntable/mixer.rb +20 -19
- data/lib/active_record/turntable/pool_proxy.rb +20 -14
- data/lib/active_record/turntable/query_cache.rb +1 -1
- data/lib/active_record/turntable/railties/databases.rake +12 -12
- data/lib/active_record/turntable/seq_shard.rb +1 -1
- data/lib/active_record/turntable/sequencer/barrage.rb +3 -2
- data/lib/active_record/turntable/sequencer.rb +33 -29
- data/lib/active_record/turntable/shard.rb +8 -8
- data/lib/active_record/turntable/sharding_condition.rb +14 -14
- data/lib/active_record/turntable/sql_tree_patch.rb +7 -3
- data/lib/active_record/turntable/util.rb +4 -2
- data/lib/active_record/turntable/version.rb +1 -1
- data/lib/active_record/turntable.rb +6 -5
- data/lib/activerecord-turntable.rb +1 -0
- metadata +120 -101
- data/lib/active_record/turntable/helpers/test_helper.rb +0 -25
- data/lib/active_record/turntable/helpers.rb +0 -9
- data/spec/active_record/turntable/active_record_ext/association_preloader_spec.rb +0 -78
- data/spec/active_record/turntable/active_record_ext/association_spec.rb +0 -81
- data/spec/active_record/turntable/active_record_ext/clever_load_spec.rb +0 -72
- data/spec/active_record/turntable/active_record_ext/fixture_set_spec.rb +0 -27
- data/spec/active_record/turntable/active_record_ext/locking_optimistic_spec.rb +0 -28
- data/spec/active_record/turntable/active_record_ext/migration_spec.rb +0 -38
- data/spec/active_record/turntable/active_record_ext/persistence_spec.rb +0 -211
- data/spec/active_record/turntable/active_record_ext/sequencer_spec.rb +0 -22
- data/spec/active_record/turntable/active_record_ext/test_fixtures_spec.rb +0 -34
- data/spec/active_record/turntable/algorithm/modulo_algorithm_spec.rb +0 -34
- data/spec/active_record/turntable/algorithm/range_algorithm_spec.rb +0 -34
- data/spec/active_record/turntable/algorithm/range_bsearch_algorithm_spec.rb +0 -34
- data/spec/active_record/turntable/algorithm_spec.rb +0 -100
- data/spec/active_record/turntable/base_spec.rb +0 -13
- data/spec/active_record/turntable/cluster_spec.rb +0 -48
- data/spec/active_record/turntable/config_spec.rb +0 -17
- data/spec/active_record/turntable/connection_proxy_spec.rb +0 -252
- data/spec/active_record/turntable/finder_spec.rb +0 -40
- data/spec/active_record/turntable/mixer/fader_spec.rb +0 -4
- data/spec/active_record/turntable/mixer_spec.rb +0 -112
- data/spec/active_record/turntable/query_cache_spec.rb +0 -28
- data/spec/active_record/turntable/sequencer/api_spec.rb +0 -38
- data/spec/active_record/turntable/sequencer/barrage_spec.rb +0 -22
- data/spec/active_record/turntable/sequencer/mysql_spec.rb +0 -22
- data/spec/active_record/turntable/shard_spec.rb +0 -21
- data/spec/active_record/turntable/sql_tree_patch_spec.rb +0 -34
- data/spec/active_record/turntable/transaction_spec.rb +0 -35
- data/spec/active_record/turntable_spec.rb +0 -30
- data/spec/config/database.yml +0 -35
- data/spec/config/turntable.yml +0 -56
- data/spec/fabricators/.gitkeep +0 -0
- data/spec/fabricators/turntable_fabricator.rb +0 -12
- data/spec/fixtures/cards.yml +0 -11
- data/spec/migrations/.gitkeep +0 -0
- data/spec/migrations/001_create_users.rb +0 -17
- data/spec/migrations/002_create_user_statuses.rb +0 -16
- data/spec/migrations/003_create_cards.rb +0 -14
- data/spec/migrations/004_create_cards_users.rb +0 -15
- data/spec/models/card.rb +0 -3
- data/spec/models/cards_user.rb +0 -10
- data/spec/models/cards_users_histories.rb +0 -7
- data/spec/models/events_users_history.rb +0 -7
- data/spec/models/user.rb +0 -7
- data/spec/models/user_status.rb +0 -6
- data/spec/spec_helper.rb +0 -38
- data/spec/support/matchers/be_saved_to.rb +0 -6
- data/spec/support/turntable_helper.rb +0 -30
|
@@ -18,7 +18,7 @@ module ActiveRecord
|
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
def create_current_turntable_cluster(environment = env)
|
|
21
|
-
each_current_turntable_cluster_configuration(
|
|
21
|
+
each_current_turntable_cluster_configuration(environment) { |_name, configuration|
|
|
22
22
|
puts "[turntable] *** executing to database: #{configuration['database']}"
|
|
23
23
|
create configuration
|
|
24
24
|
}
|
|
@@ -26,26 +26,27 @@ module ActiveRecord
|
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
def drop_current_turntable_cluster(environment = env)
|
|
29
|
-
each_current_turntable_cluster_configuration(
|
|
29
|
+
each_current_turntable_cluster_configuration(environment) { |_name, configuration|
|
|
30
30
|
puts "[turntable] *** executing to database: #{configuration['database']}"
|
|
31
31
|
drop configuration
|
|
32
32
|
}
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
def each_current_turntable_cluster_connected(
|
|
36
|
-
|
|
35
|
+
def each_current_turntable_cluster_connected(environment)
|
|
36
|
+
old_connection_pool = ActiveRecord::Base.connection_pool
|
|
37
|
+
each_current_turntable_cluster_configuration(environment) do |name, configuration|
|
|
37
38
|
ActiveRecord::Base.clear_active_connections!
|
|
38
39
|
ActiveRecord::Base.establish_connection(configuration)
|
|
39
40
|
ActiveRecord::Migration.current_shard = name
|
|
40
41
|
yield(name, configuration)
|
|
41
42
|
end
|
|
42
43
|
ActiveRecord::Base.clear_active_connections!
|
|
43
|
-
ActiveRecord::Base.establish_connection
|
|
44
|
+
ActiveRecord::Base.establish_connection old_connection_pool.spec.config
|
|
44
45
|
end
|
|
45
46
|
|
|
46
|
-
def each_current_turntable_cluster_configuration(
|
|
47
|
+
def each_current_turntable_cluster_configuration(environment)
|
|
47
48
|
environments = [environment]
|
|
48
|
-
environments << "test" if
|
|
49
|
+
environments << "test" if environment == "development"
|
|
49
50
|
|
|
50
51
|
current_turntable_cluster_configurations(*environments).each do |name, configuration|
|
|
51
52
|
yield(name, configuration) unless configuration["database"].blank?
|
|
@@ -70,8 +71,9 @@ module ActiveRecord
|
|
|
70
71
|
configurations = []
|
|
71
72
|
environments.each do |environ|
|
|
72
73
|
config = ActiveRecord::Base.configurations[environ]
|
|
74
|
+
next unless config
|
|
73
75
|
%w(shards seq).each do |name|
|
|
74
|
-
configurations += config[name].to_a
|
|
76
|
+
configurations += config[name].to_a if config.has_key?(name)
|
|
75
77
|
end
|
|
76
78
|
end
|
|
77
79
|
configurations
|
|
@@ -7,6 +7,7 @@ module ActiveRecord
|
|
|
7
7
|
class FixtureSet
|
|
8
8
|
extend ActiveRecord::Turntable::Util
|
|
9
9
|
|
|
10
|
+
# rubocop:disable Style/MultilineMethodCallBraceLayout
|
|
10
11
|
def self.create_fixtures(fixtures_directory, fixture_set_names, class_names = {}, config = ActiveRecord::Base)
|
|
11
12
|
fixture_set_names = Array(fixture_set_names).map(&:to_s)
|
|
12
13
|
class_names = ClassCache.new class_names, config
|
|
@@ -24,7 +25,7 @@ module ActiveRecord
|
|
|
24
25
|
|
|
25
26
|
fixture_sets = files_to_read.map do |fs_name|
|
|
26
27
|
klass = class_names[fs_name]
|
|
27
|
-
conn = klass
|
|
28
|
+
conn = klass ? klass.connection : connection
|
|
28
29
|
fixtures_map[fs_name] = new( # ActiveRecord::FixtureSet.new
|
|
29
30
|
conn,
|
|
30
31
|
fs_name,
|
|
@@ -35,12 +36,16 @@ module ActiveRecord
|
|
|
35
36
|
update_all_loaded_fixtures fixtures_map
|
|
36
37
|
|
|
37
38
|
ActiveRecord::Base.force_transaction_all_shards!(requires_new: true) do
|
|
39
|
+
deleted_tables = Hash.new { |h, k| h[k] = Set.new }
|
|
38
40
|
fixture_sets.each do |fs|
|
|
39
41
|
conn = fs.model_class.respond_to?(:connection) ? fs.model_class.connection : connection
|
|
40
42
|
table_rows = fs.table_rows
|
|
41
43
|
|
|
42
44
|
table_rows.each_key do |table|
|
|
43
|
-
conn.
|
|
45
|
+
unless deleted_tables[conn].include? table
|
|
46
|
+
conn.delete "DELETE FROM #{conn.quote_table_name(table)}", "Fixture Delete"
|
|
47
|
+
end
|
|
48
|
+
deleted_tables[conn] << table
|
|
44
49
|
end
|
|
45
50
|
|
|
46
51
|
table_rows.each do |fixture_set_name, rows|
|
|
@@ -48,11 +53,9 @@ module ActiveRecord
|
|
|
48
53
|
conn.insert_fixture(row, fixture_set_name)
|
|
49
54
|
end
|
|
50
55
|
end
|
|
51
|
-
end
|
|
52
56
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
fixture_sets.each do |fs|
|
|
57
|
+
# Cap primary key sequences to max(pk).
|
|
58
|
+
if connection.respond_to?(:reset_pk_sequence!)
|
|
56
59
|
connection.reset_pk_sequence!(fs.table_name)
|
|
57
60
|
end
|
|
58
61
|
end
|
|
@@ -63,16 +66,14 @@ module ActiveRecord
|
|
|
63
66
|
end
|
|
64
67
|
cached_fixtures(connection, fixture_set_names)
|
|
65
68
|
end
|
|
69
|
+
# rubocop:enable Style/MultilineMethodCallLayout
|
|
66
70
|
end
|
|
67
71
|
|
|
68
72
|
module TestFixtures
|
|
69
|
-
|
|
70
|
-
|
|
73
|
+
# rubocop:disable Style/ClassVars, Style/RedundantException
|
|
71
74
|
def setup_fixtures(config = ActiveRecord::Base)
|
|
72
|
-
return if ActiveRecord::Base.configurations.blank?
|
|
73
|
-
|
|
74
75
|
if pre_loaded_fixtures && !use_transactional_fixtures
|
|
75
|
-
raise "pre_loaded_fixtures requires use_transactional_fixtures"
|
|
76
|
+
raise RuntimeError, "pre_loaded_fixtures requires use_transactional_fixtures"
|
|
76
77
|
end
|
|
77
78
|
|
|
78
79
|
@fixture_cache = {}
|
|
@@ -94,14 +95,15 @@ module ActiveRecord
|
|
|
94
95
|
end
|
|
95
96
|
# Load fixtures for every test.
|
|
96
97
|
else
|
|
97
|
-
ActiveRecord::
|
|
98
|
+
ActiveRecord::FixtureSet.reset_cache
|
|
98
99
|
@@already_loaded_fixtures[self.class] = nil
|
|
99
100
|
@loaded_fixtures = load_fixtures(config)
|
|
100
101
|
end
|
|
101
102
|
|
|
102
103
|
# Instantiate fixtures for every test if requested.
|
|
103
|
-
instantiate_fixtures
|
|
104
|
+
instantiate_fixtures if use_instantiated_fixtures
|
|
104
105
|
end
|
|
106
|
+
# rubocop:enable Style/ClassVars, Style/RedundantException
|
|
105
107
|
|
|
106
108
|
def enlist_fixture_connections
|
|
107
109
|
ActiveRecord::Base.connection_handler.connection_pool_list.map(&:connection) +
|
|
@@ -6,6 +6,12 @@ module ActiveRecord::Turntable
|
|
|
6
6
|
# @note prepend to add shard name logging
|
|
7
7
|
def sql(event)
|
|
8
8
|
payload = event.payload
|
|
9
|
+
|
|
10
|
+
if self.class::IGNORE_PAYLOAD_NAMES.include?(payload[:name])
|
|
11
|
+
self.class.runtime += event.duration
|
|
12
|
+
return
|
|
13
|
+
end
|
|
14
|
+
|
|
9
15
|
if payload[:turntable_shard_name]
|
|
10
16
|
payload[:name] = "#{payload[:name]} [Shard: #{payload[:turntable_shard_name]}]"
|
|
11
17
|
end
|
|
@@ -27,8 +27,9 @@ module ActiveRecord::Turntable
|
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
# @note Override to add sharding scope on `touch`
|
|
30
|
+
# rubocop:disable Style/UnlessElse
|
|
30
31
|
def touch(*names, time: nil)
|
|
31
|
-
raise ActiveRecordError, "cannot touch on a new record object" unless persisted?
|
|
32
|
+
raise ActiveRecord::ActiveRecordError, "cannot touch on a new record object" unless persisted?
|
|
32
33
|
|
|
33
34
|
time ||= current_time_from_proper_timezone
|
|
34
35
|
attributes = timestamp_attributes_for_update_in_model
|
|
@@ -68,11 +69,12 @@ module ActiveRecord::Turntable
|
|
|
68
69
|
true
|
|
69
70
|
end
|
|
70
71
|
end
|
|
72
|
+
# rubocop:enable Style/UnlessElse
|
|
71
73
|
|
|
72
74
|
# @note Override to add sharding scope on `update_columns`
|
|
73
75
|
def update_columns(attributes)
|
|
74
|
-
raise ActiveRecordError, "cannot update a new record" if new_record?
|
|
75
|
-
raise ActiveRecordError, "cannot update a destroyed record" if destroyed?
|
|
76
|
+
raise ActiveRecord::ActiveRecordError, "cannot update a new record" if new_record?
|
|
77
|
+
raise ActiveRecord::ActiveRecordError, "cannot update a destroyed record" if destroyed?
|
|
76
78
|
|
|
77
79
|
attributes.each_key do |key|
|
|
78
80
|
verify_readonly_attribute(key.to_s)
|
|
@@ -95,30 +97,30 @@ module ActiveRecord::Turntable
|
|
|
95
97
|
|
|
96
98
|
private
|
|
97
99
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
100
|
+
# @note Override to add sharding scope on destroying
|
|
101
|
+
def relation_for_destroy
|
|
102
|
+
klass = self.class
|
|
103
|
+
relation = klass.unscoped.where(klass.primary_key => id)
|
|
102
104
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
+
if klass.turntable_enabled? && klass.primary_key != klass.turntable_shard_key.to_s
|
|
106
|
+
relation = relation.where(klass.turntable_shard_key => self[klass.turntable_shard_key])
|
|
107
|
+
end
|
|
108
|
+
relation
|
|
105
109
|
end
|
|
106
|
-
relation
|
|
107
|
-
end
|
|
108
110
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
111
|
+
# @note Override to add sharding scope on updating
|
|
112
|
+
def _update_record(attribute_names = self.attribute_names)
|
|
113
|
+
klass = self.class
|
|
114
|
+
attributes_values = arel_attributes_with_values_for_update(attribute_names)
|
|
115
|
+
if attributes_values.empty?
|
|
116
|
+
0
|
|
117
|
+
else
|
|
118
|
+
scope = if klass.turntable_enabled? && (klass.primary_key != klass.turntable_shard_key.to_s)
|
|
119
|
+
klass.unscoped.where(klass.turntable_shard_key => self.send(turntable_shard_key))
|
|
120
|
+
end
|
|
121
|
+
klass.unscoped._update_record attributes_values, id, id_was, scope
|
|
122
|
+
end
|
|
120
123
|
end
|
|
121
|
-
end
|
|
122
124
|
end
|
|
123
125
|
end
|
|
124
126
|
end
|
|
@@ -2,93 +2,26 @@
|
|
|
2
2
|
module ActiveRecord::Turntable
|
|
3
3
|
module ActiveRecordExt
|
|
4
4
|
module SchemaDumper
|
|
5
|
+
SEQUENCE_TABLE_REGEXP = /\A(.*)_id_seq\z/
|
|
5
6
|
|
|
6
7
|
private
|
|
7
8
|
|
|
8
9
|
# @note Override to dump database sequencer method
|
|
9
10
|
def table(table, stream)
|
|
10
|
-
|
|
11
|
+
unless matchdata = table.match(SEQUENCE_TABLE_REGEXP)
|
|
12
|
+
return super
|
|
13
|
+
end
|
|
14
|
+
|
|
11
15
|
begin
|
|
12
16
|
tbl = StringIO.new
|
|
13
17
|
|
|
14
|
-
|
|
15
|
-
if @connection.respond_to?(:primary_keys)
|
|
16
|
-
pk = @connection.primary_keys(table)
|
|
17
|
-
pk = pk.first unless pk.size > 1
|
|
18
|
-
else
|
|
19
|
-
pk = @connection.primary_key(table)
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
if table =~ /\A(.*)_id_seq\z/
|
|
23
|
-
tbl.print " create_sequence_for #{remove_prefix_and_suffix(Regexp.last_match(1)).inspect}"
|
|
24
|
-
else
|
|
25
|
-
tbl.print " create_table #{remove_prefix_and_suffix(table).inspect}"
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
case pk
|
|
29
|
-
when String
|
|
30
|
-
tbl.print ", primary_key: #{pk.inspect}" unless pk == 'id'
|
|
31
|
-
pkcol = columns.detect { |c| c.name == pk }
|
|
32
|
-
pkcolspec = @connection.column_spec_for_primary_key(pkcol)
|
|
33
|
-
if pkcolspec.present?
|
|
34
|
-
pkcolspec.each do |key, value|
|
|
35
|
-
tbl.print ", #{key}: #{value}"
|
|
36
|
-
end
|
|
37
|
-
end
|
|
38
|
-
when Array
|
|
39
|
-
tbl.print ", primary_key: #{pk.inspect}"
|
|
40
|
-
else
|
|
41
|
-
tbl.print ", id: false"
|
|
42
|
-
end
|
|
18
|
+
tbl.print " create_sequence_for #{remove_prefix_and_suffix(matchdata[1]).inspect}"
|
|
43
19
|
tbl.print ", force: :cascade"
|
|
44
20
|
|
|
45
21
|
table_options = @connection.table_options(table)
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if comment = @connection.table_comment(table).presence
|
|
49
|
-
tbl.print ", comment: #{comment.inspect}"
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
tbl.puts " do |t|"
|
|
53
|
-
|
|
54
|
-
# then dump all non-primary key columns
|
|
55
|
-
column_specs = columns.map do |column|
|
|
56
|
-
raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" unless @connection.valid_type?(column.type)
|
|
57
|
-
next if column.name == pk
|
|
58
|
-
@connection.column_spec(column)
|
|
59
|
-
end.compact
|
|
60
|
-
|
|
61
|
-
# find all migration keys used in this table
|
|
62
|
-
keys = @connection.migration_keys
|
|
63
|
-
|
|
64
|
-
# figure out the lengths for each column based on above keys
|
|
65
|
-
lengths = keys.map { |key|
|
|
66
|
-
column_specs.map { |spec|
|
|
67
|
-
spec[key] ? spec[key].length + 2 : 0
|
|
68
|
-
}.max
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
# the string we're going to sprintf our values against, with standardized column widths
|
|
72
|
-
format_string = lengths.map{ |len| "%-#{len}s" }
|
|
73
|
-
|
|
74
|
-
# find the max length for the 'type' column, which is special
|
|
75
|
-
type_length = column_specs.map{ |column| column[:type].length }.max
|
|
76
|
-
|
|
77
|
-
# add column type definition to our format string
|
|
78
|
-
format_string.unshift " t.%-#{type_length}s "
|
|
79
|
-
|
|
80
|
-
format_string *= ''
|
|
81
|
-
|
|
82
|
-
column_specs.each do |colspec|
|
|
83
|
-
values = keys.zip(lengths).map{ |key, len| colspec.key?(key) ? colspec[key] + ", " : " " * len }
|
|
84
|
-
values.unshift colspec[:type]
|
|
85
|
-
tbl.print((format_string % values).gsub(/,\s*$/, ''))
|
|
86
|
-
tbl.puts
|
|
22
|
+
if table_options.present?
|
|
23
|
+
tbl.print ", #{format_options(table_options)}"
|
|
87
24
|
end
|
|
88
|
-
|
|
89
|
-
indexes_in_create(table, tbl)
|
|
90
|
-
|
|
91
|
-
tbl.puts " end"
|
|
92
25
|
tbl.puts
|
|
93
26
|
|
|
94
27
|
tbl.rewind
|
|
@@ -18,17 +18,16 @@ module ActiveRecord::Turntable::Algorithm
|
|
|
18
18
|
|
|
19
19
|
# { connection_name => weight, ... }
|
|
20
20
|
def calculate_used_shards_with_weight(sequence_value)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
shards = @config[:shards][0..idx]
|
|
21
|
+
current_shard_idx = calculate_idx(sequence_value)
|
|
22
|
+
shards = @config[:shards][0..current_shard_idx]
|
|
24
23
|
weighted_hash = Hash.new { |h, k| h[k] = 0 }
|
|
25
24
|
prev_max = 0
|
|
26
25
|
shards.each_with_index do |h, idx|
|
|
27
26
|
weighted_hash[h[:connection]] += if idx < shards.size - 1
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
27
|
+
h[:less_than] - prev_max - 1
|
|
28
|
+
else
|
|
29
|
+
sequence_value - prev_max
|
|
30
|
+
end
|
|
32
31
|
prev_max = h[:less_than] - 1
|
|
33
32
|
end
|
|
34
33
|
weighted_hash
|
|
@@ -22,17 +22,16 @@ module ActiveRecord::Turntable::Algorithm
|
|
|
22
22
|
|
|
23
23
|
# { connection_name => weight, ... }
|
|
24
24
|
def calculate_used_shards_with_weight(sequence_value)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
shards = @config[:shards][0..idx]
|
|
25
|
+
current_shard_idx = calculate_idx(sequence_value)
|
|
26
|
+
shards = @config[:shards][0..current_shard_idx]
|
|
28
27
|
weighted_hash = Hash.new { |h, k| h[k] = 0 }
|
|
29
28
|
prev_max = 0
|
|
30
29
|
shards.each_with_index do |h, idx|
|
|
31
30
|
weighted_hash[h[:connection]] += if idx < shards.size - 1
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
31
|
+
h[:less_than] - prev_max - 1
|
|
32
|
+
else
|
|
33
|
+
sequence_value - prev_max
|
|
34
|
+
end
|
|
36
35
|
prev_max = h[:less_than] - 1
|
|
37
36
|
end
|
|
38
37
|
weighted_hash
|
|
@@ -22,21 +22,6 @@ module ActiveRecord::Turntable
|
|
|
22
22
|
include ClusterHelperMethods
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
-
module ConnectionExtension
|
|
26
|
-
module ClassMethods
|
|
27
|
-
def connection_specification_name
|
|
28
|
-
return super unless turntable_enabled?
|
|
29
|
-
self.connection_specification_name = "turntable_pool_proxy::#{name}"
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
def self.prepended(base)
|
|
34
|
-
class << base
|
|
35
|
-
prepend ClassMethods
|
|
36
|
-
end
|
|
37
|
-
end
|
|
38
|
-
end
|
|
39
|
-
|
|
40
25
|
module ClassMethods
|
|
41
26
|
# @param [Symbol] cluster_name cluster name for this class
|
|
42
27
|
# @param [Symbol] shard_key_name shard key attribute name
|
|
@@ -52,8 +37,7 @@ module ActiveRecord::Turntable
|
|
|
52
37
|
self.turntable_clusters[cluster_name] ||= Cluster.new(
|
|
53
38
|
turntable_config[:clusters][cluster_name],
|
|
54
39
|
options
|
|
55
|
-
|
|
56
|
-
prepend ConnectionExtension
|
|
40
|
+
)
|
|
57
41
|
turntable_replace_connection_pool
|
|
58
42
|
end
|
|
59
43
|
|
|
@@ -62,6 +46,7 @@ module ActiveRecord::Turntable
|
|
|
62
46
|
cp = ConnectionProxy.new(self, turntable_cluster)
|
|
63
47
|
pp = PoolProxy.new(cp)
|
|
64
48
|
|
|
49
|
+
self.connection_specification_name = "turntable_pool_proxy::#{name}"
|
|
65
50
|
ch.send(:owner_to_pool)[connection_specification_name] = pp
|
|
66
51
|
end
|
|
67
52
|
|
|
@@ -30,7 +30,9 @@ module ActiveRecord::Turntable
|
|
|
30
30
|
end
|
|
31
31
|
|
|
32
32
|
def force_connect_all_shards!
|
|
33
|
-
conf = configurations[
|
|
33
|
+
conf = configurations[::ActiveRecord::ConnectionHandling::DEFAULT_ENV.call.to_sym]
|
|
34
|
+
return unless conf
|
|
35
|
+
|
|
34
36
|
shards = HashWithIndifferentAccess.new
|
|
35
37
|
shards = shards.merge(conf[:shards]) if conf[:shards]
|
|
36
38
|
shards = shards.merge(conf[:seq]) if conf[:seq]
|
|
@@ -44,20 +46,21 @@ module ActiveRecord::Turntable
|
|
|
44
46
|
shards_weight = self.turntable_cluster.weighted_shards(self.current_sequence)
|
|
45
47
|
sum = shards_weight.values.inject(&:+)
|
|
46
48
|
idx = rand(sum)
|
|
47
|
-
shard,
|
|
49
|
+
shard, _weight = shards_weight.find { |_k, v|
|
|
48
50
|
(idx -= v) < 0
|
|
49
51
|
}
|
|
52
|
+
shard ||= shards_weight.keys.first
|
|
50
53
|
self.connection.with_recursive_shards(shard.name, *klasses, &block)
|
|
51
54
|
end
|
|
52
55
|
|
|
53
56
|
def all_cluster_transaction(options = {})
|
|
54
57
|
clusters = turntable_clusters.values
|
|
55
|
-
recursive_cluster_transaction(clusters) { yield }
|
|
58
|
+
recursive_cluster_transaction(clusters, options) { yield }
|
|
56
59
|
end
|
|
57
60
|
|
|
58
61
|
def recursive_cluster_transaction(clusters, options = {}, &block)
|
|
59
62
|
current_cluster = clusters.shift
|
|
60
|
-
current_cluster.shards_transaction do
|
|
63
|
+
current_cluster.shards_transaction([], options) do
|
|
61
64
|
if clusters.present?
|
|
62
65
|
recursive_cluster_transaction(clusters, options, &block)
|
|
63
66
|
else
|
|
@@ -53,6 +53,7 @@ module ActiveRecord::Turntable
|
|
|
53
53
|
end
|
|
54
54
|
end
|
|
55
55
|
|
|
56
|
+
# rubocop:disable Style/MethodMissing
|
|
56
57
|
def method_missing(method, *args, &block)
|
|
57
58
|
clear_query_cache_if_needed(method)
|
|
58
59
|
if shard_fixed?
|
|
@@ -69,6 +70,7 @@ module ActiveRecord::Turntable
|
|
|
69
70
|
connection.send(method, *args, &block)
|
|
70
71
|
end
|
|
71
72
|
end
|
|
73
|
+
# rubocop:enable Style/MethodMissing
|
|
72
74
|
|
|
73
75
|
def respond_to_missing?(method, include_private = false)
|
|
74
76
|
connection.send(:respond_to?, method, include_private)
|
|
@@ -198,7 +200,7 @@ module ActiveRecord::Turntable
|
|
|
198
200
|
end
|
|
199
201
|
|
|
200
202
|
delegate :connected?, :automatic_reconnect, :automatic_reconnect=, :checkout_timeout, :dead_connection_timeout,
|
|
201
|
-
:spec, :connections, :size, :reaper,
|
|
203
|
+
:spec, :connections, :size, :reaper, to: :connection_pool
|
|
202
204
|
|
|
203
205
|
%w(columns columns_hash column_defaults primary_keys).each do |name|
|
|
204
206
|
define_method(name.to_sym) do
|
|
@@ -206,7 +208,7 @@ module ActiveRecord::Turntable
|
|
|
206
208
|
end
|
|
207
209
|
end
|
|
208
210
|
|
|
209
|
-
%w(
|
|
211
|
+
%w(data_source_exists?).each do |name|
|
|
210
212
|
define_method(name.to_sym) do |*args|
|
|
211
213
|
master.connection_pool.with_connection do |c|
|
|
212
214
|
c.schema_cache.send(name.to_sym, *args)
|
|
@@ -52,7 +52,6 @@ module ActiveRecord::Turntable::Migration
|
|
|
52
52
|
options = options.merge(id: false)
|
|
53
53
|
|
|
54
54
|
# TODO: pkname should be pulled from table definitions
|
|
55
|
-
pkname = "id"
|
|
56
55
|
sequence_table_name = ActiveRecord::Turntable::Sequencer.sequence_name(table_name, "id")
|
|
57
56
|
create_table(sequence_table_name, options) do |t|
|
|
58
57
|
t.integer :id, limit: 8
|
|
@@ -62,7 +61,6 @@ module ActiveRecord::Turntable::Migration
|
|
|
62
61
|
|
|
63
62
|
def drop_sequence_for(table_name, options = {})
|
|
64
63
|
# TODO: pkname should be pulled from table definitions
|
|
65
|
-
pkname = "id"
|
|
66
64
|
sequence_table_name = ActiveRecord::Turntable::Sequencer.sequence_name(table_name, "id")
|
|
67
65
|
drop_table(sequence_table_name)
|
|
68
66
|
end
|
|
@@ -108,7 +106,7 @@ module ActiveRecord::Turntable::Migration
|
|
|
108
106
|
def up(migrations_paths, target_version = nil)
|
|
109
107
|
super
|
|
110
108
|
|
|
111
|
-
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration|
|
|
109
|
+
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected(current_environment) do |name, configuration|
|
|
112
110
|
puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})"
|
|
113
111
|
super(migrations_paths, target_version)
|
|
114
112
|
end
|
|
@@ -117,7 +115,7 @@ module ActiveRecord::Turntable::Migration
|
|
|
117
115
|
def down(migrations_paths, target_version = nil, &block)
|
|
118
116
|
super
|
|
119
117
|
|
|
120
|
-
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration|
|
|
118
|
+
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected(current_environment) do |name, configuration|
|
|
121
119
|
puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})"
|
|
122
120
|
super(migrations_paths, target_version, &block)
|
|
123
121
|
end
|
|
@@ -126,7 +124,7 @@ module ActiveRecord::Turntable::Migration
|
|
|
126
124
|
def run(*args)
|
|
127
125
|
super
|
|
128
126
|
|
|
129
|
-
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected do |name, configuration|
|
|
127
|
+
ActiveRecord::Tasks::DatabaseTasks.each_current_turntable_cluster_connected(current_environment) do |name, configuration|
|
|
130
128
|
puts "[turntable] *** Migrating database: #{configuration['database']}(Shard: #{name})"
|
|
131
129
|
super(*args)
|
|
132
130
|
end
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# rubocop:disable Style/CaseEquality
|
|
1
2
|
require "active_support/core_ext/object/try"
|
|
2
3
|
require "active_record/turntable/sql_tree_patch"
|
|
3
4
|
|
|
@@ -11,7 +12,7 @@ module ActiveRecord::Turntable
|
|
|
11
12
|
|
|
12
13
|
delegate :logger, to: ActiveRecord::Base
|
|
13
14
|
|
|
14
|
-
NOT_USED_FOR_SHARDING_OPERATORS_REGEXP = /\A(NOT IN|IS|IS NOT|BETWEEN|LIKE
|
|
15
|
+
NOT_USED_FOR_SHARDING_OPERATORS_REGEXP = /\A(NOT IN|IS|IS NOT|BETWEEN|LIKE|!=|<<|>>|<>|>=|<=|[*+%|&><-])\z/
|
|
15
16
|
|
|
16
17
|
def initialize(proxy)
|
|
17
18
|
@proxy = proxy
|
|
@@ -60,7 +61,7 @@ module ActiveRecord::Turntable
|
|
|
60
61
|
lkeys = find_shard_keys(tree.lhs, table_name, shard_key)
|
|
61
62
|
rkeys = find_shard_keys(tree.rhs, table_name, shard_key)
|
|
62
63
|
if lkeys.present? && rkeys.present?
|
|
63
|
-
lkeys + rkeys
|
|
64
|
+
(lkeys + rkeys).uniq
|
|
64
65
|
else
|
|
65
66
|
[]
|
|
66
67
|
end
|
|
@@ -68,22 +69,22 @@ module ActiveRecord::Turntable
|
|
|
68
69
|
lkeys = find_shard_keys(tree.lhs, table_name, shard_key)
|
|
69
70
|
rkeys = find_shard_keys(tree.rhs, table_name, shard_key)
|
|
70
71
|
if lkeys.present? || rkeys.present?
|
|
71
|
-
lkeys + rkeys
|
|
72
|
+
(lkeys + rkeys).uniq
|
|
72
73
|
else
|
|
73
74
|
[]
|
|
74
75
|
end
|
|
75
76
|
when "IN", "=", "=="
|
|
76
77
|
field = tree.lhs.respond_to?(:table) ? tree.lhs : nil
|
|
77
78
|
if tree.rhs.is_a?(SQLTree::Node::SubQuery)
|
|
78
|
-
if field.try(:table) == table_name
|
|
79
|
+
if (field.try(:table) == table_name) && (field.name == shard_key)
|
|
79
80
|
find_shard_keys(tree.rhs.where, table_name, shard_key)
|
|
80
81
|
else
|
|
81
82
|
[]
|
|
82
83
|
end
|
|
83
84
|
else
|
|
84
85
|
values = Array(tree.rhs)
|
|
85
|
-
if field.try(:table) == table_name
|
|
86
|
-
|
|
86
|
+
if (field.try(:table) == table_name) && (field.name == shard_key) &&
|
|
87
|
+
!tree.rhs.is_a?(SQLTree::Node::SubQuery)
|
|
87
88
|
values.map(&:value).compact
|
|
88
89
|
else
|
|
89
90
|
[]
|
|
@@ -113,14 +114,15 @@ module ActiveRecord::Turntable
|
|
|
113
114
|
end
|
|
114
115
|
|
|
115
116
|
def bind_sql(sql, binds)
|
|
117
|
+
binds = binds ? binds.dup : []
|
|
116
118
|
# TODO: substitution value should be determined by adapter
|
|
117
|
-
query = sql.is_a?(String) ? sql : @proxy.to_sql(sql, binds
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
119
|
+
query = sql.is_a?(String) ? sql : @proxy.to_sql(sql, binds)
|
|
120
|
+
if query.include?("\0") && binds.is_a?(Array) && binds[0].is_a?(Array) && binds[0][0].is_a?(ActiveRecord::ConnectionAdapters::Column)
|
|
121
|
+
binds = binds.dup
|
|
122
|
+
query.gsub("\0") { @proxy.master.connection.quote(*binds.shift.reverse) }
|
|
123
|
+
else
|
|
124
|
+
query
|
|
125
|
+
end
|
|
124
126
|
end
|
|
125
127
|
|
|
126
128
|
def build_select_fader(tree, method, query, *args, &block)
|
|
@@ -133,6 +135,7 @@ module ActiveRecord::Turntable
|
|
|
133
135
|
@proxy.klass.table_name,
|
|
134
136
|
@proxy.klass.turntable_shard_key.to_s)
|
|
135
137
|
end
|
|
138
|
+
shard_keys.uniq!
|
|
136
139
|
|
|
137
140
|
if shard_keys.size == 1 # shard
|
|
138
141
|
return Fader::SpecifiedShard.new(@proxy,
|
|
@@ -142,8 +145,7 @@ module ActiveRecord::Turntable
|
|
|
142
145
|
tree.select.first.to_sql == '1 AS "one"' # for `SELECT 1 AS one` (AR::Base.exists?)
|
|
143
146
|
return Fader::SelectShardsMergeResult.new(@proxy,
|
|
144
147
|
build_shards_with_same_query(@proxy.shards.values, query),
|
|
145
|
-
method, query, *args, &block
|
|
146
|
-
)
|
|
148
|
+
method, query, *args, &block)
|
|
147
149
|
elsif tree.group_by || tree.order_by || tree.limit.try(:value).to_i > 0
|
|
148
150
|
raise CannotSpecifyShardError, "cannot specify shard for query: #{tree.to_sql}"
|
|
149
151
|
elsif shard_keys.present?
|
|
@@ -155,8 +157,7 @@ module ActiveRecord::Turntable
|
|
|
155
157
|
else
|
|
156
158
|
return Fader::SelectShardsMergeResult.new(@proxy,
|
|
157
159
|
Hash[shard_keys.map { |k| [@proxy.cluster.shard_for(k), query] }],
|
|
158
|
-
method, query, *args, &block
|
|
159
|
-
)
|
|
160
|
+
method, query, *args, &block)
|
|
160
161
|
end
|
|
161
162
|
else # scan all shards
|
|
162
163
|
if SQLTree::Node::SelectDeclaration === tree.select.first &&
|
|
@@ -177,8 +178,7 @@ module ActiveRecord::Turntable
|
|
|
177
178
|
end
|
|
178
179
|
return Fader::SelectShardsMergeResult.new(@proxy,
|
|
179
180
|
build_shards_with_same_query(@proxy.shards.values, query),
|
|
180
|
-
method, query, *args, &block
|
|
181
|
-
)
|
|
181
|
+
method, query, *args, &block)
|
|
182
182
|
else
|
|
183
183
|
raise CannotSpecifyShardError, "cannot specify shard for query: #{tree.to_sql}"
|
|
184
184
|
end
|
|
@@ -231,3 +231,4 @@ module ActiveRecord::Turntable
|
|
|
231
231
|
end
|
|
232
232
|
end
|
|
233
233
|
end
|
|
234
|
+
# rubocop:enable Style/CaseEquality
|