switchman 2.0.5 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +10 -2
- data/app/models/switchman/shard.rb +235 -270
- data/app/models/switchman/unsharded_record.rb +7 -0
- data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
- data/db/migrate/20130328224244_create_default_shard.rb +5 -5
- data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
- data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
- data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
- data/lib/switchman/action_controller/caching.rb +2 -2
- data/lib/switchman/active_record/abstract_adapter.rb +1 -0
- data/lib/switchman/active_record/association.rb +78 -89
- data/lib/switchman/active_record/attribute_methods.rb +106 -52
- data/lib/switchman/active_record/base.rb +58 -59
- data/lib/switchman/active_record/calculations.rb +73 -66
- data/lib/switchman/active_record/connection_pool.rb +14 -41
- data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
- data/lib/switchman/active_record/database_configurations.rb +34 -0
- data/lib/switchman/active_record/finder_methods.rb +11 -16
- data/lib/switchman/active_record/log_subscriber.rb +4 -8
- data/lib/switchman/active_record/migration.rb +18 -15
- data/lib/switchman/active_record/model_schema.rb +1 -1
- data/lib/switchman/active_record/persistence.rb +4 -6
- data/lib/switchman/active_record/postgresql_adapter.rb +45 -144
- data/lib/switchman/active_record/predicate_builder.rb +1 -1
- data/lib/switchman/active_record/query_cache.rb +18 -19
- data/lib/switchman/active_record/query_methods.rb +144 -200
- data/lib/switchman/active_record/reflection.rb +6 -10
- data/lib/switchman/active_record/relation.rb +27 -21
- data/lib/switchman/active_record/spawn_methods.rb +27 -29
- data/lib/switchman/active_record/statement_cache.rb +18 -35
- data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
- data/lib/switchman/active_support/cache.rb +3 -5
- data/lib/switchman/arel.rb +13 -8
- data/lib/switchman/database_server.rb +122 -144
- data/lib/switchman/default_shard.rb +52 -16
- data/lib/switchman/engine.rb +61 -57
- data/lib/switchman/environment.rb +4 -8
- data/lib/switchman/errors.rb +1 -0
- data/lib/switchman/guard_rail/relation.rb +5 -7
- data/lib/switchman/guard_rail.rb +6 -19
- data/lib/switchman/r_spec_helper.rb +29 -37
- data/lib/switchman/rails.rb +14 -12
- data/lib/switchman/sharded_instrumenter.rb +1 -1
- data/lib/switchman/standard_error.rb +15 -3
- data/lib/switchman/test_helper.rb +7 -11
- data/lib/switchman/version.rb +1 -1
- data/lib/switchman.rb +3 -3
- data/lib/tasks/switchman.rake +54 -69
- metadata +90 -49
- data/lib/switchman/active_record/batches.rb +0 -11
- data/lib/switchman/active_record/connection_handler.rb +0 -172
- data/lib/switchman/active_record/where_clause_factory.rb +0 -36
- data/lib/switchman/connection_pool_proxy.rb +0 -169
- data/lib/switchman/schema_cache.rb +0 -20
|
@@ -3,27 +3,27 @@
|
|
|
3
3
|
module Switchman
|
|
4
4
|
module ActiveRecord
|
|
5
5
|
module SpawnMethods
|
|
6
|
-
def shard_values_for_merge(
|
|
7
|
-
if shard_value !=
|
|
8
|
-
if
|
|
6
|
+
def shard_values_for_merge(rhs)
|
|
7
|
+
if shard_value != rhs.shard_value
|
|
8
|
+
if rhs.shard_source_value == :implicit
|
|
9
9
|
final_shard_value = shard_value
|
|
10
10
|
final_primary_shard = primary_shard
|
|
11
11
|
final_shard_source_value = shard_source_value
|
|
12
12
|
elsif shard_source_value == :implicit
|
|
13
|
-
final_shard_value =
|
|
14
|
-
final_primary_shard =
|
|
15
|
-
final_shard_source_value =
|
|
13
|
+
final_shard_value = rhs.shard_value
|
|
14
|
+
final_primary_shard = rhs.primary_shard
|
|
15
|
+
final_shard_source_value = rhs.shard_source_value
|
|
16
16
|
else
|
|
17
|
-
final_shard_source_value = [
|
|
18
|
-
shard_source_value == source_value ||
|
|
17
|
+
final_shard_source_value = %i[explicit association].detect do |source_value|
|
|
18
|
+
shard_source_value == source_value || rhs.shard_source_value == source_value
|
|
19
19
|
end
|
|
20
|
-
raise
|
|
20
|
+
raise 'unknown shard_source_value' unless final_shard_source_value
|
|
21
21
|
|
|
22
22
|
# have to merge shard_value
|
|
23
23
|
lhs_shard_value = all_shards
|
|
24
|
-
rhs_shard_value =
|
|
25
|
-
if
|
|
26
|
-
|
|
24
|
+
rhs_shard_value = rhs.all_shards
|
|
25
|
+
if ::ActiveRecord::Relation === lhs_shard_value &&
|
|
26
|
+
::ActiveRecord::Relation === rhs_shard_value
|
|
27
27
|
final_shard_value = lhs_shard_value.merge(rhs_shard_value)
|
|
28
28
|
final_primary_shard = Shard.default
|
|
29
29
|
else
|
|
@@ -32,27 +32,25 @@ module Switchman
|
|
|
32
32
|
final_shard_value = final_shard_value.first if final_shard_value.length == 1
|
|
33
33
|
end
|
|
34
34
|
end
|
|
35
|
-
elsif shard_source_value !=
|
|
36
|
-
final_shard_source_value = [
|
|
37
|
-
shard_source_value == source_value ||
|
|
35
|
+
elsif shard_source_value != rhs.shard_source_value
|
|
36
|
+
final_shard_source_value = %i[explicit association implicit].detect do |source_value|
|
|
37
|
+
shard_source_value == source_value || rhs.shard_source_value == source_value
|
|
38
38
|
end
|
|
39
|
-
raise
|
|
40
|
-
else
|
|
41
|
-
# nothing fancy
|
|
39
|
+
raise 'unknown shard_source_value' unless final_shard_source_value
|
|
42
40
|
end
|
|
43
41
|
|
|
44
42
|
[final_shard_value, final_primary_shard, final_shard_source_value]
|
|
45
43
|
end
|
|
46
44
|
|
|
47
|
-
def merge!(
|
|
48
|
-
return super unless ::ActiveRecord::Relation ===
|
|
45
|
+
def merge!(rhs)
|
|
46
|
+
return super unless ::ActiveRecord::Relation === rhs
|
|
49
47
|
|
|
50
48
|
# have to figure out shard stuff *before* conditions are merged
|
|
51
|
-
final_shard_value, final_primary_shard, final_shard_source_value = shard_values_for_merge(
|
|
49
|
+
final_shard_value, final_primary_shard, final_shard_source_value = shard_values_for_merge(rhs)
|
|
52
50
|
|
|
53
51
|
return super unless final_shard_source_value
|
|
54
52
|
|
|
55
|
-
|
|
53
|
+
unless final_shard_value
|
|
56
54
|
super
|
|
57
55
|
self.shard_source_value = final_shard_source_value
|
|
58
56
|
return self
|
|
@@ -61,16 +59,16 @@ module Switchman
|
|
|
61
59
|
return none! if final_shard_value == []
|
|
62
60
|
|
|
63
61
|
# change the primary shard if necessary before merging
|
|
64
|
-
if primary_shard != final_primary_shard &&
|
|
62
|
+
if primary_shard != final_primary_shard && rhs.primary_shard != final_primary_shard
|
|
65
63
|
shard!(final_primary_shard)
|
|
66
|
-
|
|
67
|
-
super(
|
|
64
|
+
rhs = rhs.shard(final_primary_shard)
|
|
65
|
+
super(rhs)
|
|
68
66
|
elsif primary_shard != final_primary_shard
|
|
69
67
|
shard!(final_primary_shard)
|
|
70
|
-
super(
|
|
71
|
-
elsif
|
|
72
|
-
|
|
73
|
-
super(
|
|
68
|
+
super(rhs)
|
|
69
|
+
elsif rhs.primary_shard != final_primary_shard
|
|
70
|
+
rhs = rhs.shard(final_primary_shard)
|
|
71
|
+
super(rhs)
|
|
74
72
|
else
|
|
75
73
|
super
|
|
76
74
|
end
|
|
@@ -7,18 +7,13 @@ module Switchman
|
|
|
7
7
|
def create(connection, &block)
|
|
8
8
|
relation = block.call ::ActiveRecord::StatementCache::Params.new
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
new(relation.arel, bind_map, relation.klass)
|
|
14
|
-
else
|
|
15
|
-
bind_map = ::ActiveRecord::StatementCache::BindMap.new(relation.bound_attributes)
|
|
16
|
-
new relation.arel, bind_map
|
|
17
|
-
end
|
|
10
|
+
_query_builder, binds = connection.cacheable_query(self, relation.arel)
|
|
11
|
+
bind_map = ::ActiveRecord::StatementCache::BindMap.new(binds)
|
|
12
|
+
new(relation.arel, bind_map, relation.klass)
|
|
18
13
|
end
|
|
19
14
|
end
|
|
20
15
|
|
|
21
|
-
def initialize(arel, bind_map, klass=nil)
|
|
16
|
+
def initialize(arel, bind_map, klass = nil)
|
|
22
17
|
@arel = arel
|
|
23
18
|
@bind_map = bind_map
|
|
24
19
|
@klass = klass
|
|
@@ -31,49 +26,39 @@ module Switchman
|
|
|
31
26
|
# (e.g. infer from the primary key or use the current shard)
|
|
32
27
|
|
|
33
28
|
def execute(*args)
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
klass = @klass
|
|
37
|
-
else
|
|
38
|
-
params, klass, connection = args
|
|
39
|
-
end
|
|
29
|
+
params, connection = args
|
|
30
|
+
klass = @klass
|
|
40
31
|
target_shard = nil
|
|
41
|
-
if primary_index = bind_map.primary_value_index
|
|
32
|
+
if (primary_index = bind_map.primary_value_index)
|
|
42
33
|
primary_value = params[primary_index]
|
|
43
34
|
target_shard = Shard.local_id_for(primary_value)[1]
|
|
44
35
|
end
|
|
45
|
-
current_shard = Shard.current(klass.
|
|
36
|
+
current_shard = Shard.current(klass.connection_classes)
|
|
46
37
|
target_shard ||= current_shard
|
|
47
38
|
|
|
48
39
|
bind_values = bind_map.bind(params, current_shard, target_shard)
|
|
49
40
|
|
|
50
|
-
target_shard.activate(klass.
|
|
41
|
+
target_shard.activate(klass.connection_classes) do
|
|
51
42
|
sql = qualified_query_builder(target_shard, klass).sql_for(bind_values, connection)
|
|
52
43
|
klass.find_by_sql(sql, bind_values)
|
|
53
44
|
end
|
|
54
45
|
end
|
|
55
46
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel)
|
|
59
|
-
end
|
|
60
|
-
else
|
|
61
|
-
def qualified_query_builder(shard, klass)
|
|
62
|
-
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
|
|
63
|
-
end
|
|
47
|
+
def qualified_query_builder(shard, klass)
|
|
48
|
+
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
|
|
64
49
|
end
|
|
65
50
|
|
|
66
51
|
module BindMap
|
|
67
52
|
# performs id transposition here instead of query_methods.rb
|
|
68
53
|
def bind(values, current_shard, target_shard)
|
|
69
54
|
bas = @bound_attributes.dup
|
|
70
|
-
@indexes.each_with_index do |offset,i|
|
|
55
|
+
@indexes.each_with_index do |offset, i|
|
|
71
56
|
ba = bas[offset]
|
|
72
|
-
if ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.sharded
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
57
|
+
new_value = if ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.sharded
|
|
58
|
+
Shard.relative_id_for(values[i], current_shard, target_shard || current_shard)
|
|
59
|
+
else
|
|
60
|
+
values[i]
|
|
61
|
+
end
|
|
77
62
|
bas[offset] = ba.with_cast_value(new_value)
|
|
78
63
|
end
|
|
79
64
|
bas
|
|
@@ -83,9 +68,7 @@ module Switchman
|
|
|
83
68
|
primary_ba_index = @bound_attributes.index do |ba|
|
|
84
69
|
ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.primary
|
|
85
70
|
end
|
|
86
|
-
if primary_ba_index
|
|
87
|
-
@indexes.index(primary_ba_index)
|
|
88
|
-
end
|
|
71
|
+
@indexes.index(primary_ba_index) if primary_ba_index
|
|
89
72
|
end
|
|
90
73
|
end
|
|
91
74
|
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Switchman
|
|
4
|
+
module ActiveRecord
|
|
5
|
+
module Tasks
|
|
6
|
+
module DatabaseTasks
|
|
7
|
+
def drop(*)
|
|
8
|
+
super
|
|
9
|
+
# no really, it's gone
|
|
10
|
+
Switchman.cache.delete('default_shard')
|
|
11
|
+
Shard.default(reload: true)
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -8,10 +8,8 @@ module Switchman
|
|
|
8
8
|
store = super
|
|
9
9
|
# can't use defined?, because it's a _ruby_ autoloaded constant,
|
|
10
10
|
# so just checking that will cause it to get required
|
|
11
|
-
if store.
|
|
12
|
-
|
|
13
|
-
end
|
|
14
|
-
store.options[:namespace] ||= lambda { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
|
|
11
|
+
::ActiveSupport::Cache::RedisCacheStore.prepend(RedisCacheStore) if store.instance_of?(ActiveSupport::Cache::RedisCacheStore) && !::ActiveSupport::Cache::RedisCacheStore.ancestors.include?(RedisCacheStore)
|
|
12
|
+
store.options[:namespace] ||= -> { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
|
|
15
13
|
store
|
|
16
14
|
end
|
|
17
15
|
end
|
|
@@ -22,7 +20,7 @@ module Switchman
|
|
|
22
20
|
# unfortunately, it uses the keys command, which is extraordinarily inefficient in a large redis instance
|
|
23
21
|
# fortunately, we can assume we control the entire instance, because we set up the namespacing, so just
|
|
24
22
|
# always unset it temporarily for clear calls
|
|
25
|
-
namespace = nil
|
|
23
|
+
namespace = nil # rubocop:disable Lint/ShadowedArgument
|
|
26
24
|
super
|
|
27
25
|
end
|
|
28
26
|
end
|
data/lib/switchman/arel.rb
CHANGED
|
@@ -3,29 +3,34 @@
|
|
|
3
3
|
module Switchman
|
|
4
4
|
module Arel
|
|
5
5
|
module Table
|
|
6
|
-
def
|
|
7
|
-
|
|
6
|
+
def klass
|
|
7
|
+
@klass || ::ActiveRecord::Base
|
|
8
8
|
end
|
|
9
9
|
end
|
|
10
|
+
|
|
10
11
|
module Visitors
|
|
11
12
|
module ToSql
|
|
12
|
-
|
|
13
|
+
# rubocop:disable Naming/MethodName
|
|
14
|
+
|
|
15
|
+
def visit_Arel_Nodes_TableAlias(*args)
|
|
13
16
|
o, collector = args
|
|
14
17
|
collector = visit o.relation, collector
|
|
15
|
-
collector <<
|
|
18
|
+
collector << ' '
|
|
16
19
|
collector << quote_local_table_name(o.name)
|
|
17
20
|
end
|
|
18
21
|
|
|
19
|
-
def visit_Arel_Attributes_Attribute
|
|
22
|
+
def visit_Arel_Attributes_Attribute(*args)
|
|
20
23
|
o = args.first
|
|
21
24
|
join_name = o.relation.table_alias || o.relation.name
|
|
22
25
|
result = "#{quote_local_table_name join_name}.#{quote_column_name o.name}"
|
|
23
|
-
|
|
24
|
-
result
|
|
26
|
+
args.last << result
|
|
25
27
|
end
|
|
26
28
|
|
|
27
|
-
|
|
29
|
+
# rubocop:enable Naming/MethodName
|
|
30
|
+
|
|
31
|
+
def quote_local_table_name(name)
|
|
28
32
|
return name if ::Arel::Nodes::SqlLiteral === name
|
|
33
|
+
|
|
29
34
|
@connection.quote_local_table_name(name)
|
|
30
35
|
end
|
|
31
36
|
end
|