ar-octopus 0.8.5 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/.travis.yml +6 -9
- data/Appraisals +8 -8
- data/README.mkdn +47 -15
- data/Rakefile +1 -0
- data/ar-octopus.gemspec +9 -12
- data/gemfiles/rails42.gemfile +2 -2
- data/gemfiles/{rails32.gemfile → rails5.gemfile} +2 -2
- data/gemfiles/{rails4.gemfile → rails51.gemfile} +2 -2
- data/gemfiles/{rails41.gemfile → rails52.gemfile} +2 -2
- data/lib/octopus/abstract_adapter.rb +4 -10
- data/lib/octopus/association.rb +1 -0
- data/lib/octopus/association_shard_tracking.rb +41 -71
- data/lib/octopus/collection_association.rb +9 -3
- data/lib/octopus/exception.rb +4 -0
- data/lib/octopus/finder_methods.rb +8 -0
- data/lib/octopus/load_balancing/round_robin.rb +2 -1
- data/lib/octopus/log_subscriber.rb +6 -2
- data/lib/octopus/migration.rb +123 -55
- data/lib/octopus/model.rb +42 -27
- data/lib/octopus/persistence.rb +33 -27
- data/lib/octopus/proxy.rb +147 -272
- data/lib/octopus/proxy_config.rb +251 -0
- data/lib/octopus/query_cache_for_shards.rb +24 -0
- data/lib/octopus/railtie.rb +0 -2
- data/lib/octopus/relation_proxy.rb +36 -1
- data/lib/octopus/result_patch.rb +19 -0
- data/lib/octopus/scope_proxy.rb +12 -5
- data/lib/octopus/shard_tracking.rb +8 -3
- data/lib/octopus/slave_group.rb +3 -3
- data/lib/octopus/version.rb +1 -1
- data/lib/octopus.rb +71 -18
- data/spec/config/shards.yml +12 -0
- data/spec/migrations/10_create_users_using_replication.rb +1 -1
- data/spec/migrations/11_add_field_in_all_slaves.rb +1 -1
- data/spec/migrations/12_create_users_using_block.rb +1 -1
- data/spec/migrations/13_create_users_using_block_and_using.rb +1 -1
- data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +1 -1
- data/spec/migrations/15_create_user_on_shards_of_default_group_with_versions.rb +1 -1
- data/spec/migrations/1_create_users_on_master.rb +1 -1
- data/spec/migrations/2_create_users_on_canada.rb +1 -1
- data/spec/migrations/3_create_users_on_both_shards.rb +1 -1
- data/spec/migrations/4_create_users_on_shards_of_a_group.rb +1 -1
- data/spec/migrations/5_create_users_on_multiples_groups.rb +1 -1
- data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +1 -1
- data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +1 -1
- data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +1 -1
- data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +1 -1
- data/spec/octopus/association_shard_tracking_spec.rb +344 -16
- data/spec/octopus/load_balancing/round_robin_spec.rb +15 -0
- data/spec/octopus/log_subscriber_spec.rb +1 -1
- data/spec/octopus/migration_spec.rb +45 -11
- data/spec/octopus/model_spec.rb +204 -16
- data/spec/octopus/octopus_spec.rb +2 -2
- data/spec/octopus/proxy_spec.rb +44 -40
- data/spec/octopus/query_cache_for_shards_spec.rb +40 -0
- data/spec/octopus/relation_proxy_spec.rb +71 -23
- data/spec/octopus/replicated_slave_grouped_spec.rb +27 -0
- data/spec/octopus/replication_spec.rb +72 -2
- data/spec/octopus/scope_proxy_spec.rb +41 -7
- data/spec/spec_helper.rb +2 -0
- data/spec/support/database_connection.rb +1 -1
- data/spec/support/database_models.rb +1 -1
- data/spec/support/octopus_helper.rb +14 -6
- data/spec/tasks/octopus.rake_spec.rb +1 -1
- metadata +40 -30
- data/.ruby-version +0 -1
- data/init.rb +0 -1
- data/lib/octopus/has_and_belongs_to_many_association.rb +0 -9
- data/rails/init.rb +0 -1
data/lib/octopus/migration.rb
CHANGED
@@ -19,7 +19,9 @@ module Octopus
|
|
19
19
|
def self.included(base)
|
20
20
|
base.extend(ClassMethods)
|
21
21
|
|
22
|
-
base.
|
22
|
+
base.send :alias_method, :announce_without_octopus, :announce
|
23
|
+
base.send :alias_method, :announce, :announce_with_octopus
|
24
|
+
|
23
25
|
base.class_attribute :current_shard, :current_group, :current_group_specified, :instance_reader => false, :instance_writer => false
|
24
26
|
end
|
25
27
|
|
@@ -51,7 +53,7 @@ module Octopus
|
|
51
53
|
shards.merge(Array.wrap(shard))
|
52
54
|
end
|
53
55
|
|
54
|
-
shards.to_a.presence || [
|
56
|
+
shards.to_a.presence || [Octopus.master_shard]
|
55
57
|
end
|
56
58
|
end
|
57
59
|
end
|
@@ -60,81 +62,145 @@ end
|
|
60
62
|
module Octopus
|
61
63
|
module Migrator
|
62
64
|
def self.included(base)
|
63
|
-
|
65
|
+
unless Octopus.atleast_rails52?
|
66
|
+
base.extend(ClassMethods)
|
67
|
+
|
68
|
+
base.class_eval do
|
69
|
+
class << self
|
70
|
+
alias_method :migrate_without_octopus, :migrate
|
71
|
+
alias_method :migrate, :migrate_with_octopus
|
72
|
+
|
73
|
+
alias_method :up_without_octopus, :up
|
74
|
+
alias_method :up, :up_with_octopus
|
64
75
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
76
|
+
alias_method :down_without_octopus, :down
|
77
|
+
alias_method :down, :down_with_octopus
|
78
|
+
|
79
|
+
alias_method :run_without_octopus, :run
|
80
|
+
alias_method :run, :run_with_octopus
|
81
|
+
|
82
|
+
alias_method :rollback_without_octopus, :rollback
|
83
|
+
alias_method :rollback, :rollback_with_octopus
|
84
|
+
end
|
71
85
|
end
|
72
86
|
end
|
73
87
|
|
74
|
-
base.
|
75
|
-
base.
|
76
|
-
base.alias_method_chain :migrations, :octopus
|
77
|
-
end
|
88
|
+
base.send :alias_method, :run_without_octopus, :run
|
89
|
+
base.send :alias_method, :run, :run_with_octopus
|
78
90
|
|
79
|
-
|
80
|
-
|
81
|
-
rescue ActiveRecord::UnknownMigrationVersionError => e
|
82
|
-
raise unless migrations(true).detect { |m| m.version == e.version }
|
83
|
-
end
|
91
|
+
base.send :alias_method, :migrate_without_octopus, :migrate
|
92
|
+
base.send :alias_method, :migrate, :migrate_with_octopus
|
84
93
|
|
85
|
-
|
86
|
-
|
87
|
-
rescue ActiveRecord::UnknownMigrationVersionError => e
|
88
|
-
raise unless migrations(true).detect { |m| m.version == e.version }
|
94
|
+
base.send :alias_method, :migrations_without_octopus, :migrations
|
95
|
+
base.send :alias_method, :migrations, :migrations_with_octopus
|
89
96
|
end
|
97
|
+
if Octopus.atleast_rails52?
|
98
|
+
### Post RAILS 5.2 Migration methods
|
99
|
+
|
100
|
+
def run_with_octopus(&block)
|
101
|
+
return run_without_octopus(&block) unless connection.is_a?(Octopus::Proxy)
|
102
|
+
shards = migrations.map(&:shards).flatten.map(&:to_s)
|
103
|
+
connection.send_queries_to_multiple_shards(shards) do
|
104
|
+
run_without_octopus(&block)
|
105
|
+
end
|
106
|
+
rescue ActiveRecord::UnknownMigrationVersionError => e
|
107
|
+
raise unless migrations(true).detect { |m| m.version == e.version }
|
108
|
+
end
|
90
109
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
110
|
+
def migrate_with_octopus(&block)
|
111
|
+
return migrate_without_octopus(&block) unless connection.is_a?(Octopus::Proxy)
|
112
|
+
shards = migrations.map(&:shards).flatten.map(&:to_s)
|
113
|
+
connection.send_queries_to_multiple_shards(shards) do
|
114
|
+
migrate_without_octopus(&block)
|
115
|
+
end
|
116
|
+
rescue ActiveRecord::UnknownMigrationVersionError => e
|
117
|
+
raise unless migrations(true).detect { |m| m.version == e.version }
|
118
|
+
end
|
95
119
|
|
96
|
-
|
97
|
-
|
120
|
+
def migrations_with_octopus(shard_agnostic = true)
|
121
|
+
migrations = migrations_without_octopus
|
122
|
+
return migrations if !connection.is_a?(Octopus::Proxy) || shard_agnostic
|
98
123
|
|
99
|
-
|
100
|
-
|
101
|
-
|
124
|
+
migrations.select { |m| m.shards.include?(connection.current_shard.to_sym) }
|
125
|
+
end
|
126
|
+
|
127
|
+
private
|
128
|
+
|
129
|
+
def connection
|
130
|
+
ActiveRecord::Base.connection
|
131
|
+
end
|
132
|
+
|
133
|
+
else
|
134
|
+
### Pre RAILS 5.2 Migration methods
|
135
|
+
|
136
|
+
def run_with_octopus(&block)
|
137
|
+
run_without_octopus(&block)
|
138
|
+
rescue ActiveRecord::UnknownMigrationVersionError => e
|
139
|
+
raise unless migrations(true).detect { |m| m.version == e.version }
|
140
|
+
end
|
102
141
|
|
103
|
-
|
104
|
-
|
105
|
-
|
142
|
+
def migrate_with_octopus(&block)
|
143
|
+
migrate_without_octopus(&block)
|
144
|
+
rescue ActiveRecord::UnknownMigrationVersionError => e
|
145
|
+
raise unless migrations(true).detect { |m| m.version == e.version }
|
106
146
|
end
|
107
147
|
|
108
|
-
def
|
109
|
-
|
110
|
-
|
148
|
+
def migrations_with_octopus(shard_agnostic = false)
|
149
|
+
connection = ActiveRecord::Base.connection
|
150
|
+
migrations = migrations_without_octopus
|
151
|
+
return migrations if !connection.is_a?(Octopus::Proxy) || shard_agnostic
|
111
152
|
|
112
|
-
|
113
|
-
up_without_octopus(migrations_paths, target_version, &block)
|
114
|
-
end
|
153
|
+
migrations.select { |m| m.shards.include?(connection.current_shard.to_sym) }
|
115
154
|
end
|
116
155
|
|
117
|
-
|
118
|
-
|
119
|
-
|
156
|
+
module ClassMethods
|
157
|
+
def migrate_with_octopus(migrations_paths, target_version = nil, &block)
|
158
|
+
return migrate_without_octopus(migrations_paths, target_version, &block) unless connection.is_a?(Octopus::Proxy)
|
120
159
|
|
121
|
-
|
122
|
-
|
160
|
+
connection.send_queries_to_multiple_shards(connection.shard_names) do
|
161
|
+
migrate_without_octopus(migrations_paths, target_version, &block)
|
162
|
+
end
|
123
163
|
end
|
124
|
-
end
|
125
164
|
|
126
|
-
|
127
|
-
|
165
|
+
def up_with_octopus(migrations_paths, target_version = nil, &block)
|
166
|
+
return up_without_octopus(migrations_paths, target_version, &block) unless connection.is_a?(Octopus::Proxy)
|
167
|
+
return up_without_octopus(migrations_paths, target_version, &block) unless connection.current_shard.to_s == Octopus.master_shard.to_s
|
128
168
|
|
129
|
-
|
130
|
-
|
169
|
+
connection.send_queries_to_multiple_shards(connection.shard_names) do
|
170
|
+
up_without_octopus(migrations_paths, target_version, &block)
|
171
|
+
end
|
131
172
|
end
|
132
|
-
end
|
133
173
|
|
134
|
-
|
174
|
+
def down_with_octopus(migrations_paths, target_version = nil, &block)
|
175
|
+
return down_without_octopus(migrations_paths, target_version, &block) unless connection.is_a?(Octopus::Proxy)
|
176
|
+
return down_without_octopus(migrations_paths, target_version, &block) unless connection.current_shard.to_s == Octopus.master_shard.to_s
|
135
177
|
|
136
|
-
|
137
|
-
|
178
|
+
connection.send_queries_to_multiple_shards(connection.shard_names) do
|
179
|
+
down_without_octopus(migrations_paths, target_version, &block)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def run_with_octopus(direction, migrations_paths, target_version)
|
184
|
+
return run_without_octopus(direction, migrations_paths, target_version) unless connection.is_a?(Octopus::Proxy)
|
185
|
+
|
186
|
+
connection.send_queries_to_multiple_shards(connection.shard_names) do
|
187
|
+
run_without_octopus(direction, migrations_paths, target_version)
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
def rollback_with_octopus(migrations_paths, steps = 1)
|
192
|
+
return rollback_without_octopus(migrations_paths, steps) unless connection.is_a?(Octopus::Proxy)
|
193
|
+
|
194
|
+
connection.send_queries_to_multiple_shards(connection.shard_names) do
|
195
|
+
rollback_without_octopus(migrations_paths, steps)
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
private
|
200
|
+
|
201
|
+
def connection
|
202
|
+
ActiveRecord::Base.connection
|
203
|
+
end
|
138
204
|
end
|
139
205
|
end
|
140
206
|
end
|
@@ -151,7 +217,9 @@ end
|
|
151
217
|
module Octopus
|
152
218
|
module UnknownMigrationVersionError
|
153
219
|
def self.included(base)
|
154
|
-
base.
|
220
|
+
base.send :alias_method, :initialize_without_octopus, :initialize
|
221
|
+
base.send :alias_method, :initialize, :initialize_with_octopus
|
222
|
+
|
155
223
|
base.send(:attr_accessor, :version)
|
156
224
|
end
|
157
225
|
|
data/lib/octopus/model.rb
CHANGED
@@ -9,20 +9,16 @@ module Octopus
|
|
9
9
|
end
|
10
10
|
|
11
11
|
module SharedMethods
|
12
|
-
def
|
13
|
-
|
12
|
+
def using(shard)
|
13
|
+
if block_given?
|
14
|
+
raise Octopus::Exception, <<-EOF
|
15
|
+
#{name}.using is not allowed to receive a block, it works just like a regular scope.
|
14
16
|
|
15
|
-
|
16
|
-
|
17
|
+
If you are trying to scope everything to a specific shard, use Octopus.using instead.
|
18
|
+
EOF
|
17
19
|
end
|
18
20
|
|
19
|
-
reset_column_information
|
20
|
-
instance_variable_set(:@quoted_table_name, nil)
|
21
|
-
end
|
22
|
-
|
23
|
-
def using(shard)
|
24
21
|
if Octopus.enabled?
|
25
|
-
clean_table_name
|
26
22
|
Octopus::ScopeProxy.new(shard, self)
|
27
23
|
else
|
28
24
|
self
|
@@ -37,19 +33,29 @@ module Octopus
|
|
37
33
|
base.send(:alias_method, :equality_without_octopus, :==)
|
38
34
|
base.send(:alias_method, :==, :equality_with_octopus)
|
39
35
|
base.send(:alias_method, :eql?, :==)
|
40
|
-
base.send(:
|
36
|
+
base.send(:alias_method, :perform_validations_without_octopus, :perform_validations)
|
37
|
+
base.send(:alias_method, :perform_validations, :perform_validations_with_octopus)
|
41
38
|
end
|
42
39
|
|
43
40
|
def set_current_shard
|
44
41
|
return unless Octopus.enabled?
|
42
|
+
shard = self.class.connection_proxy.current_shard
|
43
|
+
self.current_shard = shard if self.class.allowed_shard?(shard)
|
44
|
+
end
|
45
45
|
|
46
|
-
|
47
|
-
|
48
|
-
else
|
49
|
-
shard = self.class.connection_proxy.last_current_shard || self.class.connection_proxy.current_shard
|
50
|
-
end
|
46
|
+
def init_with(coder)
|
47
|
+
obj = super
|
51
48
|
|
52
|
-
|
49
|
+
return obj unless Octopus.enabled?
|
50
|
+
return obj if obj.class.connection_proxy.current_model_replicated?
|
51
|
+
|
52
|
+
current_shard_value = coder['attributes']['current_shard'].value if coder['attributes']['current_shard'].present? && coder['attributes']['current_shard'].value.present?
|
53
|
+
|
54
|
+
coder['attributes'].send(:attributes).send(:values).delete('current_shard')
|
55
|
+
coder['attributes'].send(:attributes).send(:delegate_hash).delete('current_shard')
|
56
|
+
|
57
|
+
obj.current_shard = current_shard_value if current_shard_value.present?
|
58
|
+
obj
|
53
59
|
end
|
54
60
|
|
55
61
|
def should_set_current_shard?
|
@@ -57,7 +63,7 @@ module Octopus
|
|
57
63
|
end
|
58
64
|
|
59
65
|
def equality_with_octopus(comparison_object)
|
60
|
-
equality_without_octopus(comparison_object) && comparison_object.current_shard == current_shard
|
66
|
+
equality_without_octopus(comparison_object) && comparison_object.current_shard.to_s == current_shard.to_s
|
61
67
|
end
|
62
68
|
|
63
69
|
def perform_validations_with_octopus(*args)
|
@@ -95,20 +101,29 @@ module Octopus
|
|
95
101
|
end
|
96
102
|
|
97
103
|
def hijack_methods
|
98
|
-
around_save :run_on_shard, :unless => -> { self.class.custom_octopus_connection }
|
99
104
|
after_initialize :set_current_shard
|
100
105
|
|
106
|
+
around_save :run_on_shard, :unless => lambda { self.class.custom_octopus_connection }
|
107
|
+
|
108
|
+
class_attribute :custom_octopus_connection
|
109
|
+
|
101
110
|
class << self
|
102
|
-
attr_accessor :custom_octopus_connection
|
103
111
|
attr_accessor :custom_octopus_table_name
|
104
112
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
113
|
+
alias_method :connection_without_octopus, :connection
|
114
|
+
alias_method :connection, :connection_with_octopus
|
115
|
+
|
116
|
+
alias_method :connection_pool_without_octopus, :connection_pool
|
117
|
+
alias_method :connection_pool, :connection_pool_with_octopus
|
118
|
+
|
119
|
+
alias_method :clear_all_connections_without_octopus!, :clear_all_connections!
|
120
|
+
alias_method :clear_all_connections!, :clear_all_connections_with_octopus!
|
121
|
+
|
122
|
+
alias_method :clear_active_connections_without_octopus!, :clear_active_connections!
|
123
|
+
alias_method :clear_active_connections!, :clear_active_connections_with_octopus!
|
110
124
|
|
111
|
-
|
125
|
+
alias_method :connected_without_octopus?, :connected?
|
126
|
+
alias_method :connected?, :connected_with_octopus?
|
112
127
|
|
113
128
|
def table_name=(value = nil)
|
114
129
|
self.custom_octopus_table_name = true
|
@@ -133,7 +148,7 @@ module Octopus
|
|
133
148
|
|
134
149
|
def allowed_shard?(shard)
|
135
150
|
if custom_octopus_connection
|
136
|
-
allowed_shards && shard && allowed_shards.include?(shard)
|
151
|
+
allowed_shards && shard && (allowed_shards.include?(shard.to_s) || allowed_shards.include?(shard.to_sym))
|
137
152
|
else
|
138
153
|
true
|
139
154
|
end
|
data/lib/octopus/persistence.rb
CHANGED
@@ -1,39 +1,45 @@
|
|
1
1
|
module Octopus
|
2
|
-
module
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
end
|
2
|
+
module Persistence
|
3
|
+
def update_attribute(*args)
|
4
|
+
run_on_shard { super }
|
5
|
+
end
|
7
6
|
|
8
|
-
|
9
|
-
|
10
|
-
|
7
|
+
def update_attributes(*args)
|
8
|
+
run_on_shard { super }
|
9
|
+
end
|
11
10
|
|
12
|
-
|
13
|
-
|
14
|
-
|
11
|
+
def update_attributes!(*args)
|
12
|
+
run_on_shard { super }
|
13
|
+
end
|
15
14
|
|
16
|
-
|
17
|
-
|
18
|
-
|
15
|
+
def reload(*args)
|
16
|
+
run_on_shard { super }
|
17
|
+
end
|
19
18
|
|
20
|
-
|
21
|
-
|
22
|
-
|
19
|
+
def delete
|
20
|
+
run_on_shard { super }
|
21
|
+
end
|
23
22
|
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
def destroy
|
24
|
+
run_on_shard { super }
|
25
|
+
end
|
27
26
|
|
28
|
-
|
29
|
-
|
30
|
-
|
27
|
+
def touch(*args)
|
28
|
+
run_on_shard { super }
|
29
|
+
end
|
30
|
+
|
31
|
+
def update_column(*args)
|
32
|
+
run_on_shard { super }
|
33
|
+
end
|
34
|
+
|
35
|
+
def increment!(*args)
|
36
|
+
run_on_shard { super }
|
37
|
+
end
|
31
38
|
|
32
|
-
|
33
|
-
|
34
|
-
end
|
39
|
+
def decrement!(*args)
|
40
|
+
run_on_shard { super }
|
35
41
|
end
|
36
42
|
end
|
37
43
|
end
|
38
44
|
|
39
|
-
ActiveRecord::Base.send(:include, Octopus::
|
45
|
+
ActiveRecord::Base.send(:include, Octopus::Persistence)
|