ar-octopus 0.8.1 → 0.10.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +6 -14
- data/.gitignore +1 -0
- data/.rspec +1 -1
- data/.rubocop.yml +46 -0
- data/.rubocop_todo.yml +56 -0
- data/.travis.yml +7 -12
- data/Appraisals +11 -4
- data/Gemfile +1 -1
- data/README.mkdn +138 -63
- data/Rakefile +23 -16
- data/ar-octopus.gemspec +23 -20
- data/gemfiles/rails42.gemfile +7 -0
- data/gemfiles/{rails32.gemfile → rails5.gemfile} +2 -2
- data/gemfiles/{rails4.gemfile → rails51.gemfile} +2 -2
- data/gemfiles/rails52.gemfile +7 -0
- data/lib/ar-octopus.rb +1 -1
- data/lib/octopus/{rails3/abstract_adapter.rb → abstract_adapter.rb} +4 -15
- data/lib/octopus/association.rb +8 -99
- data/lib/octopus/association_shard_tracking.rb +74 -0
- data/lib/octopus/collection_association.rb +17 -0
- data/lib/octopus/collection_proxy.rb +16 -0
- data/lib/octopus/exception.rb +4 -0
- data/lib/octopus/finder_methods.rb +8 -0
- data/lib/octopus/load_balancing/round_robin.rb +20 -0
- data/lib/octopus/load_balancing.rb +4 -0
- data/lib/octopus/{rails3/log_subscriber.rb → log_subscriber.rb} +6 -2
- data/lib/octopus/migration.rb +187 -110
- data/lib/octopus/model.rb +151 -131
- data/lib/octopus/persistence.rb +45 -0
- data/lib/octopus/proxy.rb +297 -232
- data/lib/octopus/proxy_config.rb +251 -0
- data/lib/octopus/query_cache_for_shards.rb +24 -0
- data/lib/octopus/railtie.rb +1 -3
- data/lib/octopus/relation_proxy.rb +70 -0
- data/lib/octopus/result_patch.rb +19 -0
- data/lib/octopus/scope_proxy.rb +54 -36
- data/lib/octopus/shard_tracking/attribute.rb +22 -0
- data/lib/octopus/shard_tracking/dynamic.rb +11 -0
- data/lib/octopus/shard_tracking.rb +46 -0
- data/lib/octopus/singular_association.rb +9 -0
- data/lib/octopus/slave_group.rb +13 -0
- data/lib/octopus/version.rb +1 -1
- data/lib/octopus.rb +125 -33
- data/lib/tasks/octopus.rake +2 -2
- data/sample_app/Gemfile +3 -3
- data/sample_app/autotest/discover.rb +2 -2
- data/sample_app/config/application.rb +1 -1
- data/sample_app/config/boot.rb +1 -1
- data/sample_app/config/environments/test.rb +1 -1
- data/sample_app/config/initializers/session_store.rb +1 -1
- data/sample_app/config/initializers/wrap_parameters.rb +1 -1
- data/sample_app/config/routes.rb +1 -1
- data/sample_app/db/migrate/20100720210335_create_sample_users.rb +2 -2
- data/sample_app/db/schema.rb +10 -10
- data/sample_app/db/seeds.rb +3 -3
- data/sample_app/features/step_definitions/seeds_steps.rb +4 -4
- data/sample_app/features/step_definitions/web_steps.rb +3 -4
- data/sample_app/features/support/env.rb +3 -4
- data/sample_app/features/support/paths.rb +4 -4
- data/sample_app/lib/tasks/cucumber.rake +43 -44
- data/sample_app/spec/spec_helper.rb +3 -3
- data/spec/config/shards.yml +78 -0
- data/spec/migrations/10_create_users_using_replication.rb +4 -4
- data/spec/migrations/11_add_field_in_all_slaves.rb +4 -4
- data/spec/migrations/12_create_users_using_block.rb +8 -8
- data/spec/migrations/13_create_users_using_block_and_using.rb +5 -5
- data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +3 -3
- data/spec/migrations/15_create_user_on_shards_of_default_group_with_versions.rb +3 -3
- data/spec/migrations/1_create_users_on_master.rb +4 -4
- data/spec/migrations/2_create_users_on_canada.rb +4 -4
- data/spec/migrations/3_create_users_on_both_shards.rb +4 -4
- data/spec/migrations/4_create_users_on_shards_of_a_group.rb +4 -4
- data/spec/migrations/5_create_users_on_multiples_groups.rb +3 -3
- data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +4 -4
- data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +4 -4
- data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +4 -4
- data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +5 -5
- data/spec/octopus/association_shard_tracking_spec.rb +1036 -0
- data/spec/octopus/collection_proxy_spec.rb +16 -0
- data/spec/octopus/load_balancing/round_robin_spec.rb +15 -0
- data/spec/octopus/log_subscriber_spec.rb +5 -5
- data/spec/octopus/migration_spec.rb +83 -49
- data/spec/octopus/model_spec.rb +544 -292
- data/spec/octopus/octopus_spec.rb +64 -31
- data/spec/octopus/proxy_spec.rb +145 -141
- data/spec/octopus/query_cache_for_shards_spec.rb +40 -0
- data/spec/octopus/relation_proxy_spec.rb +132 -0
- data/spec/octopus/replicated_slave_grouped_spec.rb +91 -0
- data/spec/octopus/replication_spec.rb +140 -65
- data/spec/octopus/scope_proxy_spec.rb +90 -10
- data/spec/octopus/sharded_replicated_slave_grouped_spec.rb +55 -0
- data/spec/octopus/sharded_spec.rb +10 -10
- data/spec/spec_helper.rb +8 -6
- data/spec/support/active_record/connection_adapters/modify_config_adapter.rb +1 -3
- data/spec/support/database_connection.rb +2 -2
- data/spec/support/database_models.rb +18 -17
- data/spec/support/octopus_helper.rb +32 -25
- data/spec/support/query_count.rb +1 -3
- data/spec/support/shared_contexts.rb +3 -3
- data/spec/tasks/octopus.rake_spec.rb +10 -10
- metadata +112 -70
- data/.ruby-version +0 -1
- data/init.rb +0 -1
- data/lib/octopus/association_collection.rb +0 -49
- data/lib/octopus/has_and_belongs_to_many_association.rb +0 -17
- data/lib/octopus/rails3/persistence.rb +0 -39
- data/lib/octopus/rails3/singular_association.rb +0 -34
- data/rails/init.rb +0 -1
- data/spec/octopus/association_spec.rb +0 -712
data/lib/ar-octopus.rb
CHANGED
@@ -1 +1 @@
|
|
1
|
-
require
|
1
|
+
require 'octopus'
|
@@ -2,13 +2,7 @@
|
|
2
2
|
module Octopus
|
3
3
|
module AbstractAdapter
|
4
4
|
module OctopusShard
|
5
|
-
|
6
|
-
parent = ActiveSupport::BasicObject
|
7
|
-
if Octopus.rails4?
|
8
|
-
parent = ActiveSupport::ProxyObject
|
9
|
-
end
|
10
|
-
|
11
|
-
class InstrumenterDecorator < parent
|
5
|
+
class InstrumenterDecorator < ActiveSupport::ProxyObject
|
12
6
|
def initialize(adapter, instrumenter)
|
13
7
|
@adapter = adapter
|
14
8
|
@instrumenter = instrumenter
|
@@ -24,21 +18,16 @@ module Octopus
|
|
24
18
|
end
|
25
19
|
end
|
26
20
|
|
27
|
-
def self.included(base)
|
28
|
-
base.alias_method_chain :initialize, :octopus_shard
|
29
|
-
end
|
30
|
-
|
31
21
|
def octopus_shard
|
32
22
|
@config[:octopus_shard]
|
33
23
|
end
|
34
24
|
|
35
|
-
def
|
36
|
-
|
25
|
+
def initialize(*args)
|
26
|
+
super
|
37
27
|
@instrumenter = InstrumenterDecorator.new(self, @instrumenter)
|
38
28
|
end
|
39
|
-
|
40
29
|
end
|
41
30
|
end
|
42
31
|
end
|
43
32
|
|
44
|
-
ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:
|
33
|
+
ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:prepend, Octopus::AbstractAdapter::OctopusShard)
|
data/lib/octopus/association.rb
CHANGED
@@ -1,105 +1,14 @@
|
|
1
|
-
module Octopus
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
module QueryOnCurrentShard
|
7
|
-
|
8
|
-
METHODS= %w[
|
9
|
-
all
|
10
|
-
average
|
11
|
-
count
|
12
|
-
empty?
|
13
|
-
exists?
|
14
|
-
find
|
15
|
-
find_by_sql
|
16
|
-
first
|
17
|
-
last
|
18
|
-
maximum
|
19
|
-
minimum
|
20
|
-
pluck
|
21
|
-
scoping
|
22
|
-
size
|
23
|
-
sum
|
24
|
-
to_a
|
25
|
-
]
|
26
|
-
|
27
|
-
METHODS.each do |m|
|
28
|
-
define_method m.to_sym do |*args,&block|
|
29
|
-
if self.respond_to?(:proxy_association) and self.proxy_association
|
30
|
-
self.proxy_association.owner.run_on_shard { super(*args, &block) }
|
31
|
-
else
|
32
|
-
super(*args, &block)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
end
|
38
|
-
|
39
|
-
module InstanceMethods
|
40
|
-
def set_connection_on_association(record)
|
41
|
-
return unless ::Octopus.enabled?
|
42
|
-
return if !self.class.connection.respond_to?(:current_shard) || !self.respond_to?(:current_shard)
|
43
|
-
if !record.current_shard.nil? && !self.current_shard.nil? && record.current_shard != self.current_shard
|
44
|
-
raise "Association Error: Records are from different shards"
|
45
|
-
end
|
46
|
-
|
47
|
-
record.current_shard = self.class.connection.current_shard = self.current_shard if should_set_current_shard?
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
if Octopus.rails4?
|
52
|
-
def has_many(association_id, scope=nil, options={}, &extension)
|
53
|
-
if options == {} && scope.is_a?(Hash)
|
54
|
-
default_octopus_opts(scope)
|
55
|
-
else
|
56
|
-
default_octopus_opts(options)
|
57
|
-
end
|
58
|
-
super
|
59
|
-
end
|
60
|
-
else
|
61
|
-
def has_many(association_id, options={}, &extension)
|
62
|
-
default_octopus_opts(options)
|
63
|
-
super
|
1
|
+
module Octopus
|
2
|
+
module Association
|
3
|
+
def self.included(base)
|
4
|
+
base.send(:include, Octopus::ShardTracking::Dynamic)
|
5
|
+
base.sharded_methods :target_scope
|
64
6
|
end
|
65
|
-
end
|
66
7
|
|
67
|
-
|
68
|
-
|
69
|
-
def has_and_belongs_to_many(association_id, scope=nil, options={}, &extension)
|
70
|
-
if options == {} && scope.is_a?(Hash)
|
71
|
-
default_octopus_opts(scope)
|
72
|
-
else
|
73
|
-
default_octopus_opts(options)
|
74
|
-
end
|
75
|
-
super
|
76
|
-
end
|
77
|
-
else
|
78
|
-
def has_and_belongs_to_many(association_id, options={}, &extension)
|
79
|
-
default_octopus_opts(options)
|
80
|
-
super
|
8
|
+
def current_shard
|
9
|
+
owner.current_shard
|
81
10
|
end
|
82
11
|
end
|
83
|
-
|
84
|
-
def default_octopus_opts(options)
|
85
|
-
if options[:before_add].is_a?(Array)
|
86
|
-
options[:before_add] << :set_connection_on_association
|
87
|
-
elsif options[:before_add].is_a?(Symbol)
|
88
|
-
options[:before_add] = [:set_connection_on_association, options[:before_add]]
|
89
|
-
else
|
90
|
-
options[:before_add] = :set_connection_on_association
|
91
|
-
end
|
92
|
-
|
93
|
-
if options[:before_remove].is_a?(Array)
|
94
|
-
options[:before_remove] << :set_connection_on_association
|
95
|
-
elsif options[:before_remove].is_a?(Symbol)
|
96
|
-
options[:before_remove] = [:set_connection_on_association, options[:before_remove]]
|
97
|
-
else
|
98
|
-
options[:before_remove] = :set_connection_on_association
|
99
|
-
end
|
100
|
-
|
101
|
-
options[:extend] = [ Octopus::Association::QueryOnCurrentShard, options[:extend] ].flatten.compact
|
102
|
-
end
|
103
12
|
end
|
104
13
|
|
105
|
-
ActiveRecord::
|
14
|
+
ActiveRecord::Associations::Association.send(:include, Octopus::Association)
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Octopus
|
2
|
+
module AssociationShardTracking
|
3
|
+
class MismatchedShards < StandardError
|
4
|
+
attr_reader :record, :current_shard
|
5
|
+
|
6
|
+
def initialize(record, current_shard)
|
7
|
+
@record = record
|
8
|
+
@current_shard = current_shard
|
9
|
+
end
|
10
|
+
|
11
|
+
def message
|
12
|
+
[
|
13
|
+
"Association Error: Records are from different shards",
|
14
|
+
"Record: #{record.inspect}",
|
15
|
+
"Current Shard: #{current_shard.inspect}",
|
16
|
+
"Current Record Shard: #{record.current_shard.inspect}",
|
17
|
+
].join(" ")
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.extended(base)
|
22
|
+
base.send(:include, InstanceMethods)
|
23
|
+
end
|
24
|
+
|
25
|
+
module InstanceMethods
|
26
|
+
def connection_on_association=(record)
|
27
|
+
return unless ::Octopus.enabled?
|
28
|
+
return if !self.class.connection.respond_to?(:current_shard) || !self.respond_to?(:current_shard)
|
29
|
+
|
30
|
+
if !record.current_shard.nil? && !current_shard.nil? && record.current_shard.to_s != current_shard.to_s
|
31
|
+
raise MismatchedShards.new(record, current_shard)
|
32
|
+
end
|
33
|
+
|
34
|
+
record.current_shard = self.class.connection.current_shard = current_shard if should_set_current_shard?
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def has_many(name, scope = nil, **options, &extension)
|
39
|
+
if options == {} && scope.is_a?(Hash)
|
40
|
+
default_octopus_opts(scope)
|
41
|
+
else
|
42
|
+
default_octopus_opts(options)
|
43
|
+
end
|
44
|
+
super
|
45
|
+
end
|
46
|
+
|
47
|
+
def has_and_belongs_to_many(association_id, scope = nil, options = {}, &extension)
|
48
|
+
assign_octopus_opts(scope, options)
|
49
|
+
super
|
50
|
+
end
|
51
|
+
|
52
|
+
def default_octopus_opts(options)
|
53
|
+
options[:before_add] = [ :connection_on_association=, options[:before_add] ].compact.flatten
|
54
|
+
options[:before_remove] = [ :connection_on_association=, options[:before_remove] ].compact.flatten
|
55
|
+
end
|
56
|
+
|
57
|
+
def assign_octopus_opts(scope, options)
|
58
|
+
if options == {} && scope.is_a?(Hash)
|
59
|
+
default_octopus_opts(scope)
|
60
|
+
else
|
61
|
+
default_octopus_opts(options)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
if Octopus.atleast_rails51?
|
66
|
+
def has_and_belongs_to_many(association_id, scope = nil, **options, &extension)
|
67
|
+
assign_octopus_opts(scope, options)
|
68
|
+
super
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
ActiveRecord::Base.extend(Octopus::AssociationShardTracking)
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Octopus
|
2
|
+
module CollectionAssociation
|
3
|
+
def self.included(base)
|
4
|
+
if Octopus.rails51? || Octopus.rails52?
|
5
|
+
base.sharded_methods :reader, :writer, :ids_reader, :ids_writer, :create, :create!,
|
6
|
+
:build, :include?,
|
7
|
+
:load_target, :reload, :size, :select
|
8
|
+
else
|
9
|
+
base.sharded_methods :reader, :writer, :ids_reader, :ids_writer, :create, :create!,
|
10
|
+
:build, :any?, :count, :empty?, :first, :include?, :last, :length,
|
11
|
+
:load_target, :many?, :reload, :size, :select, :uniq
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
ActiveRecord::Associations::CollectionAssociation.send(:include, Octopus::CollectionAssociation)
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Octopus
|
2
|
+
module CollectionProxy
|
3
|
+
def self.included(base)
|
4
|
+
base.send(:include, Octopus::ShardTracking::Dynamic)
|
5
|
+
base.sharded_methods :any?, :build, :count, :create, :create!, :concat, :delete, :delete_all,
|
6
|
+
:destroy, :destroy_all, :empty?, :find, :first, :include?, :last, :length,
|
7
|
+
:many?, :pluck, :replace, :select, :size, :sum, :to_a, :uniq
|
8
|
+
end
|
9
|
+
|
10
|
+
def current_shard
|
11
|
+
@association.owner.current_shard
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
ActiveRecord::Associations::CollectionProxy.send(:include, Octopus::CollectionProxy)
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'octopus/load_balancing'
|
2
|
+
|
3
|
+
# The round-robin load balancing of slaves belonging to the same shard.
|
4
|
+
# It is a pool that contains slaves which queries are distributed to.
|
5
|
+
module Octopus
|
6
|
+
module LoadBalancing
|
7
|
+
class RoundRobin
|
8
|
+
def initialize(slaves_list)
|
9
|
+
raise Octopus::Exception.new("No slaves available") if slaves_list.empty?
|
10
|
+
@slaves_list = slaves_list
|
11
|
+
@slave_index = 0
|
12
|
+
end
|
13
|
+
|
14
|
+
# Returns the next available slave in the pool
|
15
|
+
def next(options)
|
16
|
+
@slaves_list[@slave_index = (@slave_index + 1) % @slaves_list.length]
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -3,8 +3,12 @@ module Octopus
|
|
3
3
|
module LogSubscriber
|
4
4
|
def self.included(base)
|
5
5
|
base.send(:attr_accessor, :octopus_shard)
|
6
|
-
|
7
|
-
base.
|
6
|
+
|
7
|
+
base.send :alias_method, :sql_without_octopus_shard, :sql
|
8
|
+
base.send :alias_method, :sql, :sql_with_octopus_shard
|
9
|
+
|
10
|
+
base.send :alias_method, :debug_without_octopus_shard, :debug
|
11
|
+
base.send :alias_method, :debug, :debug_with_octopus_shard
|
8
12
|
end
|
9
13
|
|
10
14
|
def sql_with_octopus_shard(event)
|