ar-octopus 0.8.1 → 0.10.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (109) hide show
  1. checksums.yaml +6 -14
  2. data/.gitignore +1 -0
  3. data/.rspec +1 -1
  4. data/.rubocop.yml +46 -0
  5. data/.rubocop_todo.yml +56 -0
  6. data/.travis.yml +7 -12
  7. data/Appraisals +11 -4
  8. data/Gemfile +1 -1
  9. data/README.mkdn +138 -63
  10. data/Rakefile +23 -16
  11. data/ar-octopus.gemspec +23 -20
  12. data/gemfiles/rails42.gemfile +7 -0
  13. data/gemfiles/{rails32.gemfile → rails5.gemfile} +2 -2
  14. data/gemfiles/{rails4.gemfile → rails51.gemfile} +2 -2
  15. data/gemfiles/rails52.gemfile +7 -0
  16. data/lib/ar-octopus.rb +1 -1
  17. data/lib/octopus/{rails3/abstract_adapter.rb → abstract_adapter.rb} +4 -15
  18. data/lib/octopus/association.rb +8 -99
  19. data/lib/octopus/association_shard_tracking.rb +74 -0
  20. data/lib/octopus/collection_association.rb +17 -0
  21. data/lib/octopus/collection_proxy.rb +16 -0
  22. data/lib/octopus/exception.rb +4 -0
  23. data/lib/octopus/finder_methods.rb +8 -0
  24. data/lib/octopus/load_balancing/round_robin.rb +20 -0
  25. data/lib/octopus/load_balancing.rb +4 -0
  26. data/lib/octopus/{rails3/log_subscriber.rb → log_subscriber.rb} +6 -2
  27. data/lib/octopus/migration.rb +187 -110
  28. data/lib/octopus/model.rb +151 -131
  29. data/lib/octopus/persistence.rb +45 -0
  30. data/lib/octopus/proxy.rb +297 -232
  31. data/lib/octopus/proxy_config.rb +251 -0
  32. data/lib/octopus/query_cache_for_shards.rb +24 -0
  33. data/lib/octopus/railtie.rb +1 -3
  34. data/lib/octopus/relation_proxy.rb +70 -0
  35. data/lib/octopus/result_patch.rb +19 -0
  36. data/lib/octopus/scope_proxy.rb +54 -36
  37. data/lib/octopus/shard_tracking/attribute.rb +22 -0
  38. data/lib/octopus/shard_tracking/dynamic.rb +11 -0
  39. data/lib/octopus/shard_tracking.rb +46 -0
  40. data/lib/octopus/singular_association.rb +9 -0
  41. data/lib/octopus/slave_group.rb +13 -0
  42. data/lib/octopus/version.rb +1 -1
  43. data/lib/octopus.rb +125 -33
  44. data/lib/tasks/octopus.rake +2 -2
  45. data/sample_app/Gemfile +3 -3
  46. data/sample_app/autotest/discover.rb +2 -2
  47. data/sample_app/config/application.rb +1 -1
  48. data/sample_app/config/boot.rb +1 -1
  49. data/sample_app/config/environments/test.rb +1 -1
  50. data/sample_app/config/initializers/session_store.rb +1 -1
  51. data/sample_app/config/initializers/wrap_parameters.rb +1 -1
  52. data/sample_app/config/routes.rb +1 -1
  53. data/sample_app/db/migrate/20100720210335_create_sample_users.rb +2 -2
  54. data/sample_app/db/schema.rb +10 -10
  55. data/sample_app/db/seeds.rb +3 -3
  56. data/sample_app/features/step_definitions/seeds_steps.rb +4 -4
  57. data/sample_app/features/step_definitions/web_steps.rb +3 -4
  58. data/sample_app/features/support/env.rb +3 -4
  59. data/sample_app/features/support/paths.rb +4 -4
  60. data/sample_app/lib/tasks/cucumber.rake +43 -44
  61. data/sample_app/spec/spec_helper.rb +3 -3
  62. data/spec/config/shards.yml +78 -0
  63. data/spec/migrations/10_create_users_using_replication.rb +4 -4
  64. data/spec/migrations/11_add_field_in_all_slaves.rb +4 -4
  65. data/spec/migrations/12_create_users_using_block.rb +8 -8
  66. data/spec/migrations/13_create_users_using_block_and_using.rb +5 -5
  67. data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +3 -3
  68. data/spec/migrations/15_create_user_on_shards_of_default_group_with_versions.rb +3 -3
  69. data/spec/migrations/1_create_users_on_master.rb +4 -4
  70. data/spec/migrations/2_create_users_on_canada.rb +4 -4
  71. data/spec/migrations/3_create_users_on_both_shards.rb +4 -4
  72. data/spec/migrations/4_create_users_on_shards_of_a_group.rb +4 -4
  73. data/spec/migrations/5_create_users_on_multiples_groups.rb +3 -3
  74. data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +4 -4
  75. data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +4 -4
  76. data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +4 -4
  77. data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +5 -5
  78. data/spec/octopus/association_shard_tracking_spec.rb +1036 -0
  79. data/spec/octopus/collection_proxy_spec.rb +16 -0
  80. data/spec/octopus/load_balancing/round_robin_spec.rb +15 -0
  81. data/spec/octopus/log_subscriber_spec.rb +5 -5
  82. data/spec/octopus/migration_spec.rb +83 -49
  83. data/spec/octopus/model_spec.rb +544 -292
  84. data/spec/octopus/octopus_spec.rb +64 -31
  85. data/spec/octopus/proxy_spec.rb +145 -141
  86. data/spec/octopus/query_cache_for_shards_spec.rb +40 -0
  87. data/spec/octopus/relation_proxy_spec.rb +132 -0
  88. data/spec/octopus/replicated_slave_grouped_spec.rb +91 -0
  89. data/spec/octopus/replication_spec.rb +140 -65
  90. data/spec/octopus/scope_proxy_spec.rb +90 -10
  91. data/spec/octopus/sharded_replicated_slave_grouped_spec.rb +55 -0
  92. data/spec/octopus/sharded_spec.rb +10 -10
  93. data/spec/spec_helper.rb +8 -6
  94. data/spec/support/active_record/connection_adapters/modify_config_adapter.rb +1 -3
  95. data/spec/support/database_connection.rb +2 -2
  96. data/spec/support/database_models.rb +18 -17
  97. data/spec/support/octopus_helper.rb +32 -25
  98. data/spec/support/query_count.rb +1 -3
  99. data/spec/support/shared_contexts.rb +3 -3
  100. data/spec/tasks/octopus.rake_spec.rb +10 -10
  101. metadata +112 -70
  102. data/.ruby-version +0 -1
  103. data/init.rb +0 -1
  104. data/lib/octopus/association_collection.rb +0 -49
  105. data/lib/octopus/has_and_belongs_to_many_association.rb +0 -17
  106. data/lib/octopus/rails3/persistence.rb +0 -39
  107. data/lib/octopus/rails3/singular_association.rb +0 -34
  108. data/rails/init.rb +0 -1
  109. data/spec/octopus/association_spec.rb +0 -712
@@ -2,6 +2,6 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activerecord", "~> 3.2.0"
5
+ gem "activerecord", "~> 5.0.0"
6
6
 
7
- gemspec :path=>"../"
7
+ gemspec path: "../"
@@ -2,6 +2,6 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activerecord", "~> 4.0.0"
5
+ gem "activerecord", "~> 5.1.0"
6
6
 
7
- gemspec :path=>"../"
7
+ gemspec path: "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "activerecord", "~> 5.2.0"
6
+
7
+ gemspec path: "../"
data/lib/ar-octopus.rb CHANGED
@@ -1 +1 @@
1
- require "octopus"
1
+ require 'octopus'
@@ -2,13 +2,7 @@
2
2
  module Octopus
3
3
  module AbstractAdapter
4
4
  module OctopusShard
5
-
6
- parent = ActiveSupport::BasicObject
7
- if Octopus.rails4?
8
- parent = ActiveSupport::ProxyObject
9
- end
10
-
11
- class InstrumenterDecorator < parent
5
+ class InstrumenterDecorator < ActiveSupport::ProxyObject
12
6
  def initialize(adapter, instrumenter)
13
7
  @adapter = adapter
14
8
  @instrumenter = instrumenter
@@ -24,21 +18,16 @@ module Octopus
24
18
  end
25
19
  end
26
20
 
27
- def self.included(base)
28
- base.alias_method_chain :initialize, :octopus_shard
29
- end
30
-
31
21
  def octopus_shard
32
22
  @config[:octopus_shard]
33
23
  end
34
24
 
35
- def initialize_with_octopus_shard(*args)
36
- initialize_without_octopus_shard(*args)
25
+ def initialize(*args)
26
+ super
37
27
  @instrumenter = InstrumenterDecorator.new(self, @instrumenter)
38
28
  end
39
-
40
29
  end
41
30
  end
42
31
  end
43
32
 
44
- ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:include, Octopus::AbstractAdapter::OctopusShard)
33
+ ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:prepend, Octopus::AbstractAdapter::OctopusShard)
@@ -1,105 +1,14 @@
1
- module Octopus::Association
2
- def self.extended(base)
3
- base.send(:include, InstanceMethods)
4
- end
5
-
6
- module QueryOnCurrentShard
7
-
8
- METHODS= %w[
9
- all
10
- average
11
- count
12
- empty?
13
- exists?
14
- find
15
- find_by_sql
16
- first
17
- last
18
- maximum
19
- minimum
20
- pluck
21
- scoping
22
- size
23
- sum
24
- to_a
25
- ]
26
-
27
- METHODS.each do |m|
28
- define_method m.to_sym do |*args,&block|
29
- if self.respond_to?(:proxy_association) and self.proxy_association
30
- self.proxy_association.owner.run_on_shard { super(*args, &block) }
31
- else
32
- super(*args, &block)
33
- end
34
- end
35
- end
36
-
37
- end
38
-
39
- module InstanceMethods
40
- def set_connection_on_association(record)
41
- return unless ::Octopus.enabled?
42
- return if !self.class.connection.respond_to?(:current_shard) || !self.respond_to?(:current_shard)
43
- if !record.current_shard.nil? && !self.current_shard.nil? && record.current_shard != self.current_shard
44
- raise "Association Error: Records are from different shards"
45
- end
46
-
47
- record.current_shard = self.class.connection.current_shard = self.current_shard if should_set_current_shard?
48
- end
49
- end
50
-
51
- if Octopus.rails4?
52
- def has_many(association_id, scope=nil, options={}, &extension)
53
- if options == {} && scope.is_a?(Hash)
54
- default_octopus_opts(scope)
55
- else
56
- default_octopus_opts(options)
57
- end
58
- super
59
- end
60
- else
61
- def has_many(association_id, options={}, &extension)
62
- default_octopus_opts(options)
63
- super
1
+ module Octopus
2
+ module Association
3
+ def self.included(base)
4
+ base.send(:include, Octopus::ShardTracking::Dynamic)
5
+ base.sharded_methods :target_scope
64
6
  end
65
- end
66
7
 
67
-
68
- if Octopus.rails4?
69
- def has_and_belongs_to_many(association_id, scope=nil, options={}, &extension)
70
- if options == {} && scope.is_a?(Hash)
71
- default_octopus_opts(scope)
72
- else
73
- default_octopus_opts(options)
74
- end
75
- super
76
- end
77
- else
78
- def has_and_belongs_to_many(association_id, options={}, &extension)
79
- default_octopus_opts(options)
80
- super
8
+ def current_shard
9
+ owner.current_shard
81
10
  end
82
11
  end
83
-
84
- def default_octopus_opts(options)
85
- if options[:before_add].is_a?(Array)
86
- options[:before_add] << :set_connection_on_association
87
- elsif options[:before_add].is_a?(Symbol)
88
- options[:before_add] = [:set_connection_on_association, options[:before_add]]
89
- else
90
- options[:before_add] = :set_connection_on_association
91
- end
92
-
93
- if options[:before_remove].is_a?(Array)
94
- options[:before_remove] << :set_connection_on_association
95
- elsif options[:before_remove].is_a?(Symbol)
96
- options[:before_remove] = [:set_connection_on_association, options[:before_remove]]
97
- else
98
- options[:before_remove] = :set_connection_on_association
99
- end
100
-
101
- options[:extend] = [ Octopus::Association::QueryOnCurrentShard, options[:extend] ].flatten.compact
102
- end
103
12
  end
104
13
 
105
- ActiveRecord::Base.extend(Octopus::Association)
14
+ ActiveRecord::Associations::Association.send(:include, Octopus::Association)
@@ -0,0 +1,74 @@
1
+ module Octopus
2
+ module AssociationShardTracking
3
+ class MismatchedShards < StandardError
4
+ attr_reader :record, :current_shard
5
+
6
+ def initialize(record, current_shard)
7
+ @record = record
8
+ @current_shard = current_shard
9
+ end
10
+
11
+ def message
12
+ [
13
+ "Association Error: Records are from different shards",
14
+ "Record: #{record.inspect}",
15
+ "Current Shard: #{current_shard.inspect}",
16
+ "Current Record Shard: #{record.current_shard.inspect}",
17
+ ].join(" ")
18
+ end
19
+ end
20
+
21
+ def self.extended(base)
22
+ base.send(:include, InstanceMethods)
23
+ end
24
+
25
+ module InstanceMethods
26
+ def connection_on_association=(record)
27
+ return unless ::Octopus.enabled?
28
+ return if !self.class.connection.respond_to?(:current_shard) || !self.respond_to?(:current_shard)
29
+
30
+ if !record.current_shard.nil? && !current_shard.nil? && record.current_shard.to_s != current_shard.to_s
31
+ raise MismatchedShards.new(record, current_shard)
32
+ end
33
+
34
+ record.current_shard = self.class.connection.current_shard = current_shard if should_set_current_shard?
35
+ end
36
+ end
37
+
38
+ def has_many(name, scope = nil, **options, &extension)
39
+ if options == {} && scope.is_a?(Hash)
40
+ default_octopus_opts(scope)
41
+ else
42
+ default_octopus_opts(options)
43
+ end
44
+ super
45
+ end
46
+
47
+ def has_and_belongs_to_many(association_id, scope = nil, options = {}, &extension)
48
+ assign_octopus_opts(scope, options)
49
+ super
50
+ end
51
+
52
+ def default_octopus_opts(options)
53
+ options[:before_add] = [ :connection_on_association=, options[:before_add] ].compact.flatten
54
+ options[:before_remove] = [ :connection_on_association=, options[:before_remove] ].compact.flatten
55
+ end
56
+
57
+ def assign_octopus_opts(scope, options)
58
+ if options == {} && scope.is_a?(Hash)
59
+ default_octopus_opts(scope)
60
+ else
61
+ default_octopus_opts(options)
62
+ end
63
+ end
64
+
65
+ if Octopus.atleast_rails51?
66
+ def has_and_belongs_to_many(association_id, scope = nil, **options, &extension)
67
+ assign_octopus_opts(scope, options)
68
+ super
69
+ end
70
+ end
71
+ end
72
+ end
73
+
74
+ ActiveRecord::Base.extend(Octopus::AssociationShardTracking)
@@ -0,0 +1,17 @@
1
+ module Octopus
2
+ module CollectionAssociation
3
+ def self.included(base)
4
+ if Octopus.rails51? || Octopus.rails52?
5
+ base.sharded_methods :reader, :writer, :ids_reader, :ids_writer, :create, :create!,
6
+ :build, :include?,
7
+ :load_target, :reload, :size, :select
8
+ else
9
+ base.sharded_methods :reader, :writer, :ids_reader, :ids_writer, :create, :create!,
10
+ :build, :any?, :count, :empty?, :first, :include?, :last, :length,
11
+ :load_target, :many?, :reload, :size, :select, :uniq
12
+ end
13
+ end
14
+ end
15
+ end
16
+
17
+ ActiveRecord::Associations::CollectionAssociation.send(:include, Octopus::CollectionAssociation)
@@ -0,0 +1,16 @@
1
+ module Octopus
2
+ module CollectionProxy
3
+ def self.included(base)
4
+ base.send(:include, Octopus::ShardTracking::Dynamic)
5
+ base.sharded_methods :any?, :build, :count, :create, :create!, :concat, :delete, :delete_all,
6
+ :destroy, :destroy_all, :empty?, :find, :first, :include?, :last, :length,
7
+ :many?, :pluck, :replace, :select, :size, :sum, :to_a, :uniq
8
+ end
9
+
10
+ def current_shard
11
+ @association.owner.current_shard
12
+ end
13
+ end
14
+ end
15
+
16
+ ActiveRecord::Associations::CollectionProxy.send(:include, Octopus::CollectionProxy)
@@ -0,0 +1,4 @@
1
+ module Octopus
2
+ class Exception < ::Exception
3
+ end
4
+ end
@@ -0,0 +1,8 @@
1
+ # find_nth / find_nth! must be public here to allow Octopus to call
2
+ # them on the scope proxy object
3
+ if Octopus.rails42? || Octopus.rails50?
4
+ module ActiveRecord::FinderMethods
5
+ public :find_nth
6
+ public :find_nth!
7
+ end
8
+ end
@@ -0,0 +1,20 @@
1
+ require 'octopus/load_balancing'
2
+
3
+ # The round-robin load balancing of slaves belonging to the same shard.
4
+ # It is a pool that contains slaves which queries are distributed to.
5
+ module Octopus
6
+ module LoadBalancing
7
+ class RoundRobin
8
+ def initialize(slaves_list)
9
+ raise Octopus::Exception.new("No slaves available") if slaves_list.empty?
10
+ @slaves_list = slaves_list
11
+ @slave_index = 0
12
+ end
13
+
14
+ # Returns the next available slave in the pool
15
+ def next(options)
16
+ @slaves_list[@slave_index = (@slave_index + 1) % @slaves_list.length]
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,4 @@
1
+ module Octopus
2
+ module LoadBalancing
3
+ end
4
+ end
@@ -3,8 +3,12 @@ module Octopus
3
3
  module LogSubscriber
4
4
  def self.included(base)
5
5
  base.send(:attr_accessor, :octopus_shard)
6
- base.alias_method_chain :sql, :octopus_shard
7
- base.alias_method_chain :debug, :octopus_shard
6
+
7
+ base.send :alias_method, :sql_without_octopus_shard, :sql
8
+ base.send :alias_method, :sql, :sql_with_octopus_shard
9
+
10
+ base.send :alias_method, :debug_without_octopus_shard, :debug
11
+ base.send :alias_method, :debug, :debug_with_octopus_shard
8
12
  end
9
13
 
10
14
  def sql_with_octopus_shard(event)