ar-octopus 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. data/.gitignore +11 -0
  2. data/.travis.yml +22 -0
  3. data/Appraisals +18 -0
  4. data/Gemfile +3 -12
  5. data/README.mkdn +63 -24
  6. data/Rakefile +70 -92
  7. data/ar-octopus.gemspec +25 -198
  8. data/lib/ar-octopus.rb +1 -0
  9. data/lib/octopus.rb +73 -25
  10. data/lib/octopus/association.rb +6 -5
  11. data/lib/octopus/association_collection.rb +58 -4
  12. data/lib/octopus/has_and_belongs_to_many_association.rb +4 -4
  13. data/lib/octopus/logger.rb +9 -4
  14. data/lib/octopus/migration.rb +155 -50
  15. data/lib/octopus/model.rb +98 -34
  16. data/lib/octopus/proxy.rb +124 -53
  17. data/lib/octopus/rails2/association.rb +46 -93
  18. data/lib/octopus/rails2/persistence.rb +1 -1
  19. data/lib/octopus/rails2/scope.rb +17 -0
  20. data/lib/octopus/rails3.1/singular_association.rb +34 -0
  21. data/lib/octopus/rails3.2/persistence.rb +12 -0
  22. data/lib/octopus/rails3/abstract_adapter.rb +39 -0
  23. data/lib/octopus/rails3/arel.rb +5 -5
  24. data/lib/octopus/rails3/log_subscriber.rb +22 -0
  25. data/lib/octopus/rails3/persistence.rb +10 -5
  26. data/lib/octopus/railtie.rb +13 -0
  27. data/lib/octopus/scope_proxy.rb +22 -16
  28. data/lib/octopus/version.rb +3 -0
  29. data/lib/tasks/octopus.rake +20 -0
  30. data/sample_app/Gemfile +2 -2
  31. data/sample_app/config/initializers/inflections.rb +1 -1
  32. data/sample_app/config/initializers/secret_token.rb +1 -1
  33. data/sample_app/db/migrate/20100720172730_create_items.rb +1 -1
  34. data/sample_app/db/migrate/20100720210335_create_sample_users.rb +1 -1
  35. data/sample_app/db/seeds.rb +1 -1
  36. data/sample_app/features/migrate.feature +12 -12
  37. data/sample_app/features/seed.feature +3 -3
  38. data/sample_app/features/step_definitions/web_steps.rb +5 -5
  39. data/sample_app/features/support/env.rb +8 -8
  40. data/sample_app/lib/tasks/cucumber.rake +2 -2
  41. data/sample_app/public/javascripts/effects.js +1 -1
  42. data/spec/config/shards.yml +38 -28
  43. data/spec/migrations/11_add_field_in_all_slaves.rb +1 -1
  44. data/spec/migrations/12_create_users_using_block.rb +2 -2
  45. data/spec/migrations/13_create_users_using_block_and_using.rb +2 -2
  46. data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +11 -0
  47. data/spec/migrations/1_create_users_on_master.rb +1 -1
  48. data/spec/migrations/2_create_users_on_canada.rb +1 -1
  49. data/spec/migrations/3_create_users_on_both_shards.rb +1 -1
  50. data/spec/migrations/4_create_users_on_shards_of_a_group.rb +1 -1
  51. data/spec/migrations/5_create_users_on_multiples_groups.rb +1 -1
  52. data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +1 -1
  53. data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +1 -1
  54. data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +1 -1
  55. data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +1 -1
  56. data/spec/octopus/association_spec.rb +88 -70
  57. data/spec/octopus/log_subscriber_spec.rb +22 -0
  58. data/spec/octopus/logger_spec.rb +28 -15
  59. data/spec/octopus/migration_spec.rb +47 -43
  60. data/spec/octopus/model_spec.rb +179 -13
  61. data/spec/octopus/octopus_spec.rb +26 -4
  62. data/spec/octopus/proxy_spec.rb +61 -23
  63. data/spec/octopus/{replication_specs.rb → replication_spec.rb} +33 -26
  64. data/spec/octopus/scope_proxy_spec.rb +3 -3
  65. data/spec/octopus/sharded_spec.rb +9 -9
  66. data/spec/spec_helper.rb +10 -12
  67. data/spec/support/active_record/connection_adapters/modify_config_adapter.rb +17 -0
  68. data/spec/support/database_connection.rb +2 -0
  69. data/spec/{database_models.rb → support/database_models.rb} +27 -2
  70. data/spec/support/octopus_helper.rb +50 -0
  71. data/spec/tasks/octopus.rake_spec.rb +36 -0
  72. metadata +188 -169
  73. data/Gemfile.lock +0 -68
  74. data/lib/octopus/rails3/association.rb +0 -112
  75. data/spec/database_connection.rb +0 -4
  76. data/spec/octopus/controller_spec.rb +0 -34
  77. data/spec/octopus_helper.rb +0 -37
@@ -1,46 +1,53 @@
1
1
  require "set"
2
2
 
3
3
  class Octopus::Proxy
4
- attr_accessor :current_model, :current_shard, :current_group, :block, :using_enabled, :last_current_shard, :config
4
+ attr_accessor :config
5
5
 
6
- def initialize(config)
6
+ def initialize(config = Octopus.config)
7
7
  initialize_shards(config)
8
8
  initialize_replication(config) if !config.nil? && config["replicated"]
9
9
  end
10
10
 
11
11
  def initialize_shards(config)
12
12
  @shards = HashWithIndifferentAccess.new
13
- @groups = HashWithIndifferentAccess.new
13
+ @groups = {}
14
14
  @adapters = Set.new
15
- @shards[:master] = ActiveRecord::Base.connection_pool()
16
- @config = ActiveRecord::Base.connection_pool.connection.instance_variable_get(:@config)
17
- @current_shard = :master
18
-
15
+ @shards[:master] = ActiveRecord::Base.connection_pool_without_octopus()
16
+ @config = ActiveRecord::Base.connection_pool_without_octopus.connection.instance_variable_get(:@config)
17
+
19
18
  if !config.nil? && config.has_key?("verify_connection")
20
19
  @verify_connection = config["verify_connection"]
21
20
  else
22
21
  @verify_connection = false
23
22
  end
24
-
23
+
25
24
  if !config.nil?
26
- @entire_sharded = config['entire_sharded']
27
- shards_config = config[Octopus.rails_env()]
25
+ @entire_sharded = config['entire_sharded']
26
+ shards_config = config[Octopus.rails_env()]
28
27
  end
29
-
28
+
30
29
  shards_config ||= []
31
30
 
32
31
  shards_config.each do |key, value|
33
- if value.has_key?("adapter")
32
+ if value.is_a?(String) && Octopus.rails32?
33
+ value = resolve_string_connection(value).merge(:octopus_shard => key)
34
34
  initialize_adapter(value['adapter'])
35
35
  @shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
36
- else
37
- @groups[key.to_sym] = []
36
+ elsif value.is_a?(Hash) && value.has_key?("adapter")
37
+ value.merge!(:octopus_shard => key)
38
+ initialize_adapter(value['adapter'])
39
+ @shards[key.to_sym] = connection_pool_for(value, "#{value['adapter']}_connection")
40
+ elsif value.is_a?(Hash)
41
+ @groups[key.to_s] = []
38
42
 
39
43
  value.each do |k, v|
40
44
  raise "You have duplicated shard names!" if @shards.has_key?(k.to_sym)
45
+
41
46
  initialize_adapter(v['adapter'])
42
- @shards[k.to_sym] = connection_pool_for(v, "#{v['adapter']}_connection")
43
- @groups[key.to_sym] << k.to_sym
47
+ config_with_octopus_shard = v.merge(:octopus_shard => k)
48
+
49
+ @shards[k.to_sym] = connection_pool_for(config_with_octopus_shard, "#{v['adapter']}_connection")
50
+ @groups[key.to_s] << k.to_sym
44
51
  end
45
52
  end
46
53
  end
@@ -53,8 +60,21 @@ class Octopus::Proxy
53
60
  else
54
61
  @fully_replicated = true
55
62
  end
56
- @slaves_list = @shards.keys.map {|sym| sym.to_s}.sort
57
- @slaves_list.delete('master')
63
+ @slaves_list = @shards.keys.map {|sym| sym.to_s}.sort
64
+ @slaves_list.delete('master')
65
+ @slave_index = 0
66
+ end
67
+
68
+ def current_model
69
+ Thread.current["octopus.current_model"]
70
+ end
71
+
72
+ def current_model=(model)
73
+ Thread.current["octopus.current_model"] = model.is_a?(ActiveRecord::Base) ? model.class : model
74
+ end
75
+
76
+ def current_shard
77
+ Thread.current["octopus.current_shard"] ||= :master
58
78
  end
59
79
 
60
80
  def current_shard=(shard_symbol)
@@ -64,36 +84,83 @@ class Octopus::Proxy
64
84
  raise "Nonexistent Shard Name: #{shard_symbol}" if @shards[shard_symbol].nil?
65
85
  end
66
86
 
67
- @current_shard = shard_symbol
87
+ Thread.current["octopus.current_shard"] = shard_symbol
88
+ end
89
+
90
+ def current_group
91
+ Thread.current["octopus.current_group"]
68
92
  end
69
93
 
70
94
  def current_group=(group_symbol)
71
- if group_symbol.is_a?(Array)
72
- group_symbol.each {|symbol| raise "Nonexistent Group Name: #{symbol}" if @groups[symbol].nil? }
73
- else
74
- raise "Nonexistent Group Name: #{group_symbol}" if @groups[group_symbol].nil?
95
+ # TODO: Error message should include all groups if given more than one bad name.
96
+ [group_symbol].flatten.compact.each do |group|
97
+ raise "Nonexistent Group Name: #{group}" unless has_group?(group)
75
98
  end
76
99
 
77
- @current_group = group_symbol
100
+ Thread.current["octopus.current_group"] = group_symbol
78
101
  end
79
102
 
80
- def current_model=(model)
81
- @current_model = model.is_a?(ActiveRecord::Base) ? model.class : model
103
+ def block
104
+ Thread.current["octopus.block"]
105
+ end
106
+
107
+ def block=(block)
108
+ Thread.current["octopus.block"] = block
109
+ end
110
+
111
+ def last_current_shard
112
+ Thread.current["octopus.last_current_shard"]
82
113
  end
83
114
 
84
- def select_connection()
85
- @shards[shard_name].verify_active_connections! if @verify_connection
115
+ def last_current_shard=(last_current_shard)
116
+ Thread.current["octopus.last_current_shard"] = last_current_shard
117
+ end
118
+
119
+ # Public: Whether or not a group exists with the given name converted to a
120
+ # string.
121
+ #
122
+ # Returns a boolean.
123
+ def has_group?(group)
124
+ @groups.has_key?(group.to_s)
125
+ end
126
+
127
+ # Public: Retrieves names of all loaded shards.
128
+ #
129
+ # Returns an array of shard names as symbols
130
+ def shard_names
131
+ @shards.keys
132
+ end
133
+
134
+ # Public: Retrieves the defined shards for a given group.
135
+ #
136
+ # Returns an array of shard names as symbols or nil if the group is not
137
+ # defined.
138
+ def shards_for_group(group)
139
+ @groups.fetch(group.to_s, nil)
140
+ end
141
+
142
+ def select_connection
143
+ @shards[shard_name].verify_active_connections! if @verify_connection
144
+ # Rails 3.1 sets automatic_reconnect to false when it removes
145
+ # connection pool. Octopus can potentially retain a reference to a closed
146
+ # connection pool. Previously, that would work since the pool would just
147
+ # reconnect, but in Rails 3.1 the flag prevents this.
148
+ if Octopus.rails31? || Octopus.rails32?
149
+ if !@shards[shard_name].automatic_reconnect
150
+ @shards[shard_name].automatic_reconnect = true
151
+ end
152
+ end
86
153
  @shards[shard_name].connection()
87
154
  end
88
155
 
89
156
  def shard_name
90
157
  current_shard.is_a?(Array) ? current_shard.first : current_shard
91
158
  end
92
-
159
+
93
160
  def should_clean_table_name?
94
161
  @adapters.size > 1
95
162
  end
96
-
163
+
97
164
  def run_queries_on_shard(shard, &block)
98
165
  older_shard = self.current_shard
99
166
  last_block = self.block
@@ -107,28 +174,27 @@ class Octopus::Proxy
107
174
  self.current_shard = older_shard
108
175
  end
109
176
  end
110
-
177
+
111
178
  def send_queries_to_multiple_shards(shards, &block)
112
179
  shards.each do |shard|
113
180
  self.run_queries_on_shard(shard, &block)
114
181
  end
115
182
  end
116
-
183
+
117
184
  def clean_proxy()
118
- @using_enabled = nil
119
- @current_shard = :master
120
- @current_group = nil
121
- @block = false
185
+ self.current_shard = :master
186
+ self.current_group = nil
187
+ self.block = false
122
188
  end
123
-
189
+
124
190
  def check_schema_migrations(shard)
125
- if !ActiveRecord::Base.using(shard).connection.table_exists?(ActiveRecord::Migrator.schema_migrations_table_name())
126
- ActiveRecord::Base.using(shard).connection.initialize_schema_migrations_table
191
+ if !OctopusModel.using(shard).connection.table_exists?(ActiveRecord::Migrator.schema_migrations_table_name())
192
+ OctopusModel.using(shard).connection.initialize_schema_migrations_table
127
193
  end
128
194
  end
129
-
195
+
130
196
  def transaction(options = {}, &block)
131
- if @replicated && (current_model.read_inheritable_attribute(:replicated) || @fully_replicated)
197
+ if @replicated && (current_model.replicated || @fully_replicated)
132
198
  self.run_queries_on_shard(:master) do
133
199
  select_connection.transaction(options, &block)
134
200
  end
@@ -144,7 +210,7 @@ class Octopus::Proxy
144
210
  clean_proxy()
145
211
  conn.send(method, *args, &block)
146
212
  elsif should_send_queries_to_replicated_databases?(method)
147
- send_queries_to_selected_slave(method, *args, &block)
213
+ send_queries_to_selected_slave(method, *args, &block)
148
214
  else
149
215
  select_connection().send(method, *args, &block)
150
216
  end
@@ -154,9 +220,13 @@ class Octopus::Proxy
154
220
  super || select_connection.respond_to?(method, include_private)
155
221
  end
156
222
 
223
+ def connection_pool
224
+ return @shards[current_shard]
225
+ end
226
+
157
227
  protected
158
228
  def connection_pool_for(adapter, config)
159
- ActiveRecord::ConnectionAdapters::ConnectionPool.new(ActiveRecord::Base::ConnectionSpecification.new(adapter, config))
229
+ ActiveRecord::ConnectionAdapters::ConnectionPool.new(ActiveRecord::Base::ConnectionSpecification.new(adapter.dup, config))
160
230
  end
161
231
 
162
232
  def initialize_adapter(adapter)
@@ -168,30 +238,31 @@ class Octopus::Proxy
168
238
  end
169
239
  end
170
240
 
241
+ def resolve_string_connection(spec)
242
+ ActiveRecord::Base::ConnectionSpecification::Resolver.new(spec, {}).spec.config.stringify_keys
243
+ end
244
+
171
245
  def should_clean_connection?(method)
172
246
  method.to_s =~ /insert|select|execute/ && !@replicated && !self.block
173
247
  end
174
248
 
175
249
  def should_send_queries_to_replicated_databases?(method)
176
- @replicated && method.to_s =~ /select/ && !@block
250
+ @replicated && method.to_s =~ /select/ && !self.block
177
251
  end
178
252
 
179
- def send_queries_to_selected_slave(method, *args, &block)
253
+ def send_queries_to_selected_slave(method, *args, &block)
180
254
  old_shard = self.current_shard
181
-
255
+
182
256
  begin
183
- if current_model.read_inheritable_attribute(:replicated) || @fully_replicated
184
- self.current_shard = @slaves_list.shift.to_sym
185
- @slaves_list << self.current_shard
257
+ if current_model.replicated || @fully_replicated
258
+ self.current_shard = @slaves_list[@slave_index = (@slave_index + 1) % @slaves_list.length]
186
259
  else
187
260
  self.current_shard = :master
188
261
  end
189
-
190
- sql = select_connection().send(method, *args, &block)
191
- return sql
262
+
263
+ select_connection.send(method, *args, &block)
192
264
  ensure
193
265
  self.current_shard = old_shard
194
- @using_enabled = nil
195
266
  end
196
267
  end
197
268
  end
@@ -2,129 +2,82 @@ module Octopus
2
2
  module Rails2
3
3
  module Association
4
4
  def association_accessor_methods(reflection, association_proxy_class)
5
- define_method(reflection.name) do |*params|
6
- force_reload = params.first unless params.empty?
7
- reload_connection()
8
- association = association_instance_get(reflection.name)
9
-
10
- if association.nil? || force_reload
11
- association = association_proxy_class.new(self, reflection)
12
- retval = force_reload ? reflection.klass.uncached { association.reload } : association.reload
13
- if retval.nil? and association_proxy_class == ActiveRecord::Associations::BelongsToAssociation
14
- association_instance_set(reflection.name, nil)
15
- return nil
16
- end
17
- association_instance_set(reflection.name, association)
18
- end
5
+ super
19
6
 
20
- association.target.nil? ? nil : association
7
+ define_method("#{reflection.name}_with_octopus") do |*params|
8
+ reload_connection
9
+ send("#{reflection.name}_without_octopus", *params)
21
10
  end
22
11
 
23
- define_method("loaded_#{reflection.name}?") do
24
- reload_connection()
25
- association = association_instance_get(reflection.name)
26
- association && association.loaded?
12
+ define_method("loaded_#{reflection.name}_with_octopus?") do
13
+ reload_connection
14
+ send("loaded_#{reflection.name}_without_octopus?")
27
15
  end
28
16
 
29
- define_method("#{reflection.name}=") do |new_value|
30
- association = association_instance_get(reflection.name)
31
- reload_connection()
32
- if association.nil? || association.target != new_value
33
- association = association_proxy_class.new(self, reflection)
34
- end
35
-
36
- if association_proxy_class == ActiveRecord::Associations::HasOneThroughAssociation
37
- association.create_through_record(new_value)
38
- if new_record?
39
- association_instance_set(reflection.name, new_value.nil? ? nil : association)
40
- else
41
- self.send(reflection.name, new_value)
42
- end
43
- else
44
- association.replace(new_value)
45
- association_instance_set(reflection.name, new_value.nil? ? nil : association)
46
- end
17
+ define_method("#{reflection.name}_with_octopus=") do |new_value|
18
+ reload_connection
19
+ send("#{reflection.name}_without_octopus=", new_value)
47
20
  end
48
21
 
49
- define_method("set_#{reflection.name}_target") do |target|
50
- reload_connection()
51
- return if target.nil? and association_proxy_class == ActiveRecord::Associations::BelongsToAssociation
52
- association = association_proxy_class.new(self, reflection)
53
- association.target = target
54
- association_instance_set(reflection.name, association)
22
+ define_method("set_#{reflection.name}_target_with_octopus") do |target|
23
+ reload_connection
24
+ send("set_#{reflection.name}_target_without_octopus", target)
55
25
  end
26
+
27
+ alias_method_chain reflection.name, "octopus"
28
+ alias_method_chain "loaded_#{reflection.name}?", "octopus"
29
+ alias_method_chain "#{reflection.name}=", "octopus"
30
+ alias_method_chain "set_#{reflection.name}_target", "octopus"
56
31
  end
57
32
 
58
33
  def collection_reader_method(reflection, association_proxy_class)
59
- define_method(reflection.name) do |*params|
60
- force_reload = params.first unless params.empty?
61
- reload_connection()
62
- association = association_instance_get(reflection.name)
63
-
64
- unless association
65
- association = association_proxy_class.new(self, reflection)
66
- association_instance_set(reflection.name, association)
67
- end
34
+ super
68
35
 
69
- reflection.klass.uncached { association.reload } if force_reload
70
-
71
- association
36
+ define_method("#{reflection.name}_with_octopus") do |*params|
37
+ reload_connection
38
+ send("#{reflection.name}_without_octopus", *params)
72
39
  end
73
40
 
74
- define_method("#{reflection.name.to_s.singularize}_ids") do
75
- reload_connection()
76
- if send(reflection.name).loaded? || reflection.options[:finder_sql]
77
- send(reflection.name).map(&:id)
78
- else
79
- send(reflection.name).all(:select => "#{reflection.quoted_table_name}.#{reflection.klass.primary_key}").map(&:id)
80
- end
41
+ define_method("#{reflection.name.to_s.singularize}_ids_with_octopus") do
42
+ reload_connection
43
+ send("#{reflection.name.to_s.singularize}_ids_without_octopus")
81
44
  end
45
+
46
+ alias_method_chain reflection.name, "octopus"
47
+ alias_method_chain "#{reflection.name.to_s.singularize}_ids", "octopus"
82
48
  end
83
49
 
84
50
  def collection_accessor_methods(reflection, association_proxy_class, writer = true)
85
- collection_reader_method(reflection, association_proxy_class)
51
+ super
86
52
 
87
53
  if writer
88
- define_method("#{reflection.name}=") do |new_value|
89
- reload_connection()
90
- # Loads proxy class instance (defined in collection_reader_method) if not already loaded
91
- association = send(reflection.name)
92
- association.replace(new_value)
93
- association
54
+ define_method("#{reflection.name}_with_octopus=") do |new_value|
55
+ reload_connection
56
+ send("#{reflection.name}_without_octopus=", new_value)
94
57
  end
95
58
 
96
- define_method("#{reflection.name.to_s.singularize}_ids=") do |new_value|
97
- reload_connection()
98
- ids = (new_value || []).reject { |nid| nid.blank? }.map(&:to_i)
99
- send("#{reflection.name}=", reflection.klass.find(ids).index_by(&:id).values_at(*ids))
59
+ define_method("#{reflection.name.to_s.singularize}_ids_with_octopus=") do |new_value|
60
+ reload_connection
61
+ send("#{reflection.name.to_s.singularize}_ids_without_octopus=", new_value)
100
62
  end
63
+
64
+ alias_method_chain "#{reflection.name}=", "octopus"
65
+ alias_method_chain "#{reflection.name.to_s.singularize}_ids=", "octopus"
101
66
  end
102
67
  end
103
68
 
104
69
  def association_constructor_method(constructor, reflection, association_proxy_class)
105
- define_method("#{constructor}_#{reflection.name}") do |*params|
106
- reload_connection()
107
- attributees = params.first unless params.empty?
108
- replace_existing = params[1].nil? ? true : params[1]
109
- association = association_instance_get(reflection.name)
110
-
111
- unless association
112
- association = association_proxy_class.new(self, reflection)
113
- association_instance_set(reflection.name, association)
114
- end
70
+ super
115
71
 
116
- if association_proxy_class == ActiveRecord::Associations::HasOneAssociation
117
- ret_val = association.send(constructor, attributees, replace_existing)
118
- else
119
- ret_val = association.send(constructor, attributees)
120
- end
72
+ define_method("#{constructor}_#{reflection.name}_with_octopus") do |*params|
73
+ reload_connection
74
+ result = send("#{constructor}_#{reflection.name}_without_octopus", *params)
121
75
 
122
- if should_set_current_shard?
123
- ret_val.current_shard = self.current_shard
124
- end
125
-
126
- return ret_val
76
+ result.current_shard = current_shard if should_set_current_shard?
77
+ result
127
78
  end
79
+
80
+ alias_method_chain "#{constructor}_#{reflection.name}", "octopus"
128
81
  end
129
82
  end
130
83
  end