ar-octopus 0.4.0 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (77) hide show
  1. data/.gitignore +11 -0
  2. data/.travis.yml +22 -0
  3. data/Appraisals +18 -0
  4. data/Gemfile +3 -12
  5. data/README.mkdn +63 -24
  6. data/Rakefile +70 -92
  7. data/ar-octopus.gemspec +25 -198
  8. data/lib/ar-octopus.rb +1 -0
  9. data/lib/octopus.rb +73 -25
  10. data/lib/octopus/association.rb +6 -5
  11. data/lib/octopus/association_collection.rb +58 -4
  12. data/lib/octopus/has_and_belongs_to_many_association.rb +4 -4
  13. data/lib/octopus/logger.rb +9 -4
  14. data/lib/octopus/migration.rb +155 -50
  15. data/lib/octopus/model.rb +98 -34
  16. data/lib/octopus/proxy.rb +124 -53
  17. data/lib/octopus/rails2/association.rb +46 -93
  18. data/lib/octopus/rails2/persistence.rb +1 -1
  19. data/lib/octopus/rails2/scope.rb +17 -0
  20. data/lib/octopus/rails3.1/singular_association.rb +34 -0
  21. data/lib/octopus/rails3.2/persistence.rb +12 -0
  22. data/lib/octopus/rails3/abstract_adapter.rb +39 -0
  23. data/lib/octopus/rails3/arel.rb +5 -5
  24. data/lib/octopus/rails3/log_subscriber.rb +22 -0
  25. data/lib/octopus/rails3/persistence.rb +10 -5
  26. data/lib/octopus/railtie.rb +13 -0
  27. data/lib/octopus/scope_proxy.rb +22 -16
  28. data/lib/octopus/version.rb +3 -0
  29. data/lib/tasks/octopus.rake +20 -0
  30. data/sample_app/Gemfile +2 -2
  31. data/sample_app/config/initializers/inflections.rb +1 -1
  32. data/sample_app/config/initializers/secret_token.rb +1 -1
  33. data/sample_app/db/migrate/20100720172730_create_items.rb +1 -1
  34. data/sample_app/db/migrate/20100720210335_create_sample_users.rb +1 -1
  35. data/sample_app/db/seeds.rb +1 -1
  36. data/sample_app/features/migrate.feature +12 -12
  37. data/sample_app/features/seed.feature +3 -3
  38. data/sample_app/features/step_definitions/web_steps.rb +5 -5
  39. data/sample_app/features/support/env.rb +8 -8
  40. data/sample_app/lib/tasks/cucumber.rake +2 -2
  41. data/sample_app/public/javascripts/effects.js +1 -1
  42. data/spec/config/shards.yml +38 -28
  43. data/spec/migrations/11_add_field_in_all_slaves.rb +1 -1
  44. data/spec/migrations/12_create_users_using_block.rb +2 -2
  45. data/spec/migrations/13_create_users_using_block_and_using.rb +2 -2
  46. data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +11 -0
  47. data/spec/migrations/1_create_users_on_master.rb +1 -1
  48. data/spec/migrations/2_create_users_on_canada.rb +1 -1
  49. data/spec/migrations/3_create_users_on_both_shards.rb +1 -1
  50. data/spec/migrations/4_create_users_on_shards_of_a_group.rb +1 -1
  51. data/spec/migrations/5_create_users_on_multiples_groups.rb +1 -1
  52. data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +1 -1
  53. data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +1 -1
  54. data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +1 -1
  55. data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +1 -1
  56. data/spec/octopus/association_spec.rb +88 -70
  57. data/spec/octopus/log_subscriber_spec.rb +22 -0
  58. data/spec/octopus/logger_spec.rb +28 -15
  59. data/spec/octopus/migration_spec.rb +47 -43
  60. data/spec/octopus/model_spec.rb +179 -13
  61. data/spec/octopus/octopus_spec.rb +26 -4
  62. data/spec/octopus/proxy_spec.rb +61 -23
  63. data/spec/octopus/{replication_specs.rb → replication_spec.rb} +33 -26
  64. data/spec/octopus/scope_proxy_spec.rb +3 -3
  65. data/spec/octopus/sharded_spec.rb +9 -9
  66. data/spec/spec_helper.rb +10 -12
  67. data/spec/support/active_record/connection_adapters/modify_config_adapter.rb +17 -0
  68. data/spec/support/database_connection.rb +2 -0
  69. data/spec/{database_models.rb → support/database_models.rb} +27 -2
  70. data/spec/support/octopus_helper.rb +50 -0
  71. data/spec/tasks/octopus.rake_spec.rb +36 -0
  72. metadata +188 -169
  73. data/Gemfile.lock +0 -68
  74. data/lib/octopus/rails3/association.rb +0 -112
  75. data/spec/database_connection.rb +0 -4
  76. data/spec/octopus/controller_spec.rb +0 -34
  77. data/spec/octopus_helper.rb +0 -37
@@ -4,12 +4,12 @@ module Octopus::HasAndBelongsToManyAssociation
4
4
  alias_method_chain :insert_record, :octopus
5
5
  end
6
6
  end
7
-
7
+
8
8
  def insert_record_with_octopus(record, force = true, validate = true)
9
9
  if should_wrap_the_connection?
10
- Octopus.using(@owner.current_shard) { insert_record_without_octopus(record, force, validate) }
11
- else
12
- insert_record_without_octopus(record, force, validate)
10
+ Octopus.using(@owner.current_shard) { insert_record_without_octopus(record, force, validate) }
11
+ else
12
+ insert_record_without_octopus(record, force, validate)
13
13
  end
14
14
  end
15
15
  end
@@ -1,13 +1,18 @@
1
1
  require "logger"
2
2
 
3
3
  class Octopus::Logger < Logger
4
+ def initialize(logdev, shift_age = 0, shift_size = 1048576)
5
+ ActiveSupport::Deprecation.warn "Octopus::Logger is deprecated and will be removed in Octopus 0.6.x", caller
6
+ super
7
+ end
8
+
4
9
  def format_message(severity, timestamp, progname, msg)
5
- str = super
6
-
10
+ str = super
11
+
7
12
  if ActiveRecord::Base.connection.respond_to?(:current_shard)
8
- str += "Shard: #{ActiveRecord::Base.connection.current_shard} -"
13
+ str += "Shard: #{ActiveRecord::Base.connection.current_shard} -"
9
14
  end
10
-
15
+
11
16
  str
12
17
  end
13
18
  end
@@ -1,73 +1,178 @@
1
- module Octopus::Migration
2
- def self.extended(base)
3
- class << base
4
- def announce_with_octopus(message)
5
- announce_without_octopus("#{message} - #{get_current_shard}")
6
- end
7
-
8
- alias_method_chain :migrate, :octopus
9
- alias_method_chain :announce, :octopus
10
- attr_accessor :current_shard
1
+ require "set"
2
+ require "active_support/core_ext/module/aliasing"
3
+
4
+ if Octopus.rails2?
5
+ require "active_support/core_ext/array/wrapper"
6
+ else
7
+ require "active_support/core_ext/array/wrap"
8
+ end
9
+
10
+ module Octopus::Migration
11
+ module InstanceOrClassMethods
12
+ def announce_with_octopus(message)
13
+ announce_without_octopus("#{message} - #{get_current_shard}")
14
+ end
15
+
16
+ def get_current_shard
17
+ "Shard: #{connection.current_shard}" if connection.respond_to?(:current_shard)
11
18
  end
12
19
  end
13
20
 
14
- def using(*args)
15
- if self.connection().is_a?(Octopus::Proxy)
16
- args.each do |shard|
17
- self.connection().check_schema_migrations(shard)
21
+ include InstanceOrClassMethods if Octopus.rails31? || Octopus.rails32?
22
+
23
+ def self.included(base)
24
+ base.send(:extend, ClassMethods)
25
+
26
+ if Octopus.rails31? || Octopus.rails32?
27
+ base.alias_method_chain :announce, :octopus
28
+ else
29
+ base.class_eval do
30
+ class << self
31
+ alias_method_chain :announce, :octopus
32
+ end
18
33
  end
34
+ end
35
+
36
+ base.class_attribute :current_shard, :current_group, :instance_reader => false, :instance_writer => false
37
+ end
38
+
39
+ module ClassMethods
40
+ include InstanceOrClassMethods unless Octopus.rails31? || Octopus.rails32?
41
+
42
+ def using(*args)
43
+ return self unless connection.is_a?(Octopus::Proxy)
19
44
 
20
- self.connection().block = true
21
45
  self.current_shard = args
22
- self.connection().current_shard = args
46
+ self
23
47
  end
24
48
 
25
- return self
26
- end
49
+ def using_group(*groups)
50
+ return self unless connection.is_a?(Octopus::Proxy)
51
+
52
+ self.current_group = groups
53
+ self
54
+ end
27
55
 
28
- def using_group(*args)
29
- if self.connection().is_a?(Octopus::Proxy)
30
- args.each do |group_shard|
31
- shards = self.connection().instance_variable_get(:@groups)[group_shard] || []
56
+ def shards
57
+ shards = Set.new
32
58
 
33
- shards.each do |shard|
34
- self.connection().check_schema_migrations(shard)
59
+ if groups = current_group
60
+ Array.wrap(groups).each do |group|
61
+ group_shards = connection.shards_for_group(group)
62
+ shards.merge(group_shards) if group_shards
35
63
  end
64
+ elsif shard = current_shard
65
+ shards.merge(Array.wrap(shard))
36
66
  end
37
67
 
38
- self.connection().block = true
39
- self.connection().current_group = args
68
+ shards.to_a.presence || [:master]
40
69
  end
41
-
42
- return self
43
70
  end
44
-
45
- def get_current_shard
46
- "Shard: #{ActiveRecord::Base.connection.current_shard()}" if ActiveRecord::Base.connection.respond_to?(:current_shard)
71
+ end
72
+
73
+ module Octopus::Migrator
74
+ def self.included(base)
75
+ base.send(:extend, ClassMethods)
76
+
77
+ base.class_eval do
78
+ class << self
79
+ alias_method_chain :migrate, :octopus
80
+ alias_method_chain :up, :octopus
81
+ alias_method_chain :down, :octopus
82
+ alias_method_chain :run, :octopus
83
+ end
84
+ end
85
+
86
+ base.alias_method_chain :run, :octopus
87
+ base.alias_method_chain :migrate, :octopus
88
+ base.alias_method_chain :migrations, :octopus
89
+ end
90
+
91
+ def run_with_octopus(&block)
92
+ run_without_octopus(&block)
93
+ rescue ActiveRecord::UnknownMigrationVersionError => e
94
+ raise unless migrations(true).find {|m| m.version == e.version}
95
+ end
96
+
97
+ def migrate_with_octopus(&block)
98
+ migrate_without_octopus(&block)
99
+ rescue ActiveRecord::UnknownMigrationVersionError => e
100
+ raise unless migrations(true).find {|m| m.version == e.version}
47
101
  end
48
102
 
103
+ def migrations_with_octopus(shard_agnostic = false)
104
+ connection = ActiveRecord::Base.connection
105
+ migrations = migrations_without_octopus
106
+ return migrations if !connection.is_a?(Octopus::Proxy) || shard_agnostic
49
107
 
50
- def migrate_with_octopus(direction)
51
- conn = ActiveRecord::Base.connection
52
- return migrate_without_octopus(direction) unless conn.is_a?(Octopus::Proxy)
53
- self.connection().current_shard = self.current_shard if self.current_shard != nil
54
-
55
- groups = conn.instance_variable_get(:@groups)
56
-
57
- begin
58
- if conn.current_group.is_a?(Array)
59
- conn.current_group.each { |group| conn.send_queries_to_multiple_shards(groups[group]) { migrate_without_octopus(direction) } }
60
- elsif conn.current_group.is_a?(Symbol)
61
- conn.send_queries_to_multiple_shards(groups[conn.current_group]) { migrate_without_octopus(direction) }
62
- elsif conn.current_shard.is_a?(Array)
63
- conn.send_queries_to_multiple_shards(conn.current_shard) { migrate_without_octopus(direction) }
64
- else
65
- migrate_without_octopus(direction)
108
+ migrations.select {|m| m.shards.include?(connection.current_shard.to_sym)}
109
+ end
110
+
111
+ module ClassMethods
112
+ def migrate_with_octopus(migrations_paths, target_version = nil, &block)
113
+ return migrate_without_octopus(migrations_paths, target_version, &block) unless connection.is_a?(Octopus::Proxy)
114
+
115
+ connection.send_queries_to_multiple_shards(connection.shard_names) do
116
+ migrate_without_octopus(migrations_paths, target_version, &block)
117
+ end
118
+ end
119
+
120
+ def up_with_octopus(migrations_paths, target_version = nil, &block)
121
+ return up_without_octopus(migrations_paths, target_version, &block) unless connection.is_a?(Octopus::Proxy)
122
+ return up_without_octopus(migrations_paths, target_version, &block) unless connection.current_shard == :master
123
+
124
+ connection.send_queries_to_multiple_shards(connection.shard_names) do
125
+ up_without_octopus(migrations_paths, target_version, &block)
126
+ end
127
+ end
128
+
129
+ def down_with_octopus(migrations_paths, target_version = nil, &block)
130
+ return down_without_octopus(migrations_paths, target_version, &block) unless connection.is_a?(Octopus::Proxy)
131
+ return down_without_octopus(migrations_paths, target_version, &block) unless connection.current_shard == :master
132
+
133
+ connection.send_queries_to_multiple_shards(connection.shard_names) do
134
+ down_without_octopus(migrations_paths, target_version, &block)
135
+ end
136
+ end
137
+
138
+ def run_with_octopus(direction, migrations_paths, target_version)
139
+ return run_without_octopus(direction, migrations_paths, target_version) unless connection.is_a?(Octopus::Proxy)
140
+
141
+ connection.send_queries_to_multiple_shards(connection.shard_names) do
142
+ run_without_octopus(direction, migrations_paths, target_version)
66
143
  end
67
- ensure
68
- conn.clean_proxy()
69
144
  end
145
+
146
+ private
147
+ def connection
148
+ ActiveRecord::Base.connection
149
+ end
150
+ end
151
+ end
152
+
153
+ module Octopus::MigrationProxy
154
+ def shards
155
+ if Octopus.rails31? || Octopus.rails32?
156
+ migration.class.shards
157
+ else
158
+ migration.shards
159
+ end
160
+ end
161
+ end
162
+
163
+ module Octopus::UnknownMigrationVersionError
164
+ def self.included(base)
165
+ base.alias_method_chain :initialize, :octopus
166
+ base.send(:attr_accessor, :version)
167
+ end
168
+
169
+ def initialize_with_octopus(version)
170
+ @version = version
171
+ initialize_without_octopus(version)
70
172
  end
71
173
  end
72
174
 
73
- ActiveRecord::Migration.extend(Octopus::Migration)
175
+ ActiveRecord::Migration.send(:include, Octopus::Migration)
176
+ ActiveRecord::Migrator.send(:include, Octopus::Migrator)
177
+ ActiveRecord::MigrationProxy.send(:include, Octopus::MigrationProxy)
178
+ ActiveRecord::UnknownMigrationVersionError.send(:include, Octopus::UnknownMigrationVersionError)
@@ -1,15 +1,19 @@
1
- module Octopus::Model
2
- def self.extended(base)
1
+ require 'active_support/deprecation'
2
+
3
+ module Octopus::Model
4
+ def self.extended(base)
3
5
  base.send(:include, InstanceMethods)
4
6
  base.extend(ClassMethods)
5
7
  base.hijack_connection()
8
+ base.hijack_initializer()
6
9
  end
7
10
 
8
11
  module SharedMethods
9
12
  def clean_table_name
10
13
  return unless self.connection_proxy.should_clean_table_name?
11
- if self != ActiveRecord::Base && self.respond_to?(:reset_table_name) && !self.read_inheritable_attribute(:set_table_name)
12
- self.reset_table_name()
14
+
15
+ if self != ActiveRecord::Base && self.respond_to?(:reset_table_name) && !self.custom_octopus_table_name
16
+ self.reset_table_name()
13
17
  end
14
18
 
15
19
  if Octopus.rails3?
@@ -19,14 +23,12 @@ module Octopus::Model
19
23
  end
20
24
 
21
25
  def using(shard)
22
- return self if defined?(::Rails) && !Octopus.environments.include?(Rails.env.to_s)
23
-
24
- hijack_initializer() if !respond_to?(:set_current_shard)
25
- clean_table_name()
26
-
27
- self.connection_proxy.using_enabled = true
28
-
29
- return Octopus::ScopeProxy.new(shard, self)
26
+ if Octopus.enabled?
27
+ clean_table_name
28
+ Octopus::ScopeProxy.new(shard, self)
29
+ else
30
+ self
31
+ end
30
32
  end
31
33
 
32
34
  def hijack_initializer()
@@ -34,10 +36,12 @@ module Octopus::Model
34
36
  before_save :reload_connection
35
37
 
36
38
  def set_current_shard
39
+ return unless Octopus.enabled?
40
+
37
41
  if new_record? || self.class.connection_proxy.block
38
- self.current_shard = self.class.connection_proxy.current_shard
42
+ self.current_shard = self.class.connection_proxy.current_shard
39
43
  else
40
- self.current_shard = self.class.connection_proxy.last_current_shard
44
+ self.current_shard = self.class.connection_proxy.last_current_shard || self.class.connection_proxy.current_shard
41
45
  end
42
46
  end
43
47
 
@@ -52,22 +56,33 @@ module Octopus::Model
52
56
 
53
57
  def hijack_connection()
54
58
  def self.should_use_normal_connection?
55
- (defined?(Rails) && Octopus.config() && !Octopus.environments.include?(Rails.env.to_s)) || self.read_inheritable_attribute(:establish_connection)
59
+ !Octopus.enabled? || self.custom_octopus_connection
56
60
  end
57
-
61
+
58
62
  def self.connection_proxy
59
- Thread.current[:connection_proxy] ||= Octopus::Proxy.new(Octopus.config())
63
+ @@connection_proxy ||= Octopus::Proxy.new
60
64
  end
61
65
 
62
- def self.connection_with_octopus()
63
- return connection_without_octopus() if should_use_normal_connection?
66
+ def self.connection_with_octopus
67
+ if should_use_normal_connection?
68
+ connection_without_octopus
69
+ else
70
+ self.connection_proxy.current_model = self
71
+ self.connection_proxy
72
+ end
73
+ end
64
74
 
65
- self.connection_proxy().current_model = self
66
- self.connection_proxy()
75
+ def self.connection_pool_with_octopus
76
+ if should_use_normal_connection?
77
+ connection_pool_without_octopus
78
+ else
79
+ connection_proxy.connection_pool
80
+ end
67
81
  end
68
82
 
69
83
  class << self
70
84
  alias_method_chain :connection, :octopus
85
+ alias_method_chain :connection_pool, :octopus
71
86
  end
72
87
  end
73
88
  end
@@ -75,36 +90,85 @@ module Octopus::Model
75
90
  module InstanceMethods
76
91
  include SharedMethods
77
92
 
93
+ def self.included(base)
94
+ base.send(:alias_method, :equality_without_octopus, :==)
95
+ base.send(:alias_method, :==, :equality_with_octopus)
96
+ base.send(:alias_method, :eql?, :==)
97
+ end
98
+
78
99
  def should_set_current_shard?
79
100
  self.respond_to?(:current_shard) && !self.current_shard.nil?
80
101
  end
81
102
 
103
+ def reload_connection_safe(&block)
104
+ return yield unless should_set_current_shard?
105
+ original = self.class.connection_proxy.current_shard
106
+ self.class.connection_proxy.current_shard = self.current_shard
107
+ result = yield
108
+ self.class.connection_proxy.current_shard = original
109
+ result
110
+ end
111
+
82
112
  def reload_connection()
83
- self.class.connection_proxy.current_shard = self.current_shard() if should_set_current_shard?
113
+ return unless should_set_current_shard?
114
+ self.class.connection_proxy.current_shard = self.current_shard
115
+ end
116
+
117
+ def equality_with_octopus(comparison_object)
118
+ equality_without_octopus(comparison_object) && comparison_object.current_shard == current_shard
84
119
  end
85
120
  end
86
121
 
87
122
  module ClassMethods
88
123
  include SharedMethods
89
124
 
90
- def replicated_model()
91
- write_inheritable_attribute(:replicated, true)
125
+ def self.extended(base)
126
+ base.class_attribute(:replicated)
127
+ base.class_attribute(:sharded)
128
+ base.hijack_methods
92
129
  end
93
-
94
- def sharded_model()
95
- write_inheritable_attribute(:sharded, true)
96
- end
97
-
98
- def octopus_establish_connection(spec = nil)
99
- write_inheritable_attribute(:establish_connection, true)
130
+
131
+ def replicated_model
132
+ self.replicated = true
133
+ end
134
+
135
+ def sharded_model
136
+ self.sharded = true
137
+ end
138
+
139
+ def hijack_methods
140
+ class << self
141
+ attr_accessor :custom_octopus_connection
142
+ attr_accessor :custom_octopus_table_name
143
+
144
+ alias_method_chain(:set_table_name, :octopus)
145
+
146
+ if Octopus.rails32?
147
+ def table_name=(value = nil)
148
+ self.custom_octopus_table_name = true
149
+ super
150
+ end
151
+ end
152
+ end
153
+ end
154
+
155
+ def set_table_name_with_octopus(value = nil, &block)
156
+ self.custom_octopus_table_name = true
157
+ set_table_name_without_octopus(value, &block)
158
+ end
159
+
160
+ def octopus_establish_connection(spec = ENV['DATABASE_URL'])
161
+ self.custom_octopus_connection = true if spec
100
162
  establish_connection(spec)
101
163
  end
102
164
 
103
- def octopus_set_table_name(value = nil, &block)
104
- write_inheritable_attribute(:set_table_name, true)
105
- set_table_name(value, &block)
165
+ def octopus_set_table_name(value = nil)
166
+ ActiveSupport::Deprecation.warn "Calling `octopus_set_table_name` is deprecated and will be removed in Octopus 1.0.", caller
167
+ set_table_name(value)
106
168
  end
107
169
  end
108
170
  end
109
171
 
110
172
  ActiveRecord::Base.extend(Octopus::Model)
173
+
174
+ class OctopusModel < ActiveRecord::Base; end;