ar-octopus 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +11 -0
- data/.travis.yml +22 -0
- data/Appraisals +18 -0
- data/Gemfile +3 -12
- data/README.mkdn +63 -24
- data/Rakefile +70 -92
- data/ar-octopus.gemspec +25 -198
- data/lib/ar-octopus.rb +1 -0
- data/lib/octopus.rb +73 -25
- data/lib/octopus/association.rb +6 -5
- data/lib/octopus/association_collection.rb +58 -4
- data/lib/octopus/has_and_belongs_to_many_association.rb +4 -4
- data/lib/octopus/logger.rb +9 -4
- data/lib/octopus/migration.rb +155 -50
- data/lib/octopus/model.rb +98 -34
- data/lib/octopus/proxy.rb +124 -53
- data/lib/octopus/rails2/association.rb +46 -93
- data/lib/octopus/rails2/persistence.rb +1 -1
- data/lib/octopus/rails2/scope.rb +17 -0
- data/lib/octopus/rails3.1/singular_association.rb +34 -0
- data/lib/octopus/rails3.2/persistence.rb +12 -0
- data/lib/octopus/rails3/abstract_adapter.rb +39 -0
- data/lib/octopus/rails3/arel.rb +5 -5
- data/lib/octopus/rails3/log_subscriber.rb +22 -0
- data/lib/octopus/rails3/persistence.rb +10 -5
- data/lib/octopus/railtie.rb +13 -0
- data/lib/octopus/scope_proxy.rb +22 -16
- data/lib/octopus/version.rb +3 -0
- data/lib/tasks/octopus.rake +20 -0
- data/sample_app/Gemfile +2 -2
- data/sample_app/config/initializers/inflections.rb +1 -1
- data/sample_app/config/initializers/secret_token.rb +1 -1
- data/sample_app/db/migrate/20100720172730_create_items.rb +1 -1
- data/sample_app/db/migrate/20100720210335_create_sample_users.rb +1 -1
- data/sample_app/db/seeds.rb +1 -1
- data/sample_app/features/migrate.feature +12 -12
- data/sample_app/features/seed.feature +3 -3
- data/sample_app/features/step_definitions/web_steps.rb +5 -5
- data/sample_app/features/support/env.rb +8 -8
- data/sample_app/lib/tasks/cucumber.rake +2 -2
- data/sample_app/public/javascripts/effects.js +1 -1
- data/spec/config/shards.yml +38 -28
- data/spec/migrations/11_add_field_in_all_slaves.rb +1 -1
- data/spec/migrations/12_create_users_using_block.rb +2 -2
- data/spec/migrations/13_create_users_using_block_and_using.rb +2 -2
- data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +11 -0
- data/spec/migrations/1_create_users_on_master.rb +1 -1
- data/spec/migrations/2_create_users_on_canada.rb +1 -1
- data/spec/migrations/3_create_users_on_both_shards.rb +1 -1
- data/spec/migrations/4_create_users_on_shards_of_a_group.rb +1 -1
- data/spec/migrations/5_create_users_on_multiples_groups.rb +1 -1
- data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +1 -1
- data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +1 -1
- data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +1 -1
- data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +1 -1
- data/spec/octopus/association_spec.rb +88 -70
- data/spec/octopus/log_subscriber_spec.rb +22 -0
- data/spec/octopus/logger_spec.rb +28 -15
- data/spec/octopus/migration_spec.rb +47 -43
- data/spec/octopus/model_spec.rb +179 -13
- data/spec/octopus/octopus_spec.rb +26 -4
- data/spec/octopus/proxy_spec.rb +61 -23
- data/spec/octopus/{replication_specs.rb → replication_spec.rb} +33 -26
- data/spec/octopus/scope_proxy_spec.rb +3 -3
- data/spec/octopus/sharded_spec.rb +9 -9
- data/spec/spec_helper.rb +10 -12
- data/spec/support/active_record/connection_adapters/modify_config_adapter.rb +17 -0
- data/spec/support/database_connection.rb +2 -0
- data/spec/{database_models.rb → support/database_models.rb} +27 -2
- data/spec/support/octopus_helper.rb +50 -0
- data/spec/tasks/octopus.rake_spec.rb +36 -0
- metadata +188 -169
- data/Gemfile.lock +0 -68
- data/lib/octopus/rails3/association.rb +0 -112
- data/spec/database_connection.rb +0 -4
- data/spec/octopus/controller_spec.rb +0 -34
- data/spec/octopus_helper.rb +0 -37
@@ -4,12 +4,12 @@ module Octopus::HasAndBelongsToManyAssociation
|
|
4
4
|
alias_method_chain :insert_record, :octopus
|
5
5
|
end
|
6
6
|
end
|
7
|
-
|
7
|
+
|
8
8
|
def insert_record_with_octopus(record, force = true, validate = true)
|
9
9
|
if should_wrap_the_connection?
|
10
|
-
Octopus.using(@owner.current_shard) { insert_record_without_octopus(record, force, validate) }
|
11
|
-
else
|
12
|
-
insert_record_without_octopus(record, force, validate)
|
10
|
+
Octopus.using(@owner.current_shard) { insert_record_without_octopus(record, force, validate) }
|
11
|
+
else
|
12
|
+
insert_record_without_octopus(record, force, validate)
|
13
13
|
end
|
14
14
|
end
|
15
15
|
end
|
data/lib/octopus/logger.rb
CHANGED
@@ -1,13 +1,18 @@
|
|
1
1
|
require "logger"
|
2
2
|
|
3
3
|
class Octopus::Logger < Logger
|
4
|
+
def initialize(logdev, shift_age = 0, shift_size = 1048576)
|
5
|
+
ActiveSupport::Deprecation.warn "Octopus::Logger is deprecated and will be removed in Octopus 0.6.x", caller
|
6
|
+
super
|
7
|
+
end
|
8
|
+
|
4
9
|
def format_message(severity, timestamp, progname, msg)
|
5
|
-
str = super
|
6
|
-
|
10
|
+
str = super
|
11
|
+
|
7
12
|
if ActiveRecord::Base.connection.respond_to?(:current_shard)
|
8
|
-
str += "Shard: #{ActiveRecord::Base.connection.current_shard} -"
|
13
|
+
str += "Shard: #{ActiveRecord::Base.connection.current_shard} -"
|
9
14
|
end
|
10
|
-
|
15
|
+
|
11
16
|
str
|
12
17
|
end
|
13
18
|
end
|
data/lib/octopus/migration.rb
CHANGED
@@ -1,73 +1,178 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
1
|
+
require "set"
|
2
|
+
require "active_support/core_ext/module/aliasing"
|
3
|
+
|
4
|
+
if Octopus.rails2?
|
5
|
+
require "active_support/core_ext/array/wrapper"
|
6
|
+
else
|
7
|
+
require "active_support/core_ext/array/wrap"
|
8
|
+
end
|
9
|
+
|
10
|
+
module Octopus::Migration
|
11
|
+
module InstanceOrClassMethods
|
12
|
+
def announce_with_octopus(message)
|
13
|
+
announce_without_octopus("#{message} - #{get_current_shard}")
|
14
|
+
end
|
15
|
+
|
16
|
+
def get_current_shard
|
17
|
+
"Shard: #{connection.current_shard}" if connection.respond_to?(:current_shard)
|
11
18
|
end
|
12
19
|
end
|
13
20
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
21
|
+
include InstanceOrClassMethods if Octopus.rails31? || Octopus.rails32?
|
22
|
+
|
23
|
+
def self.included(base)
|
24
|
+
base.send(:extend, ClassMethods)
|
25
|
+
|
26
|
+
if Octopus.rails31? || Octopus.rails32?
|
27
|
+
base.alias_method_chain :announce, :octopus
|
28
|
+
else
|
29
|
+
base.class_eval do
|
30
|
+
class << self
|
31
|
+
alias_method_chain :announce, :octopus
|
32
|
+
end
|
18
33
|
end
|
34
|
+
end
|
35
|
+
|
36
|
+
base.class_attribute :current_shard, :current_group, :instance_reader => false, :instance_writer => false
|
37
|
+
end
|
38
|
+
|
39
|
+
module ClassMethods
|
40
|
+
include InstanceOrClassMethods unless Octopus.rails31? || Octopus.rails32?
|
41
|
+
|
42
|
+
def using(*args)
|
43
|
+
return self unless connection.is_a?(Octopus::Proxy)
|
19
44
|
|
20
|
-
self.connection().block = true
|
21
45
|
self.current_shard = args
|
22
|
-
self
|
46
|
+
self
|
23
47
|
end
|
24
48
|
|
25
|
-
|
26
|
-
|
49
|
+
def using_group(*groups)
|
50
|
+
return self unless connection.is_a?(Octopus::Proxy)
|
51
|
+
|
52
|
+
self.current_group = groups
|
53
|
+
self
|
54
|
+
end
|
27
55
|
|
28
|
-
|
29
|
-
|
30
|
-
args.each do |group_shard|
|
31
|
-
shards = self.connection().instance_variable_get(:@groups)[group_shard] || []
|
56
|
+
def shards
|
57
|
+
shards = Set.new
|
32
58
|
|
33
|
-
|
34
|
-
|
59
|
+
if groups = current_group
|
60
|
+
Array.wrap(groups).each do |group|
|
61
|
+
group_shards = connection.shards_for_group(group)
|
62
|
+
shards.merge(group_shards) if group_shards
|
35
63
|
end
|
64
|
+
elsif shard = current_shard
|
65
|
+
shards.merge(Array.wrap(shard))
|
36
66
|
end
|
37
67
|
|
38
|
-
|
39
|
-
self.connection().current_group = args
|
68
|
+
shards.to_a.presence || [:master]
|
40
69
|
end
|
41
|
-
|
42
|
-
return self
|
43
70
|
end
|
44
|
-
|
45
|
-
|
46
|
-
|
71
|
+
end
|
72
|
+
|
73
|
+
module Octopus::Migrator
|
74
|
+
def self.included(base)
|
75
|
+
base.send(:extend, ClassMethods)
|
76
|
+
|
77
|
+
base.class_eval do
|
78
|
+
class << self
|
79
|
+
alias_method_chain :migrate, :octopus
|
80
|
+
alias_method_chain :up, :octopus
|
81
|
+
alias_method_chain :down, :octopus
|
82
|
+
alias_method_chain :run, :octopus
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
base.alias_method_chain :run, :octopus
|
87
|
+
base.alias_method_chain :migrate, :octopus
|
88
|
+
base.alias_method_chain :migrations, :octopus
|
89
|
+
end
|
90
|
+
|
91
|
+
def run_with_octopus(&block)
|
92
|
+
run_without_octopus(&block)
|
93
|
+
rescue ActiveRecord::UnknownMigrationVersionError => e
|
94
|
+
raise unless migrations(true).find {|m| m.version == e.version}
|
95
|
+
end
|
96
|
+
|
97
|
+
def migrate_with_octopus(&block)
|
98
|
+
migrate_without_octopus(&block)
|
99
|
+
rescue ActiveRecord::UnknownMigrationVersionError => e
|
100
|
+
raise unless migrations(true).find {|m| m.version == e.version}
|
47
101
|
end
|
48
102
|
|
103
|
+
def migrations_with_octopus(shard_agnostic = false)
|
104
|
+
connection = ActiveRecord::Base.connection
|
105
|
+
migrations = migrations_without_octopus
|
106
|
+
return migrations if !connection.is_a?(Octopus::Proxy) || shard_agnostic
|
49
107
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
108
|
+
migrations.select {|m| m.shards.include?(connection.current_shard.to_sym)}
|
109
|
+
end
|
110
|
+
|
111
|
+
module ClassMethods
|
112
|
+
def migrate_with_octopus(migrations_paths, target_version = nil, &block)
|
113
|
+
return migrate_without_octopus(migrations_paths, target_version, &block) unless connection.is_a?(Octopus::Proxy)
|
114
|
+
|
115
|
+
connection.send_queries_to_multiple_shards(connection.shard_names) do
|
116
|
+
migrate_without_octopus(migrations_paths, target_version, &block)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def up_with_octopus(migrations_paths, target_version = nil, &block)
|
121
|
+
return up_without_octopus(migrations_paths, target_version, &block) unless connection.is_a?(Octopus::Proxy)
|
122
|
+
return up_without_octopus(migrations_paths, target_version, &block) unless connection.current_shard == :master
|
123
|
+
|
124
|
+
connection.send_queries_to_multiple_shards(connection.shard_names) do
|
125
|
+
up_without_octopus(migrations_paths, target_version, &block)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def down_with_octopus(migrations_paths, target_version = nil, &block)
|
130
|
+
return down_without_octopus(migrations_paths, target_version, &block) unless connection.is_a?(Octopus::Proxy)
|
131
|
+
return down_without_octopus(migrations_paths, target_version, &block) unless connection.current_shard == :master
|
132
|
+
|
133
|
+
connection.send_queries_to_multiple_shards(connection.shard_names) do
|
134
|
+
down_without_octopus(migrations_paths, target_version, &block)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def run_with_octopus(direction, migrations_paths, target_version)
|
139
|
+
return run_without_octopus(direction, migrations_paths, target_version) unless connection.is_a?(Octopus::Proxy)
|
140
|
+
|
141
|
+
connection.send_queries_to_multiple_shards(connection.shard_names) do
|
142
|
+
run_without_octopus(direction, migrations_paths, target_version)
|
66
143
|
end
|
67
|
-
ensure
|
68
|
-
conn.clean_proxy()
|
69
144
|
end
|
145
|
+
|
146
|
+
private
|
147
|
+
def connection
|
148
|
+
ActiveRecord::Base.connection
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
module Octopus::MigrationProxy
|
154
|
+
def shards
|
155
|
+
if Octopus.rails31? || Octopus.rails32?
|
156
|
+
migration.class.shards
|
157
|
+
else
|
158
|
+
migration.shards
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
module Octopus::UnknownMigrationVersionError
|
164
|
+
def self.included(base)
|
165
|
+
base.alias_method_chain :initialize, :octopus
|
166
|
+
base.send(:attr_accessor, :version)
|
167
|
+
end
|
168
|
+
|
169
|
+
def initialize_with_octopus(version)
|
170
|
+
@version = version
|
171
|
+
initialize_without_octopus(version)
|
70
172
|
end
|
71
173
|
end
|
72
174
|
|
73
|
-
ActiveRecord::Migration.
|
175
|
+
ActiveRecord::Migration.send(:include, Octopus::Migration)
|
176
|
+
ActiveRecord::Migrator.send(:include, Octopus::Migrator)
|
177
|
+
ActiveRecord::MigrationProxy.send(:include, Octopus::MigrationProxy)
|
178
|
+
ActiveRecord::UnknownMigrationVersionError.send(:include, Octopus::UnknownMigrationVersionError)
|
data/lib/octopus/model.rb
CHANGED
@@ -1,15 +1,19 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'active_support/deprecation'
|
2
|
+
|
3
|
+
module Octopus::Model
|
4
|
+
def self.extended(base)
|
3
5
|
base.send(:include, InstanceMethods)
|
4
6
|
base.extend(ClassMethods)
|
5
7
|
base.hijack_connection()
|
8
|
+
base.hijack_initializer()
|
6
9
|
end
|
7
10
|
|
8
11
|
module SharedMethods
|
9
12
|
def clean_table_name
|
10
13
|
return unless self.connection_proxy.should_clean_table_name?
|
11
|
-
|
12
|
-
|
14
|
+
|
15
|
+
if self != ActiveRecord::Base && self.respond_to?(:reset_table_name) && !self.custom_octopus_table_name
|
16
|
+
self.reset_table_name()
|
13
17
|
end
|
14
18
|
|
15
19
|
if Octopus.rails3?
|
@@ -19,14 +23,12 @@ module Octopus::Model
|
|
19
23
|
end
|
20
24
|
|
21
25
|
def using(shard)
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
return Octopus::ScopeProxy.new(shard, self)
|
26
|
+
if Octopus.enabled?
|
27
|
+
clean_table_name
|
28
|
+
Octopus::ScopeProxy.new(shard, self)
|
29
|
+
else
|
30
|
+
self
|
31
|
+
end
|
30
32
|
end
|
31
33
|
|
32
34
|
def hijack_initializer()
|
@@ -34,10 +36,12 @@ module Octopus::Model
|
|
34
36
|
before_save :reload_connection
|
35
37
|
|
36
38
|
def set_current_shard
|
39
|
+
return unless Octopus.enabled?
|
40
|
+
|
37
41
|
if new_record? || self.class.connection_proxy.block
|
38
|
-
self.current_shard = self.class.connection_proxy.current_shard
|
42
|
+
self.current_shard = self.class.connection_proxy.current_shard
|
39
43
|
else
|
40
|
-
self.current_shard = self.class.connection_proxy.last_current_shard
|
44
|
+
self.current_shard = self.class.connection_proxy.last_current_shard || self.class.connection_proxy.current_shard
|
41
45
|
end
|
42
46
|
end
|
43
47
|
|
@@ -52,22 +56,33 @@ module Octopus::Model
|
|
52
56
|
|
53
57
|
def hijack_connection()
|
54
58
|
def self.should_use_normal_connection?
|
55
|
-
|
59
|
+
!Octopus.enabled? || self.custom_octopus_connection
|
56
60
|
end
|
57
|
-
|
61
|
+
|
58
62
|
def self.connection_proxy
|
59
|
-
|
63
|
+
@@connection_proxy ||= Octopus::Proxy.new
|
60
64
|
end
|
61
65
|
|
62
|
-
def self.connection_with_octopus
|
63
|
-
|
66
|
+
def self.connection_with_octopus
|
67
|
+
if should_use_normal_connection?
|
68
|
+
connection_without_octopus
|
69
|
+
else
|
70
|
+
self.connection_proxy.current_model = self
|
71
|
+
self.connection_proxy
|
72
|
+
end
|
73
|
+
end
|
64
74
|
|
65
|
-
|
66
|
-
|
75
|
+
def self.connection_pool_with_octopus
|
76
|
+
if should_use_normal_connection?
|
77
|
+
connection_pool_without_octopus
|
78
|
+
else
|
79
|
+
connection_proxy.connection_pool
|
80
|
+
end
|
67
81
|
end
|
68
82
|
|
69
83
|
class << self
|
70
84
|
alias_method_chain :connection, :octopus
|
85
|
+
alias_method_chain :connection_pool, :octopus
|
71
86
|
end
|
72
87
|
end
|
73
88
|
end
|
@@ -75,36 +90,85 @@ module Octopus::Model
|
|
75
90
|
module InstanceMethods
|
76
91
|
include SharedMethods
|
77
92
|
|
93
|
+
def self.included(base)
|
94
|
+
base.send(:alias_method, :equality_without_octopus, :==)
|
95
|
+
base.send(:alias_method, :==, :equality_with_octopus)
|
96
|
+
base.send(:alias_method, :eql?, :==)
|
97
|
+
end
|
98
|
+
|
78
99
|
def should_set_current_shard?
|
79
100
|
self.respond_to?(:current_shard) && !self.current_shard.nil?
|
80
101
|
end
|
81
102
|
|
103
|
+
def reload_connection_safe(&block)
|
104
|
+
return yield unless should_set_current_shard?
|
105
|
+
original = self.class.connection_proxy.current_shard
|
106
|
+
self.class.connection_proxy.current_shard = self.current_shard
|
107
|
+
result = yield
|
108
|
+
self.class.connection_proxy.current_shard = original
|
109
|
+
result
|
110
|
+
end
|
111
|
+
|
82
112
|
def reload_connection()
|
83
|
-
|
113
|
+
return unless should_set_current_shard?
|
114
|
+
self.class.connection_proxy.current_shard = self.current_shard
|
115
|
+
end
|
116
|
+
|
117
|
+
def equality_with_octopus(comparison_object)
|
118
|
+
equality_without_octopus(comparison_object) && comparison_object.current_shard == current_shard
|
84
119
|
end
|
85
120
|
end
|
86
121
|
|
87
122
|
module ClassMethods
|
88
123
|
include SharedMethods
|
89
124
|
|
90
|
-
def
|
91
|
-
|
125
|
+
def self.extended(base)
|
126
|
+
base.class_attribute(:replicated)
|
127
|
+
base.class_attribute(:sharded)
|
128
|
+
base.hijack_methods
|
92
129
|
end
|
93
|
-
|
94
|
-
def
|
95
|
-
|
96
|
-
end
|
97
|
-
|
98
|
-
def
|
99
|
-
|
130
|
+
|
131
|
+
def replicated_model
|
132
|
+
self.replicated = true
|
133
|
+
end
|
134
|
+
|
135
|
+
def sharded_model
|
136
|
+
self.sharded = true
|
137
|
+
end
|
138
|
+
|
139
|
+
def hijack_methods
|
140
|
+
class << self
|
141
|
+
attr_accessor :custom_octopus_connection
|
142
|
+
attr_accessor :custom_octopus_table_name
|
143
|
+
|
144
|
+
alias_method_chain(:set_table_name, :octopus)
|
145
|
+
|
146
|
+
if Octopus.rails32?
|
147
|
+
def table_name=(value = nil)
|
148
|
+
self.custom_octopus_table_name = true
|
149
|
+
super
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def set_table_name_with_octopus(value = nil, &block)
|
156
|
+
self.custom_octopus_table_name = true
|
157
|
+
set_table_name_without_octopus(value, &block)
|
158
|
+
end
|
159
|
+
|
160
|
+
def octopus_establish_connection(spec = ENV['DATABASE_URL'])
|
161
|
+
self.custom_octopus_connection = true if spec
|
100
162
|
establish_connection(spec)
|
101
163
|
end
|
102
164
|
|
103
|
-
def octopus_set_table_name(value = nil
|
104
|
-
|
105
|
-
set_table_name(value
|
165
|
+
def octopus_set_table_name(value = nil)
|
166
|
+
ActiveSupport::Deprecation.warn "Calling `octopus_set_table_name` is deprecated and will be removed in Octopus 1.0.", caller
|
167
|
+
set_table_name(value)
|
106
168
|
end
|
107
169
|
end
|
108
170
|
end
|
109
171
|
|
110
172
|
ActiveRecord::Base.extend(Octopus::Model)
|
173
|
+
|
174
|
+
class OctopusModel < ActiveRecord::Base; end;
|