ar-octopus 0.8.1 → 0.10.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +6 -14
- data/.gitignore +1 -0
- data/.rspec +1 -1
- data/.rubocop.yml +46 -0
- data/.rubocop_todo.yml +56 -0
- data/.travis.yml +7 -12
- data/Appraisals +11 -4
- data/Gemfile +1 -1
- data/README.mkdn +138 -63
- data/Rakefile +23 -16
- data/ar-octopus.gemspec +23 -20
- data/gemfiles/rails42.gemfile +7 -0
- data/gemfiles/{rails32.gemfile → rails5.gemfile} +2 -2
- data/gemfiles/{rails4.gemfile → rails51.gemfile} +2 -2
- data/gemfiles/rails52.gemfile +7 -0
- data/lib/ar-octopus.rb +1 -1
- data/lib/octopus/{rails3/abstract_adapter.rb → abstract_adapter.rb} +4 -15
- data/lib/octopus/association.rb +8 -99
- data/lib/octopus/association_shard_tracking.rb +74 -0
- data/lib/octopus/collection_association.rb +17 -0
- data/lib/octopus/collection_proxy.rb +16 -0
- data/lib/octopus/exception.rb +4 -0
- data/lib/octopus/finder_methods.rb +8 -0
- data/lib/octopus/load_balancing/round_robin.rb +20 -0
- data/lib/octopus/load_balancing.rb +4 -0
- data/lib/octopus/{rails3/log_subscriber.rb → log_subscriber.rb} +6 -2
- data/lib/octopus/migration.rb +187 -110
- data/lib/octopus/model.rb +151 -131
- data/lib/octopus/persistence.rb +45 -0
- data/lib/octopus/proxy.rb +297 -232
- data/lib/octopus/proxy_config.rb +251 -0
- data/lib/octopus/query_cache_for_shards.rb +24 -0
- data/lib/octopus/railtie.rb +1 -3
- data/lib/octopus/relation_proxy.rb +70 -0
- data/lib/octopus/result_patch.rb +19 -0
- data/lib/octopus/scope_proxy.rb +54 -36
- data/lib/octopus/shard_tracking/attribute.rb +22 -0
- data/lib/octopus/shard_tracking/dynamic.rb +11 -0
- data/lib/octopus/shard_tracking.rb +46 -0
- data/lib/octopus/singular_association.rb +9 -0
- data/lib/octopus/slave_group.rb +13 -0
- data/lib/octopus/version.rb +1 -1
- data/lib/octopus.rb +125 -33
- data/lib/tasks/octopus.rake +2 -2
- data/sample_app/Gemfile +3 -3
- data/sample_app/autotest/discover.rb +2 -2
- data/sample_app/config/application.rb +1 -1
- data/sample_app/config/boot.rb +1 -1
- data/sample_app/config/environments/test.rb +1 -1
- data/sample_app/config/initializers/session_store.rb +1 -1
- data/sample_app/config/initializers/wrap_parameters.rb +1 -1
- data/sample_app/config/routes.rb +1 -1
- data/sample_app/db/migrate/20100720210335_create_sample_users.rb +2 -2
- data/sample_app/db/schema.rb +10 -10
- data/sample_app/db/seeds.rb +3 -3
- data/sample_app/features/step_definitions/seeds_steps.rb +4 -4
- data/sample_app/features/step_definitions/web_steps.rb +3 -4
- data/sample_app/features/support/env.rb +3 -4
- data/sample_app/features/support/paths.rb +4 -4
- data/sample_app/lib/tasks/cucumber.rake +43 -44
- data/sample_app/spec/spec_helper.rb +3 -3
- data/spec/config/shards.yml +78 -0
- data/spec/migrations/10_create_users_using_replication.rb +4 -4
- data/spec/migrations/11_add_field_in_all_slaves.rb +4 -4
- data/spec/migrations/12_create_users_using_block.rb +8 -8
- data/spec/migrations/13_create_users_using_block_and_using.rb +5 -5
- data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +3 -3
- data/spec/migrations/15_create_user_on_shards_of_default_group_with_versions.rb +3 -3
- data/spec/migrations/1_create_users_on_master.rb +4 -4
- data/spec/migrations/2_create_users_on_canada.rb +4 -4
- data/spec/migrations/3_create_users_on_both_shards.rb +4 -4
- data/spec/migrations/4_create_users_on_shards_of_a_group.rb +4 -4
- data/spec/migrations/5_create_users_on_multiples_groups.rb +3 -3
- data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +4 -4
- data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +4 -4
- data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +4 -4
- data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +5 -5
- data/spec/octopus/association_shard_tracking_spec.rb +1036 -0
- data/spec/octopus/collection_proxy_spec.rb +16 -0
- data/spec/octopus/load_balancing/round_robin_spec.rb +15 -0
- data/spec/octopus/log_subscriber_spec.rb +5 -5
- data/spec/octopus/migration_spec.rb +83 -49
- data/spec/octopus/model_spec.rb +544 -292
- data/spec/octopus/octopus_spec.rb +64 -31
- data/spec/octopus/proxy_spec.rb +145 -141
- data/spec/octopus/query_cache_for_shards_spec.rb +40 -0
- data/spec/octopus/relation_proxy_spec.rb +132 -0
- data/spec/octopus/replicated_slave_grouped_spec.rb +91 -0
- data/spec/octopus/replication_spec.rb +140 -65
- data/spec/octopus/scope_proxy_spec.rb +90 -10
- data/spec/octopus/sharded_replicated_slave_grouped_spec.rb +55 -0
- data/spec/octopus/sharded_spec.rb +10 -10
- data/spec/spec_helper.rb +8 -6
- data/spec/support/active_record/connection_adapters/modify_config_adapter.rb +1 -3
- data/spec/support/database_connection.rb +2 -2
- data/spec/support/database_models.rb +18 -17
- data/spec/support/octopus_helper.rb +32 -25
- data/spec/support/query_count.rb +1 -3
- data/spec/support/shared_contexts.rb +3 -3
- data/spec/tasks/octopus.rake_spec.rb +10 -10
- metadata +112 -70
- data/.ruby-version +0 -1
- data/init.rb +0 -1
- data/lib/octopus/association_collection.rb +0 -49
- data/lib/octopus/has_and_belongs_to_many_association.rb +0 -17
- data/lib/octopus/rails3/persistence.rb +0 -39
- data/lib/octopus/rails3/singular_association.rb +0 -34
- data/rails/init.rb +0 -1
- data/spec/octopus/association_spec.rb +0 -712
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Octopus::CollectionProxy do
|
4
|
+
describe 'method dispatch' do
|
5
|
+
before :each do
|
6
|
+
@client = Client.using(:canada).create!
|
7
|
+
@client.items << Item.using(:canada).create!
|
8
|
+
end
|
9
|
+
|
10
|
+
it 'computes the size of the collection without loading it' do
|
11
|
+
expect(@client.items.size).to eq(1)
|
12
|
+
|
13
|
+
expect(@client.items.loaded?).to be false
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Octopus::LoadBalancing::RoundRobin do
|
4
|
+
it "raises an error when no shards are given" do
|
5
|
+
expect do
|
6
|
+
Octopus::LoadBalancing::RoundRobin.new([])
|
7
|
+
end.to raise_error Octopus::Exception
|
8
|
+
end
|
9
|
+
|
10
|
+
it "does not raise an error if slaves given" do
|
11
|
+
expect do
|
12
|
+
Octopus::LoadBalancing::RoundRobin.new([:stub])
|
13
|
+
end.to_not raise_error Octopus::Exception
|
14
|
+
end
|
15
|
+
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'spec_helper'
|
2
2
|
|
3
3
|
describe Octopus::LogSubscriber, :shards => [:canada] do
|
4
4
|
before :each do
|
@@ -9,11 +9,11 @@ describe Octopus::LogSubscriber, :shards => [:canada] do
|
|
9
9
|
end
|
10
10
|
|
11
11
|
after :each do
|
12
|
-
ActiveRecord::Base.logger =
|
12
|
+
ActiveRecord::Base.logger = Logger.new(File.open('database.log', 'a'))
|
13
13
|
end
|
14
14
|
|
15
|
-
it
|
16
|
-
User.using(:canada).create!(:name =>
|
17
|
-
@out.string.
|
15
|
+
it 'should add to the default logger the shard name the query was sent to' do
|
16
|
+
User.using(:canada).create!(:name => 'test')
|
17
|
+
expect(@out.string).to match(/Shard: canada/)
|
18
18
|
end
|
19
19
|
end
|
@@ -1,115 +1,149 @@
|
|
1
|
-
require
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
def get_all_versions
|
4
|
+
if Octopus.atleast_rails52?
|
5
|
+
migrations_root = File.expand_path(File.join(File.dirname(__FILE__), '..', 'migrations'))
|
6
|
+
ActiveRecord::MigrationContext.new(migrations_root).get_all_versions
|
7
|
+
else
|
8
|
+
ActiveRecord::Migrator.get_all_versions
|
9
|
+
end
|
10
|
+
end
|
2
11
|
|
3
12
|
describe Octopus::Migration do
|
4
|
-
it
|
13
|
+
it 'should run just in the master shard' do
|
5
14
|
OctopusHelper.migrating_to_version 1 do
|
6
|
-
User.using(:master).find_by_name(
|
7
|
-
User.using(:canada).find_by_name(
|
15
|
+
expect(User.using(:master).find_by_name('Master')).not_to be_nil
|
16
|
+
expect(User.using(:canada).find_by_name('Master')).to be_nil
|
8
17
|
end
|
9
18
|
end
|
10
19
|
|
11
|
-
it
|
20
|
+
it 'should run on specific shard' do
|
12
21
|
OctopusHelper.migrating_to_version 2 do
|
13
|
-
User.using(:master).find_by_name(
|
14
|
-
User.using(:canada).find_by_name(
|
22
|
+
expect(User.using(:master).find_by_name('Sharding')).to be_nil
|
23
|
+
expect(User.using(:canada).find_by_name('Sharding')).not_to be_nil
|
15
24
|
end
|
16
25
|
end
|
17
26
|
|
18
|
-
it
|
27
|
+
it 'should run on specifieds shards' do
|
19
28
|
OctopusHelper.migrating_to_version 3 do
|
20
|
-
User.using(:brazil).find_by_name(
|
21
|
-
User.using(:canada).find_by_name(
|
29
|
+
expect(User.using(:brazil).find_by_name('Both')).not_to be_nil
|
30
|
+
expect(User.using(:canada).find_by_name('Both')).not_to be_nil
|
22
31
|
end
|
23
32
|
end
|
24
33
|
|
25
|
-
it
|
34
|
+
it 'should run on specified group' do
|
26
35
|
OctopusHelper.migrating_to_version 4 do
|
27
|
-
User.using(:canada).find_by_name(
|
28
|
-
User.using(:brazil).find_by_name(
|
29
|
-
User.using(:russia).find_by_name(
|
36
|
+
expect(User.using(:canada).find_by_name('Group')).not_to be_nil
|
37
|
+
expect(User.using(:brazil).find_by_name('Group')).not_to be_nil
|
38
|
+
expect(User.using(:russia).find_by_name('Group')).not_to be_nil
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should rollback correctly migrations" do
|
43
|
+
migrations_root = File.expand_path(File.join(File.dirname(__FILE__), '..', 'migrations'))
|
44
|
+
if Octopus.atleast_rails52?
|
45
|
+
OctopusHelper.migrate_to_version(:up, migrations_root, 4)
|
46
|
+
else
|
47
|
+
ActiveRecord::Migrator.run(:up, migrations_root, 4)
|
48
|
+
end
|
49
|
+
|
50
|
+
expect(User.using(:canada).find_by_name('Group')).not_to be_nil
|
51
|
+
expect(User.using(:brazil).find_by_name('Group')).not_to be_nil
|
52
|
+
expect(User.using(:russia).find_by_name('Group')).not_to be_nil
|
53
|
+
|
54
|
+
|
55
|
+
Octopus.using(:canada) do
|
56
|
+
if Octopus.atleast_rails52?
|
57
|
+
OctopusHelper.migrate_to_version(:down, migrations_root, 4)
|
58
|
+
else
|
59
|
+
ActiveRecord::Migrator.rollback(migrations_root, 4)
|
60
|
+
end
|
30
61
|
end
|
62
|
+
|
63
|
+
expect(User.using(:canada).find_by_name('Group')).to be_nil
|
64
|
+
expect(User.using(:brazil).find_by_name('Group')).to be_nil
|
65
|
+
expect(User.using(:russia).find_by_name('Group')).to be_nil
|
31
66
|
end
|
32
67
|
|
33
|
-
it
|
68
|
+
it 'should run once per shard' do
|
34
69
|
OctopusHelper.migrating_to_version 5 do
|
35
|
-
User.using(:canada).where(:name => 'MultipleGroup').size.
|
36
|
-
User.using(:brazil).where(:name => 'MultipleGroup').size.
|
37
|
-
User.using(:russia).where(:name => 'MultipleGroup').size.
|
70
|
+
expect(User.using(:canada).where(:name => 'MultipleGroup').size).to eq(1)
|
71
|
+
expect(User.using(:brazil).where(:name => 'MultipleGroup').size).to eq(1)
|
72
|
+
expect(User.using(:russia).where(:name => 'MultipleGroup').size).to eq(1)
|
38
73
|
end
|
39
74
|
end
|
40
75
|
|
41
|
-
it
|
76
|
+
it 'should create users inside block' do
|
42
77
|
OctopusHelper.migrating_to_version 12 do
|
43
|
-
User.using(:brazil).where(:name => 'UsingBlock1').size.
|
44
|
-
User.using(:brazil).where(:name => 'UsingBlock2').size.
|
45
|
-
User.using(:canada).where(:name => 'UsingCanada').size.
|
46
|
-
User.using(:canada).where(:name => 'UsingCanada2').size.
|
78
|
+
expect(User.using(:brazil).where(:name => 'UsingBlock1').size).to eq(1)
|
79
|
+
expect(User.using(:brazil).where(:name => 'UsingBlock2').size).to eq(1)
|
80
|
+
expect(User.using(:canada).where(:name => 'UsingCanada').size).to eq(1)
|
81
|
+
expect(User.using(:canada).where(:name => 'UsingCanada2').size).to eq(1)
|
47
82
|
end
|
48
83
|
end
|
49
84
|
|
50
|
-
it
|
85
|
+
it 'should send the query to the correct shard' do
|
51
86
|
OctopusHelper.migrating_to_version 13 do
|
52
|
-
User.using(:brazil).where(:name => 'Brazil').size.
|
53
|
-
User.using(:brazil).where(:name => 'Canada').size.
|
54
|
-
User.using(:canada).where(:name => 'Brazil').size.
|
55
|
-
User.using(:canada).where(:name => 'Canada').size.
|
87
|
+
expect(User.using(:brazil).where(:name => 'Brazil').size).to eq(1)
|
88
|
+
expect(User.using(:brazil).where(:name => 'Canada').size).to eq(0)
|
89
|
+
expect(User.using(:canada).where(:name => 'Brazil').size).to eq(0)
|
90
|
+
expect(User.using(:canada).where(:name => 'Canada').size).to eq(1)
|
56
91
|
end
|
57
92
|
end
|
58
93
|
|
59
|
-
describe
|
60
|
-
it
|
94
|
+
describe 'when using replication' do
|
95
|
+
it 'should run writes on master when you use replication' do
|
61
96
|
OctopusHelper.using_environment :production_replicated do
|
62
97
|
OctopusHelper.migrating_to_version 10 do
|
63
|
-
Cat.find_by_name(
|
98
|
+
expect(Cat.find_by_name('Replication')).to be_nil
|
64
99
|
end
|
65
100
|
end
|
66
101
|
end
|
67
102
|
|
68
|
-
it
|
103
|
+
it 'should run in all shards, master or another shards' do
|
69
104
|
OctopusHelper.using_environment :production_replicated do
|
70
105
|
OctopusHelper.migrating_to_version 11 do
|
71
|
-
[:slave4, :slave1, :slave2, :slave3].each do |
|
72
|
-
Cat.find_by_name(
|
106
|
+
[:slave4, :slave1, :slave2, :slave3].each do |_sym|
|
107
|
+
expect(Cat.find_by_name('Slaves')).not_to be_nil
|
73
108
|
end
|
74
109
|
end
|
75
110
|
end
|
76
111
|
end
|
77
112
|
end
|
78
113
|
|
79
|
-
it
|
114
|
+
it 'should store the migration versions in each shard' do
|
80
115
|
class SchemaMigration < ActiveRecord::Base; end
|
81
116
|
|
82
117
|
OctopusHelper.migrating_to_version 14 do
|
83
|
-
Octopus.using(:canada) {
|
84
|
-
Octopus.using(:brazil) {
|
85
|
-
Octopus.using(:russia) {
|
118
|
+
expect(Octopus.using(:canada) { get_all_versions }).to include(14)
|
119
|
+
expect(Octopus.using(:brazil) { get_all_versions }).to include(14)
|
120
|
+
expect(Octopus.using(:russia) { get_all_versions }).to include(14)
|
86
121
|
end
|
87
122
|
end
|
88
123
|
|
89
|
-
it
|
124
|
+
it 'should run the migrations on shards that are missing them' do
|
90
125
|
class SchemaMigration < ActiveRecord::Base; end
|
91
126
|
|
92
127
|
Octopus.using(:master) { SchemaMigration.create(:version => 14) }
|
93
128
|
Octopus.using(:canada) { SchemaMigration.create(:version => 14) }
|
94
129
|
|
95
130
|
OctopusHelper.migrating_to_version 14 do
|
96
|
-
Octopus.using(:canada) {
|
97
|
-
Octopus.using(:brazil) {
|
98
|
-
Octopus.using(:russia) {
|
131
|
+
expect(Octopus.using(:canada) { get_all_versions }).to include(14)
|
132
|
+
expect(Octopus.using(:brazil) { get_all_versions }).to include(14)
|
133
|
+
expect(Octopus.using(:russia) { get_all_versions }).to include(14)
|
99
134
|
end
|
100
135
|
end
|
101
136
|
|
102
|
-
describe
|
103
|
-
it
|
137
|
+
describe 'when using a default_migration_group' do
|
138
|
+
it 'should run migrations on all shards in the default_migration_group' do
|
104
139
|
OctopusHelper.using_environment :octopus_with_default_migration_group do
|
105
140
|
OctopusHelper.migrating_to_version 15 do
|
106
|
-
Octopus.using(:master) {
|
107
|
-
Octopus.using(:canada) {
|
108
|
-
Octopus.using(:brazil) {
|
109
|
-
Octopus.using(:russia) {
|
141
|
+
expect(Octopus.using(:master) { get_all_versions }).not_to include(15)
|
142
|
+
expect(Octopus.using(:canada) { get_all_versions }).to include(15)
|
143
|
+
expect(Octopus.using(:brazil) { get_all_versions }).to include(15)
|
144
|
+
expect(Octopus.using(:russia) { get_all_versions }).to include(15)
|
110
145
|
end
|
111
146
|
end
|
112
147
|
end
|
113
148
|
end
|
114
|
-
|
115
149
|
end
|