ar-octopus 0.8.1 → 0.10.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +6 -14
- data/.gitignore +1 -0
- data/.rspec +1 -1
- data/.rubocop.yml +46 -0
- data/.rubocop_todo.yml +56 -0
- data/.travis.yml +7 -12
- data/Appraisals +11 -4
- data/Gemfile +1 -1
- data/README.mkdn +138 -63
- data/Rakefile +23 -16
- data/ar-octopus.gemspec +23 -20
- data/gemfiles/rails42.gemfile +7 -0
- data/gemfiles/{rails32.gemfile → rails5.gemfile} +2 -2
- data/gemfiles/{rails4.gemfile → rails51.gemfile} +2 -2
- data/gemfiles/rails52.gemfile +7 -0
- data/lib/ar-octopus.rb +1 -1
- data/lib/octopus/{rails3/abstract_adapter.rb → abstract_adapter.rb} +4 -15
- data/lib/octopus/association.rb +8 -99
- data/lib/octopus/association_shard_tracking.rb +74 -0
- data/lib/octopus/collection_association.rb +17 -0
- data/lib/octopus/collection_proxy.rb +16 -0
- data/lib/octopus/exception.rb +4 -0
- data/lib/octopus/finder_methods.rb +8 -0
- data/lib/octopus/load_balancing/round_robin.rb +20 -0
- data/lib/octopus/load_balancing.rb +4 -0
- data/lib/octopus/{rails3/log_subscriber.rb → log_subscriber.rb} +6 -2
- data/lib/octopus/migration.rb +187 -110
- data/lib/octopus/model.rb +151 -131
- data/lib/octopus/persistence.rb +45 -0
- data/lib/octopus/proxy.rb +297 -232
- data/lib/octopus/proxy_config.rb +251 -0
- data/lib/octopus/query_cache_for_shards.rb +24 -0
- data/lib/octopus/railtie.rb +1 -3
- data/lib/octopus/relation_proxy.rb +70 -0
- data/lib/octopus/result_patch.rb +19 -0
- data/lib/octopus/scope_proxy.rb +54 -36
- data/lib/octopus/shard_tracking/attribute.rb +22 -0
- data/lib/octopus/shard_tracking/dynamic.rb +11 -0
- data/lib/octopus/shard_tracking.rb +46 -0
- data/lib/octopus/singular_association.rb +9 -0
- data/lib/octopus/slave_group.rb +13 -0
- data/lib/octopus/version.rb +1 -1
- data/lib/octopus.rb +125 -33
- data/lib/tasks/octopus.rake +2 -2
- data/sample_app/Gemfile +3 -3
- data/sample_app/autotest/discover.rb +2 -2
- data/sample_app/config/application.rb +1 -1
- data/sample_app/config/boot.rb +1 -1
- data/sample_app/config/environments/test.rb +1 -1
- data/sample_app/config/initializers/session_store.rb +1 -1
- data/sample_app/config/initializers/wrap_parameters.rb +1 -1
- data/sample_app/config/routes.rb +1 -1
- data/sample_app/db/migrate/20100720210335_create_sample_users.rb +2 -2
- data/sample_app/db/schema.rb +10 -10
- data/sample_app/db/seeds.rb +3 -3
- data/sample_app/features/step_definitions/seeds_steps.rb +4 -4
- data/sample_app/features/step_definitions/web_steps.rb +3 -4
- data/sample_app/features/support/env.rb +3 -4
- data/sample_app/features/support/paths.rb +4 -4
- data/sample_app/lib/tasks/cucumber.rake +43 -44
- data/sample_app/spec/spec_helper.rb +3 -3
- data/spec/config/shards.yml +78 -0
- data/spec/migrations/10_create_users_using_replication.rb +4 -4
- data/spec/migrations/11_add_field_in_all_slaves.rb +4 -4
- data/spec/migrations/12_create_users_using_block.rb +8 -8
- data/spec/migrations/13_create_users_using_block_and_using.rb +5 -5
- data/spec/migrations/14_create_users_on_shards_of_a_group_with_versions.rb +3 -3
- data/spec/migrations/15_create_user_on_shards_of_default_group_with_versions.rb +3 -3
- data/spec/migrations/1_create_users_on_master.rb +4 -4
- data/spec/migrations/2_create_users_on_canada.rb +4 -4
- data/spec/migrations/3_create_users_on_both_shards.rb +4 -4
- data/spec/migrations/4_create_users_on_shards_of_a_group.rb +4 -4
- data/spec/migrations/5_create_users_on_multiples_groups.rb +3 -3
- data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +4 -4
- data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +4 -4
- data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +4 -4
- data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +5 -5
- data/spec/octopus/association_shard_tracking_spec.rb +1036 -0
- data/spec/octopus/collection_proxy_spec.rb +16 -0
- data/spec/octopus/load_balancing/round_robin_spec.rb +15 -0
- data/spec/octopus/log_subscriber_spec.rb +5 -5
- data/spec/octopus/migration_spec.rb +83 -49
- data/spec/octopus/model_spec.rb +544 -292
- data/spec/octopus/octopus_spec.rb +64 -31
- data/spec/octopus/proxy_spec.rb +145 -141
- data/spec/octopus/query_cache_for_shards_spec.rb +40 -0
- data/spec/octopus/relation_proxy_spec.rb +132 -0
- data/spec/octopus/replicated_slave_grouped_spec.rb +91 -0
- data/spec/octopus/replication_spec.rb +140 -65
- data/spec/octopus/scope_proxy_spec.rb +90 -10
- data/spec/octopus/sharded_replicated_slave_grouped_spec.rb +55 -0
- data/spec/octopus/sharded_spec.rb +10 -10
- data/spec/spec_helper.rb +8 -6
- data/spec/support/active_record/connection_adapters/modify_config_adapter.rb +1 -3
- data/spec/support/database_connection.rb +2 -2
- data/spec/support/database_models.rb +18 -17
- data/spec/support/octopus_helper.rb +32 -25
- data/spec/support/query_count.rb +1 -3
- data/spec/support/shared_contexts.rb +3 -3
- data/spec/tasks/octopus.rake_spec.rb +10 -10
- metadata +112 -70
- data/.ruby-version +0 -1
- data/init.rb +0 -1
- data/lib/octopus/association_collection.rb +0 -49
- data/lib/octopus/has_and_belongs_to_many_association.rb +0 -17
- data/lib/octopus/rails3/persistence.rb +0 -39
- data/lib/octopus/rails3/singular_association.rb +0 -34
- data/rails/init.rb +0 -1
- data/spec/octopus/association_spec.rb +0 -712
@@ -1,13 +1,13 @@
|
|
1
|
-
require
|
1
|
+
require 'spec_helper'
|
2
2
|
|
3
|
-
describe
|
3
|
+
describe 'when the database is not entire sharded' do
|
4
4
|
before(:each) do
|
5
|
-
Octopus.
|
6
|
-
OctopusHelper.clean_connection_proxy
|
5
|
+
allow(Octopus).to receive(:env).and_return('not_entire_sharded')
|
6
|
+
OctopusHelper.clean_connection_proxy
|
7
7
|
end
|
8
8
|
|
9
|
-
it
|
10
|
-
|
9
|
+
it 'should not send all queries to the specified slave' do
|
10
|
+
skip
|
11
11
|
# User.create!(:name => "Thiago")
|
12
12
|
#
|
13
13
|
# using_environment :not_entire_sharded do
|
@@ -19,15 +19,15 @@ describe "when the database is not entire sharded" do
|
|
19
19
|
# User.count.should == 2
|
20
20
|
end
|
21
21
|
|
22
|
-
it
|
23
|
-
Cat.create!(:name =>
|
22
|
+
it 'should pick the shard based on current_shard when you have a sharded model' do
|
23
|
+
Cat.create!(:name => 'Thiago')
|
24
24
|
|
25
25
|
OctopusHelper.using_environment :not_entire_sharded do
|
26
26
|
Octopus.using(:russia) do
|
27
|
-
Cat.create!(:name =>
|
27
|
+
Cat.create!(:name => 'Thiago')
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
31
|
-
Cat.count.
|
31
|
+
expect(Cat.count).to eq(1)
|
32
32
|
end
|
33
33
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,16 +1,18 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
1
|
+
require 'rubygems'
|
2
|
+
require 'pry'
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'octopus'
|
5
5
|
|
6
6
|
Octopus.instance_variable_set(:@directory, File.dirname(__FILE__))
|
7
7
|
|
8
|
+
BaseOctopusMigrationClass = (Octopus.rails4? ? ActiveRecord::Migration : ActiveRecord::Migration[ActiveRecord::VERSION::STRING[0..2]])
|
9
|
+
|
8
10
|
# Requires supporting files with custom matchers and macros, etc,
|
9
11
|
# in ./support/ and its subdirectories.
|
10
|
-
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
12
|
+
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
|
11
13
|
|
12
14
|
RSpec.configure do |config|
|
13
|
-
config.before(:each) do
|
15
|
+
config.before(:each) do |example|
|
14
16
|
OctopusHelper.clean_all_shards(example.metadata[:shards])
|
15
17
|
end
|
16
18
|
end
|
@@ -4,14 +4,12 @@ module ActiveRecord
|
|
4
4
|
ConnectionAdapters::ModifyConfigAdapter.new(config)
|
5
5
|
end
|
6
6
|
end
|
7
|
-
|
7
|
+
|
8
8
|
module ConnectionAdapters
|
9
9
|
class ModifyConfigAdapter < AbstractAdapter
|
10
|
-
|
11
10
|
def initialize(config)
|
12
11
|
config.replace(config.symbolize_keys)
|
13
12
|
end
|
14
|
-
|
15
13
|
end
|
16
14
|
end
|
17
15
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'logger'
|
2
2
|
|
3
|
-
ActiveRecord::Base.establish_connection(:adapter =>
|
3
|
+
ActiveRecord::Base.establish_connection(:adapter => 'mysql2', :database => 'octopus_shard_1', :username => "#{ENV['MYSQL_USER'] || 'root'}", :password => '')
|
4
4
|
ActiveRecord::Base.logger = Logger.new(File.open('database.log', 'a'))
|
@@ -1,36 +1,37 @@
|
|
1
1
|
# Rails 3.1 needs to do some introspection around the base class, which requires
|
2
2
|
# the model be a descendent of ActiveRecord::Base.
|
3
|
-
class BlankModel < ActiveRecord::Base; end
|
3
|
+
class BlankModel < ActiveRecord::Base; end
|
4
4
|
|
5
|
-
#The user class is just sharded, not replicated
|
5
|
+
# The user class is just sharded, not replicated
|
6
6
|
class User < ActiveRecord::Base
|
7
|
-
scope :thiago,
|
7
|
+
scope :thiago, -> { where(:name => 'Thiago') }
|
8
8
|
|
9
9
|
def awesome_queries
|
10
10
|
Octopus.using(:canada) do
|
11
|
-
User.create(:name =>
|
11
|
+
User.create(:name => 'teste')
|
12
12
|
end
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
|
-
#The client class isn't replicated
|
16
|
+
# The client class isn't replicated
|
17
17
|
class Client < ActiveRecord::Base
|
18
18
|
has_many :items
|
19
19
|
has_many :comments, :as => :commentable
|
20
20
|
end
|
21
21
|
|
22
|
-
#This class is replicated
|
22
|
+
# This class is replicated
|
23
23
|
class Cat < ActiveRecord::Base
|
24
|
-
replicated_model
|
25
|
-
#sharded_model()
|
24
|
+
replicated_model
|
25
|
+
# sharded_model()
|
26
26
|
end
|
27
27
|
|
28
|
-
#This class sets its own connection
|
28
|
+
# This class sets its own connection
|
29
29
|
class CustomConnection < ActiveRecord::Base
|
30
|
-
|
30
|
+
self.table_name = 'custom'
|
31
|
+
octopus_establish_connection(:adapter => 'mysql2', :database => 'octopus_shard_2', :username => "#{ENV['MYSQL_USER'] || 'root'}", :password => '')
|
31
32
|
end
|
32
33
|
|
33
|
-
#This items belongs to a client
|
34
|
+
# This items belongs to a client
|
34
35
|
class Item < ActiveRecord::Base
|
35
36
|
belongs_to :client
|
36
37
|
has_many :parts
|
@@ -75,24 +76,24 @@ end
|
|
75
76
|
|
76
77
|
class Comment < ActiveRecord::Base
|
77
78
|
belongs_to :commentable, :polymorphic => true
|
79
|
+
scope :open, -> { where(open: true) }
|
78
80
|
end
|
79
81
|
|
80
|
-
|
81
82
|
class Bacon < ActiveRecord::Base
|
82
83
|
self.table_name = 'yummy'
|
83
84
|
end
|
84
85
|
|
85
86
|
class Cheese < ActiveRecord::Base
|
86
|
-
self.table_name = 'yummy'
|
87
|
+
self.table_name = 'yummy'
|
87
88
|
end
|
88
89
|
|
89
90
|
class Ham < ActiveRecord::Base
|
90
91
|
self.table_name = 'yummy'
|
91
92
|
end
|
92
93
|
|
93
|
-
#This class sets its own connection
|
94
|
+
# This class sets its own connection
|
94
95
|
class Advert < ActiveRecord::Base
|
95
|
-
establish_connection(:adapter =>
|
96
|
+
establish_connection(:adapter => 'postgresql', :database => 'octopus_shard_1', :username => ENV['POSTGRES_USER'] || 'postgres', :password => '')
|
96
97
|
end
|
97
98
|
|
98
99
|
class MmorpgPlayer < ActiveRecord::Base
|
@@ -102,9 +103,9 @@ end
|
|
102
103
|
|
103
104
|
class Weapon < ActiveRecord::Base
|
104
105
|
belongs_to :mmorpg_player, :inverse_of => :weapons
|
105
|
-
validates
|
106
|
+
validates :hand, :uniqueness => { :scope => :mmorpg_player_id }
|
106
107
|
validates_presence_of :mmorpg_player
|
107
|
-
has_many
|
108
|
+
has_many :skills
|
108
109
|
end
|
109
110
|
|
110
111
|
class Skill < ActiveRecord::Base
|
@@ -1,55 +1,62 @@
|
|
1
1
|
module OctopusHelper
|
2
2
|
def self.clean_all_shards(shards)
|
3
3
|
if shards.nil?
|
4
|
-
shards = BlankModel.using(:master).connection.
|
4
|
+
shards = BlankModel.using(:master).connection.shards.keys
|
5
5
|
end
|
6
6
|
|
7
7
|
shards.each do |shard_symbol|
|
8
|
-
|
8
|
+
%w(schema_migrations users clients cats items keyboards computers permissions_roles roles permissions assignments projects programmers yummy adverts).each do |tables|
|
9
9
|
BlankModel.using(shard_symbol).connection.execute("DELETE FROM #{tables}")
|
10
10
|
end
|
11
|
-
|
12
11
|
if shard_symbol == 'alone_shard'
|
13
|
-
|
12
|
+
%w(mmorpg_players weapons skills).each do |table|
|
14
13
|
BlankModel.using(shard_symbol).connection.execute("DELETE FROM #{table}")
|
15
14
|
end
|
16
15
|
end
|
16
|
+
BlankModel.using(:master).connection.shards[shard_symbol].disconnect if Octopus.atleast_rails50?
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
|
-
def self.clean_connection_proxy
|
21
|
-
Thread.current[
|
22
|
-
Thread.current[
|
23
|
-
Thread.current[
|
24
|
-
Thread.current[
|
25
|
-
Thread.current[
|
20
|
+
def self.clean_connection_proxy
|
21
|
+
Thread.current['octopus.current_model'] = nil
|
22
|
+
Thread.current['octopus.current_shard'] = nil
|
23
|
+
Thread.current['octopus.current_group'] = nil
|
24
|
+
Thread.current['octopus.current_slave_group'] = nil
|
25
|
+
Thread.current['octopus.block'] = nil
|
26
26
|
|
27
|
-
ActiveRecord::Base.
|
27
|
+
ActiveRecord::Base.class_variable_set(:@@connection_proxy, nil)
|
28
28
|
end
|
29
29
|
|
30
|
-
def self.migrating_to_version(version, &
|
30
|
+
def self.migrating_to_version(version, &_block)
|
31
31
|
migrations_root = File.expand_path(File.join(File.dirname(__FILE__), '..', 'migrations'))
|
32
|
-
|
32
|
+
|
33
33
|
begin
|
34
|
-
|
34
|
+
migrate_to_version(:up, migrations_root, version)
|
35
35
|
yield
|
36
36
|
ensure
|
37
|
-
|
37
|
+
migrate_to_version(:down, migrations_root, version)
|
38
38
|
end
|
39
39
|
end
|
40
|
-
|
41
|
-
def self.
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
set_octopus_env('octopus')
|
48
|
-
clean_connection_proxy()
|
40
|
+
|
41
|
+
def self.migrate_to_version(direction, root, version)
|
42
|
+
if Octopus.atleast_rails52?
|
43
|
+
migrations = ActiveRecord::MigrationContext.new(root).migrations.select {|mig| version == mig.version }
|
44
|
+
ActiveRecord::Migrator.new(direction, migrations, version).run
|
45
|
+
else
|
46
|
+
ActiveRecord::Migrator.run(direction, root, version)
|
49
47
|
end
|
50
48
|
end
|
51
49
|
|
52
|
-
def self.
|
50
|
+
def self.using_environment(environment, &_block)
|
51
|
+
self.octopus_env = environment.to_s
|
52
|
+
clean_connection_proxy
|
53
|
+
yield
|
54
|
+
ensure
|
55
|
+
self.octopus_env = 'octopus'
|
56
|
+
clean_connection_proxy
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.octopus_env=(env)
|
53
60
|
Octopus.instance_variable_set(:@config, nil)
|
54
61
|
Octopus.stub(:env).and_return(env)
|
55
62
|
end
|
data/spec/support/query_count.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
module ActiveRecord
|
2
2
|
class QueryCounter
|
3
|
-
|
4
3
|
attr_accessor :query_count
|
5
4
|
|
6
5
|
def initialize
|
@@ -11,9 +10,8 @@ module ActiveRecord
|
|
11
10
|
lambda(&method(:callback))
|
12
11
|
end
|
13
12
|
|
14
|
-
def callback(
|
13
|
+
def callback(_name, _start, _finish, _message_id, values)
|
15
14
|
@query_count += 1 unless %w(CACHE SCHEMA).include?(values[:name])
|
16
15
|
end
|
17
|
-
|
18
16
|
end
|
19
17
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
shared_context
|
1
|
+
shared_context 'with query cache enabled' do
|
2
2
|
let!(:counter) { ActiveRecord::QueryCounter.new }
|
3
3
|
|
4
4
|
before(:each) do
|
@@ -6,12 +6,12 @@ shared_context "with query cache enabled" do
|
|
6
6
|
counter.query_count = 0
|
7
7
|
end
|
8
8
|
|
9
|
-
after(:each) do
|
9
|
+
after(:each) do
|
10
10
|
ActiveRecord::Base.connection.disable_query_cache!
|
11
11
|
end
|
12
12
|
|
13
13
|
around(:each) do |example|
|
14
|
-
active_support_subscribed(counter.to_proc, 'sql.active_record') do
|
14
|
+
active_support_subscribed(counter.to_proc, 'sql.active_record') do
|
15
15
|
example.run
|
16
16
|
end
|
17
17
|
end
|
@@ -1,31 +1,31 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'rake'
|
3
3
|
|
4
|
-
describe
|
4
|
+
describe 'octopus.rake' do
|
5
5
|
before do
|
6
|
-
load File.expand_path(
|
6
|
+
load File.expand_path('../../../lib/tasks/octopus.rake', __FILE__)
|
7
7
|
Rake::Task.define_task(:environment)
|
8
8
|
end
|
9
9
|
|
10
|
-
describe
|
10
|
+
describe 'octopus:copy_schema_versions' do
|
11
11
|
class SchemaMigration < ActiveRecord::Base; end
|
12
12
|
|
13
13
|
before do
|
14
|
-
Rake::Task[
|
14
|
+
Rake::Task['octopus:copy_schema_versions'].reenable
|
15
15
|
|
16
|
-
path = File.expand_path(
|
16
|
+
path = File.expand_path('../../migrations', __FILE__)
|
17
17
|
ActiveRecord::Migrator.migrations_paths = [path]
|
18
18
|
end
|
19
19
|
|
20
|
-
it
|
20
|
+
it 'assumes each shard migrated to the current master version' do
|
21
21
|
SchemaMigration.create(:version => 1)
|
22
22
|
SchemaMigration.create(:version => 2)
|
23
23
|
SchemaMigration.create(:version => 3)
|
24
24
|
|
25
|
-
Rake::Task[
|
25
|
+
Rake::Task['octopus:copy_schema_versions'].invoke
|
26
26
|
|
27
27
|
ActiveRecord::Base.connection.shard_names.each do |shard_name|
|
28
|
-
Octopus.using(shard_name) { ActiveRecord::
|
28
|
+
expect(Octopus.using(shard_name) { ActiveRecord::SchemaMigration.all.pluck(:version).map(&:to_i).sort }).to eq([1, 2, 3])
|
29
29
|
end
|
30
30
|
end
|
31
31
|
end
|