activerecord-sharding 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.hound.yml +2 -0
- data/.rspec +3 -0
- data/.rubocop.yml +50 -0
- data/.ruby-version +1 -0
- data/.travis.yml +10 -0
- data/Gemfile +4 -0
- data/README.md +184 -0
- data/Rakefile +6 -0
- data/activerecord-sharding.gemspec +35 -0
- data/bin/benchmark_sequencer.rb +71 -0
- data/bin/console +10 -0
- data/bin/setup +7 -0
- data/lib/active_record/sharding.rb +0 -0
- data/lib/active_record/sharding/abstract_repository.rb +29 -0
- data/lib/active_record/sharding/cluster_config.rb +32 -0
- data/lib/active_record/sharding/config.rb +34 -0
- data/lib/active_record/sharding/database_tasks.rb +234 -0
- data/lib/active_record/sharding/errors.rb +9 -0
- data/lib/active_record/sharding/model.rb +59 -0
- data/lib/active_record/sharding/modulo_router.rb +14 -0
- data/lib/active_record/sharding/railtie.rb +9 -0
- data/lib/active_record/sharding/sequencer.rb +40 -0
- data/lib/active_record/sharding/sequencer_config.rb +26 -0
- data/lib/active_record/sharding/sequencer_repository.rb +22 -0
- data/lib/active_record/sharding/shard_repository.rb +31 -0
- data/lib/active_record/sharding/version.rb +5 -0
- data/lib/activerecord-sharding.rb +30 -0
- data/lib/tasks/activerecord-sharding.rake +88 -0
- data/spec/active_record/sharding/abstract_repository_spec.rb +15 -0
- data/spec/active_record/sharding/cluster_config_spec.rb +41 -0
- data/spec/active_record/sharding/errors_spec.rb +9 -0
- data/spec/active_record/sharding/model_spec.rb +90 -0
- data/spec/active_record/sharding/modulo_router_spec.rb +22 -0
- data/spec/active_record/sharding/sequencer_spec.rb +31 -0
- data/spec/active_record/sharding/shard_repository_spec.rb +21 -0
- data/spec/active_record_sharding_spec.rb +15 -0
- data/spec/models.rb +51 -0
- data/spec/schema.rb +17 -0
- data/spec/spec_helper.rb +63 -0
- data/spec/tasks/activerecord-sharding_spec.rb +74 -0
- metadata +254 -0
@@ -0,0 +1,22 @@
|
|
1
|
+
module ActiveRecord
|
2
|
+
module Sharding
|
3
|
+
class SequencerRepository < AbstractRepository
|
4
|
+
attr_reader :base_class
|
5
|
+
|
6
|
+
def initialize(sequencer_config, base_class)
|
7
|
+
@base_class = base_class
|
8
|
+
@sequencer = { sequencer_config.name => generate_model_for_shard(sequencer_config.connection_name) }
|
9
|
+
end
|
10
|
+
|
11
|
+
def fetch(connection_name)
|
12
|
+
@sequencer.fetch connection_name
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def generate_class_name(connection_name)
|
18
|
+
"SequencerFor#{connection_name.to_s.tr('-', '_').classify}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module ActiveRecord
|
2
|
+
module Sharding
|
3
|
+
class ShardRepository < AbstractRepository
|
4
|
+
attr_reader :base_class
|
5
|
+
|
6
|
+
def initialize(cluster_config, base_class)
|
7
|
+
@base_class = base_class
|
8
|
+
|
9
|
+
shards = cluster_config.connections.map do |connection_name|
|
10
|
+
[connection_name, generate_model_for_shard(connection_name)]
|
11
|
+
end
|
12
|
+
|
13
|
+
@shards = Hash[shards]
|
14
|
+
end
|
15
|
+
|
16
|
+
def fetch(connection_name)
|
17
|
+
@shards.fetch connection_name
|
18
|
+
end
|
19
|
+
|
20
|
+
def all
|
21
|
+
@shards.values
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def generate_class_name(connection_name)
|
27
|
+
"ShardFor#{connection_name.to_s.tr('-', '_').classify}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require "active_record"
|
2
|
+
|
3
|
+
require "active_record/sharding/version"
|
4
|
+
require "active_record/sharding/errors"
|
5
|
+
require "active_record/sharding/config"
|
6
|
+
require "active_record/sharding/cluster_config"
|
7
|
+
require "active_record/sharding/modulo_router"
|
8
|
+
require "active_record/sharding/abstract_repository"
|
9
|
+
require "active_record/sharding/shard_repository"
|
10
|
+
require "active_record/sharding/database_tasks"
|
11
|
+
require "active_record/sharding/model"
|
12
|
+
require "active_record/sharding/sequencer"
|
13
|
+
require "active_record/sharding/sequencer_repository"
|
14
|
+
require "active_record/sharding/sequencer_config"
|
15
|
+
|
16
|
+
module ActiveRecord
|
17
|
+
module Sharding
|
18
|
+
class << self
|
19
|
+
def config
|
20
|
+
@config ||= Config.new
|
21
|
+
end
|
22
|
+
|
23
|
+
def configure(&block)
|
24
|
+
config.instance_eval(&block)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
require "active_record/sharding/railtie" if defined? Rails
|
@@ -0,0 +1,88 @@
|
|
1
|
+
namespace :active_record do
|
2
|
+
namespace :sharding do
|
3
|
+
desc "Show all defined clusters and their detail"
|
4
|
+
task info: %i(environment) do
|
5
|
+
ActiveRecord::Sharding::DatabaseTasks.info
|
6
|
+
end
|
7
|
+
|
8
|
+
desc "Setup all databases in all clusters"
|
9
|
+
task setup: %i(create_all load_schema_all) do
|
10
|
+
end
|
11
|
+
|
12
|
+
desc "Create all databases in all clusters"
|
13
|
+
task :create_all => :environment do
|
14
|
+
ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_clusters("create")
|
15
|
+
end
|
16
|
+
|
17
|
+
desc "Drop all databases in all clusters"
|
18
|
+
task :drop_all => :environment do
|
19
|
+
ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_clusters("drop")
|
20
|
+
end
|
21
|
+
|
22
|
+
desc "Load schema to all databases in all clusters"
|
23
|
+
task :load_schema_all => :environment do
|
24
|
+
ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_clusters("load_schema")
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "Create all databases in specific cluster"
|
28
|
+
task :create, %i(cluster_name) => %i(environment) do |_, args|
|
29
|
+
ActiveRecord::Sharding::DatabaseTasks.create_all_databases(args)
|
30
|
+
end
|
31
|
+
|
32
|
+
desc "Drop all databases in specific cluster"
|
33
|
+
task :drop, %i(cluster_name) => %i(environment) do |_, args|
|
34
|
+
ActiveRecord::Sharding::DatabaseTasks.drop_all_databases(args)
|
35
|
+
end
|
36
|
+
|
37
|
+
desc "Load schema to all databases in specific cluster"
|
38
|
+
task :load_schema, %i(cluster_name) => %i(environment) do |_, args|
|
39
|
+
ActiveRecord::Sharding::DatabaseTasks.load_schema_all_databases(args)
|
40
|
+
end
|
41
|
+
|
42
|
+
namespace :sequencer do
|
43
|
+
desc "Create database in specific sequencer"
|
44
|
+
task :create, %i(sequencer_name) => %i(environment) do |_, args|
|
45
|
+
ActiveRecord::Sharding::DatabaseTasks.create_sequencer_database(args)
|
46
|
+
end
|
47
|
+
|
48
|
+
desc "Drop database in specific sequencer"
|
49
|
+
task :drop, %i(sequencer_name) => %i(environment) do |_, args|
|
50
|
+
ActiveRecord::Sharding::DatabaseTasks.drop_sequencer_database(args)
|
51
|
+
end
|
52
|
+
|
53
|
+
desc "Create sequencer table in specific sequencer database"
|
54
|
+
task :create_table, %i(sequencer_name) => %i(environment) do |_, args|
|
55
|
+
ActiveRecord::Sharding::DatabaseTasks.create_table_sequencer_database(args)
|
56
|
+
end
|
57
|
+
|
58
|
+
desc "Insert initial record in specific sequencer database"
|
59
|
+
task :insert_initial_record, %i(sequencer_name) => %i(environment) do |_, args|
|
60
|
+
ActiveRecord::Sharding::DatabaseTasks.insert_initial_record_sequencer_database(args)
|
61
|
+
end
|
62
|
+
|
63
|
+
desc "Setup all databases in sequencers"
|
64
|
+
task setup: %i(create_all create_table_all insert_initial_record_all) do
|
65
|
+
end
|
66
|
+
|
67
|
+
desc "Create all databases in all sequencers"
|
68
|
+
task :create_all => :environment do
|
69
|
+
ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_sequencers("create")
|
70
|
+
end
|
71
|
+
|
72
|
+
desc "Create table in all sequencers databases"
|
73
|
+
task :create_table_all => :environment do
|
74
|
+
ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_sequencers("create_table")
|
75
|
+
end
|
76
|
+
|
77
|
+
desc "Insert inital record in all sequencers tables"
|
78
|
+
task :insert_initial_record_all => :environment do
|
79
|
+
ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_sequencers("insert_initial_record")
|
80
|
+
end
|
81
|
+
|
82
|
+
desc "Drop all databases in all sequencers"
|
83
|
+
task :drop_all => :environment do
|
84
|
+
ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_sequencers("drop")
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
describe ActiveRecord::Sharding::AbstractRepository do
|
2
|
+
describe "#generate_class_name" do
|
3
|
+
let!(:repository) do
|
4
|
+
Class.new(ActiveRecord::Sharding::AbstractRepository) do
|
5
|
+
def initialize
|
6
|
+
generate_class_name "test"
|
7
|
+
end
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
it "raise NotImplementedError" do
|
12
|
+
expect { repository.new }.to raise_error NotImplementedError
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
describe ActiveRecord::Sharding::ClusterConfig do
|
2
|
+
let(:config) { described_class.new(:user) }
|
3
|
+
|
4
|
+
it "returns cluster config name" do
|
5
|
+
expect(config.name).to eq :user
|
6
|
+
end
|
7
|
+
|
8
|
+
describe '#register_connection' do
|
9
|
+
let(:db_connection_001) { :production_user_001 }
|
10
|
+
let(:db_connection_002) { :production_user_002 }
|
11
|
+
let(:db_connection_003) { :production_user_003 }
|
12
|
+
|
13
|
+
before do
|
14
|
+
config.register_connection :db_connection_001
|
15
|
+
config.register_connection :db_connection_002
|
16
|
+
config.register_connection :db_connection_003
|
17
|
+
end
|
18
|
+
|
19
|
+
describe '#registerd_connection_count' do
|
20
|
+
it "returns registerd total connection count" do
|
21
|
+
expect(config.registerd_connection_count).to eq 3
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
describe '#fetch' do
|
26
|
+
it "returns database connection name by modulo number" do
|
27
|
+
expect(config.fetch 0).to eq :db_connection_001
|
28
|
+
expect(config.fetch 1).to eq :db_connection_002
|
29
|
+
expect(config.fetch 2).to eq :db_connection_003
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
describe '#validate_config!' do
|
35
|
+
context "Nothing register connection" do
|
36
|
+
it "returns raise" do
|
37
|
+
expect { config.validate_config! }.to raise_error "Nothing registerd connections."
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
describe ActiveRecord::Sharding::Model do
|
2
|
+
let!(:model) do
|
3
|
+
Class.new(ActiveRecord::Base) do
|
4
|
+
def self.name
|
5
|
+
"User"
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.sequence_id
|
9
|
+
@sequence_id ||= 0
|
10
|
+
end
|
11
|
+
|
12
|
+
class << self
|
13
|
+
attr_writer :sequence_id
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.next_sequence_id
|
17
|
+
self.sequence_id += 1
|
18
|
+
end
|
19
|
+
|
20
|
+
include ActiveRecord::Sharding::Model
|
21
|
+
use_sharding :user, :modulo
|
22
|
+
define_sharding_key :id
|
23
|
+
|
24
|
+
before_put do |attrs|
|
25
|
+
attrs[:id] = next_sequence_id unless attrs[:id]
|
26
|
+
end
|
27
|
+
|
28
|
+
define_parent_methods do
|
29
|
+
def find_from_all_by_name(name)
|
30
|
+
all_shards.map { |m| m.find_by(name: name) }.compact.first
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
let(:alice) { model.put! name: "Alice" }
|
37
|
+
|
38
|
+
describe ".put!" do
|
39
|
+
it "example" do
|
40
|
+
expect(alice.persisted?).to be true
|
41
|
+
expect(alice.name).to eq "Alice"
|
42
|
+
expect(alice.class.name).to match(/User::ShardFor/)
|
43
|
+
end
|
44
|
+
|
45
|
+
context 'next call #put!' do
|
46
|
+
let(:bob) { model.put! name: "Bob" }
|
47
|
+
let(:alice_connect_db) { alice.class.connection.pool.spec.config[:database] }
|
48
|
+
let(:bob_connect_db) { bob.class.connection.pool.spec.config[:database] }
|
49
|
+
|
50
|
+
it "different connection database" do
|
51
|
+
expect(alice_connect_db).not_to eq bob_connect_db
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
context "Not included sharding key in args" do
|
56
|
+
before do
|
57
|
+
allow(User).to receive(:sharding_key).and_return(:dammy_sharding_key)
|
58
|
+
end
|
59
|
+
|
60
|
+
it "raise ActiveRecord::Sharding::MissingShardingKeyAttribute" do
|
61
|
+
expect do
|
62
|
+
User.put! name: "foobar"
|
63
|
+
end.to raise_error ActiveRecord::Sharding::MissingShardingKeyAttribute
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
describe ".shard_for" do
|
69
|
+
before { alice }
|
70
|
+
|
71
|
+
it "example" do
|
72
|
+
user_record = model.shard_for(alice.id).find_by name: "Alice"
|
73
|
+
expect(user_record).not_to be nil
|
74
|
+
expect(user_record.name).to eq "Alice"
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
describe ".define_parent_methods" do
|
79
|
+
before do
|
80
|
+
model.put! name: "foo"
|
81
|
+
model.put! name: "bar"
|
82
|
+
end
|
83
|
+
|
84
|
+
it "enables to define class methods to parent class" do
|
85
|
+
record = model.find_from_all_by_name("foo")
|
86
|
+
expect(record).not_to be_nil
|
87
|
+
expect(record.name).to eq("foo")
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
describe ActiveRecord::Sharding::ModuloRouter do
|
2
|
+
let(:cluster_config) do
|
3
|
+
cluster_config = ActiveRecord::Sharding::ClusterConfig.new :user
|
4
|
+
cluster_config.register_connection :db_connection_001
|
5
|
+
cluster_config.register_connection :db_connection_002
|
6
|
+
cluster_config.register_connection :db_connection_003
|
7
|
+
cluster_config
|
8
|
+
end
|
9
|
+
|
10
|
+
let(:router) { described_class.new cluster_config }
|
11
|
+
|
12
|
+
describe '#route' do
|
13
|
+
it "returns database connection name by id % cluster nodes" do
|
14
|
+
expect(router.route(1)).to eq :db_connection_002
|
15
|
+
expect(router.route(2)).to eq :db_connection_003
|
16
|
+
expect(router.route(3)).to eq :db_connection_001
|
17
|
+
expect(router.route(4)).to eq :db_connection_002
|
18
|
+
expect(router.route(5)).to eq :db_connection_003
|
19
|
+
expect(router.route(6)).to eq :db_connection_001
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
describe ActiveRecord::Sharding::Model do
|
2
|
+
let!(:model) do
|
3
|
+
Class.new(ActiveRecord::Base) do
|
4
|
+
def self.name
|
5
|
+
"User"
|
6
|
+
end
|
7
|
+
|
8
|
+
include ActiveRecord::Sharding::Sequencer
|
9
|
+
use_sequencer :user
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '#current_sequence_id' do
|
14
|
+
it "returns current sequence id" do
|
15
|
+
expect(model.current_sequence_id).to be_a_kind_of Fixnum
|
16
|
+
end
|
17
|
+
end
|
18
|
+
describe '#next_sequence_id' do
|
19
|
+
let(:current_id) { model.current_sequence_id }
|
20
|
+
let(:next_id) { model.next_sequence_id }
|
21
|
+
|
22
|
+
it "returns next sequence id" do
|
23
|
+
expect(current_id + 1).to eq next_id
|
24
|
+
expect(next_id).to be_a_kind_of Fixnum
|
25
|
+
end
|
26
|
+
|
27
|
+
it "next sequence id > current sequence id" do
|
28
|
+
expect(current_id).to be < next_id
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
describe ActiveRecord::Sharding::ShardRepository do
|
2
|
+
let(:cluster_config) { ActiveRecord::Sharding.config.fetch_cluster_config(:user) }
|
3
|
+
let(:shard_repository) { described_class.new cluster_config, User }
|
4
|
+
|
5
|
+
describe '#fetch' do
|
6
|
+
it "returns User class base shard object" do
|
7
|
+
expect(shard_repository.fetch(:test_user_001).connection).to be_a ActiveRecord::ConnectionAdapters::AbstractAdapter
|
8
|
+
expect(shard_repository.fetch(:test_user_001).table_name).to eq "users"
|
9
|
+
expect(shard_repository.fetch(:test_user_001).name).to eq "User::ShardForTestUser001"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '#all' do
|
14
|
+
it "returns all shard class objects" do
|
15
|
+
expect(shard_repository.all.count).to eq 3
|
16
|
+
expect(shard_repository.all[0]).to eq shard_repository.fetch(:test_user_001)
|
17
|
+
expect(shard_repository.all[1]).to eq shard_repository.fetch(:test_user_002)
|
18
|
+
expect(shard_repository.all[2]).to eq shard_repository.fetch(:test_user_003)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
describe ActiveRecord::Sharding do
|
2
|
+
it "has a version number" do
|
3
|
+
expect(ActiveRecord::Sharding::VERSION).not_to be nil
|
4
|
+
end
|
5
|
+
|
6
|
+
describe ".config" do
|
7
|
+
it "returns Config class" do
|
8
|
+
expect(ActiveRecord::Sharding.config).to be_a ActiveRecord::Sharding::Config
|
9
|
+
end
|
10
|
+
|
11
|
+
it "returns equal instance, every call" do
|
12
|
+
expect(ActiveRecord::Sharding.config).to eq ActiveRecord::Sharding.config
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
data/spec/models.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
base = { "adapter" => "mysql2", "encoding" => "utf8", "pool" => 5, "username" => "root", "password" => "", "host" => "localhost" }
|
2
|
+
ActiveRecord::Base.configurations = {
|
3
|
+
"test_user_001" => base.merge("database" => "user_001"),
|
4
|
+
"test_user_002" => base.merge("database" => "user_002"),
|
5
|
+
"test_user_003" => base.merge("database" => "user_003"),
|
6
|
+
"test_user_sequencer" => base.merge("database" => "user_sequencer"),
|
7
|
+
"test" => base.merge("database" => "default")
|
8
|
+
}
|
9
|
+
ActiveRecord::Base.establish_connection(:test)
|
10
|
+
|
11
|
+
ActiveRecord::Sharding.configure do |config|
|
12
|
+
config.define_cluster(:user) do |cluster|
|
13
|
+
cluster.register_connection(:test_user_001)
|
14
|
+
cluster.register_connection(:test_user_002)
|
15
|
+
cluster.register_connection(:test_user_003)
|
16
|
+
end
|
17
|
+
|
18
|
+
config.define_sequencer(:user) do |sequencer|
|
19
|
+
sequencer.register_connection(:test_user_sequencer)
|
20
|
+
sequencer.register_table_name("user_id")
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class User < ActiveRecord::Base
|
25
|
+
def items
|
26
|
+
return [] unless id
|
27
|
+
Item.shard_for(id).where(user_id: id).all
|
28
|
+
end
|
29
|
+
|
30
|
+
include ActiveRecord::Sharding::Model
|
31
|
+
use_sharding :user, :modulo
|
32
|
+
define_sharding_key :id
|
33
|
+
|
34
|
+
include ActiveRecord::Sharding::Sequencer
|
35
|
+
use_sequencer :user
|
36
|
+
|
37
|
+
before_put do |attributes|
|
38
|
+
attributes[:id] = next_sequence_id unless attributes[:id]
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class Item < ActiveRecord::Base
|
43
|
+
def user
|
44
|
+
return unless user_id
|
45
|
+
User.shard_for(user_id).find_by user_id
|
46
|
+
end
|
47
|
+
|
48
|
+
include ActiveRecord::Sharding::Model
|
49
|
+
use_sharding :user, :modulo
|
50
|
+
define_sharding_key :user_id
|
51
|
+
end
|