activerecord-sharding 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.hound.yml +2 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +50 -0
  6. data/.ruby-version +1 -0
  7. data/.travis.yml +10 -0
  8. data/Gemfile +4 -0
  9. data/README.md +184 -0
  10. data/Rakefile +6 -0
  11. data/activerecord-sharding.gemspec +35 -0
  12. data/bin/benchmark_sequencer.rb +71 -0
  13. data/bin/console +10 -0
  14. data/bin/setup +7 -0
  15. data/lib/active_record/sharding.rb +0 -0
  16. data/lib/active_record/sharding/abstract_repository.rb +29 -0
  17. data/lib/active_record/sharding/cluster_config.rb +32 -0
  18. data/lib/active_record/sharding/config.rb +34 -0
  19. data/lib/active_record/sharding/database_tasks.rb +234 -0
  20. data/lib/active_record/sharding/errors.rb +9 -0
  21. data/lib/active_record/sharding/model.rb +59 -0
  22. data/lib/active_record/sharding/modulo_router.rb +14 -0
  23. data/lib/active_record/sharding/railtie.rb +9 -0
  24. data/lib/active_record/sharding/sequencer.rb +40 -0
  25. data/lib/active_record/sharding/sequencer_config.rb +26 -0
  26. data/lib/active_record/sharding/sequencer_repository.rb +22 -0
  27. data/lib/active_record/sharding/shard_repository.rb +31 -0
  28. data/lib/active_record/sharding/version.rb +5 -0
  29. data/lib/activerecord-sharding.rb +30 -0
  30. data/lib/tasks/activerecord-sharding.rake +88 -0
  31. data/spec/active_record/sharding/abstract_repository_spec.rb +15 -0
  32. data/spec/active_record/sharding/cluster_config_spec.rb +41 -0
  33. data/spec/active_record/sharding/errors_spec.rb +9 -0
  34. data/spec/active_record/sharding/model_spec.rb +90 -0
  35. data/spec/active_record/sharding/modulo_router_spec.rb +22 -0
  36. data/spec/active_record/sharding/sequencer_spec.rb +31 -0
  37. data/spec/active_record/sharding/shard_repository_spec.rb +21 -0
  38. data/spec/active_record_sharding_spec.rb +15 -0
  39. data/spec/models.rb +51 -0
  40. data/spec/schema.rb +17 -0
  41. data/spec/spec_helper.rb +63 -0
  42. data/spec/tasks/activerecord-sharding_spec.rb +74 -0
  43. metadata +254 -0
@@ -0,0 +1,22 @@
1
+ module ActiveRecord
2
+ module Sharding
3
+ class SequencerRepository < AbstractRepository
4
+ attr_reader :base_class
5
+
6
+ def initialize(sequencer_config, base_class)
7
+ @base_class = base_class
8
+ @sequencer = { sequencer_config.name => generate_model_for_shard(sequencer_config.connection_name) }
9
+ end
10
+
11
+ def fetch(connection_name)
12
+ @sequencer.fetch connection_name
13
+ end
14
+
15
+ private
16
+
17
+ def generate_class_name(connection_name)
18
+ "SequencerFor#{connection_name.to_s.tr('-', '_').classify}"
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,31 @@
1
+ module ActiveRecord
2
+ module Sharding
3
+ class ShardRepository < AbstractRepository
4
+ attr_reader :base_class
5
+
6
+ def initialize(cluster_config, base_class)
7
+ @base_class = base_class
8
+
9
+ shards = cluster_config.connections.map do |connection_name|
10
+ [connection_name, generate_model_for_shard(connection_name)]
11
+ end
12
+
13
+ @shards = Hash[shards]
14
+ end
15
+
16
+ def fetch(connection_name)
17
+ @shards.fetch connection_name
18
+ end
19
+
20
+ def all
21
+ @shards.values
22
+ end
23
+
24
+ private
25
+
26
+ def generate_class_name(connection_name)
27
+ "ShardFor#{connection_name.to_s.tr('-', '_').classify}"
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,5 @@
1
+ module ActiveRecord
2
+ module Sharding
3
+ VERSION = "0.1.0"
4
+ end
5
+ end
@@ -0,0 +1,30 @@
1
+ require "active_record"
2
+
3
+ require "active_record/sharding/version"
4
+ require "active_record/sharding/errors"
5
+ require "active_record/sharding/config"
6
+ require "active_record/sharding/cluster_config"
7
+ require "active_record/sharding/modulo_router"
8
+ require "active_record/sharding/abstract_repository"
9
+ require "active_record/sharding/shard_repository"
10
+ require "active_record/sharding/database_tasks"
11
+ require "active_record/sharding/model"
12
+ require "active_record/sharding/sequencer"
13
+ require "active_record/sharding/sequencer_repository"
14
+ require "active_record/sharding/sequencer_config"
15
+
16
+ module ActiveRecord
17
+ module Sharding
18
+ class << self
19
+ def config
20
+ @config ||= Config.new
21
+ end
22
+
23
+ def configure(&block)
24
+ config.instance_eval(&block)
25
+ end
26
+ end
27
+ end
28
+ end
29
+
30
+ require "active_record/sharding/railtie" if defined? Rails
@@ -0,0 +1,88 @@
1
+ namespace :active_record do
2
+ namespace :sharding do
3
+ desc "Show all defined clusters and their detail"
4
+ task info: %i(environment) do
5
+ ActiveRecord::Sharding::DatabaseTasks.info
6
+ end
7
+
8
+ desc "Setup all databases in all clusters"
9
+ task setup: %i(create_all load_schema_all) do
10
+ end
11
+
12
+ desc "Create all databases in all clusters"
13
+ task :create_all => :environment do
14
+ ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_clusters("create")
15
+ end
16
+
17
+ desc "Drop all databases in all clusters"
18
+ task :drop_all => :environment do
19
+ ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_clusters("drop")
20
+ end
21
+
22
+ desc "Load schema to all databases in all clusters"
23
+ task :load_schema_all => :environment do
24
+ ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_clusters("load_schema")
25
+ end
26
+
27
+ desc "Create all databases in specific cluster"
28
+ task :create, %i(cluster_name) => %i(environment) do |_, args|
29
+ ActiveRecord::Sharding::DatabaseTasks.create_all_databases(args)
30
+ end
31
+
32
+ desc "Drop all databases in specific cluster"
33
+ task :drop, %i(cluster_name) => %i(environment) do |_, args|
34
+ ActiveRecord::Sharding::DatabaseTasks.drop_all_databases(args)
35
+ end
36
+
37
+ desc "Load schema to all databases in specific cluster"
38
+ task :load_schema, %i(cluster_name) => %i(environment) do |_, args|
39
+ ActiveRecord::Sharding::DatabaseTasks.load_schema_all_databases(args)
40
+ end
41
+
42
+ namespace :sequencer do
43
+ desc "Create database in specific sequencer"
44
+ task :create, %i(sequencer_name) => %i(environment) do |_, args|
45
+ ActiveRecord::Sharding::DatabaseTasks.create_sequencer_database(args)
46
+ end
47
+
48
+ desc "Drop database in specific sequencer"
49
+ task :drop, %i(sequencer_name) => %i(environment) do |_, args|
50
+ ActiveRecord::Sharding::DatabaseTasks.drop_sequencer_database(args)
51
+ end
52
+
53
+ desc "Create sequencer table in specific sequencer database"
54
+ task :create_table, %i(sequencer_name) => %i(environment) do |_, args|
55
+ ActiveRecord::Sharding::DatabaseTasks.create_table_sequencer_database(args)
56
+ end
57
+
58
+ desc "Insert initial record in specific sequencer database"
59
+ task :insert_initial_record, %i(sequencer_name) => %i(environment) do |_, args|
60
+ ActiveRecord::Sharding::DatabaseTasks.insert_initial_record_sequencer_database(args)
61
+ end
62
+
63
+ desc "Setup all databases in sequencers"
64
+ task setup: %i(create_all create_table_all insert_initial_record_all) do
65
+ end
66
+
67
+ desc "Create all databases in all sequencers"
68
+ task :create_all => :environment do
69
+ ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_sequencers("create")
70
+ end
71
+
72
+ desc "Create table in all sequencers databases"
73
+ task :create_table_all => :environment do
74
+ ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_sequencers("create_table")
75
+ end
76
+
77
+ desc "Insert inital record in all sequencers tables"
78
+ task :insert_initial_record_all => :environment do
79
+ ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_sequencers("insert_initial_record")
80
+ end
81
+
82
+ desc "Drop all databases in all sequencers"
83
+ task :drop_all => :environment do
84
+ ActiveRecord::Sharding::DatabaseTasks.invoke_task_for_all_sequencers("drop")
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,15 @@
1
+ describe ActiveRecord::Sharding::AbstractRepository do
2
+ describe "#generate_class_name" do
3
+ let!(:repository) do
4
+ Class.new(ActiveRecord::Sharding::AbstractRepository) do
5
+ def initialize
6
+ generate_class_name "test"
7
+ end
8
+ end
9
+ end
10
+
11
+ it "raise NotImplementedError" do
12
+ expect { repository.new }.to raise_error NotImplementedError
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,41 @@
1
+ describe ActiveRecord::Sharding::ClusterConfig do
2
+ let(:config) { described_class.new(:user) }
3
+
4
+ it "returns cluster config name" do
5
+ expect(config.name).to eq :user
6
+ end
7
+
8
+ describe '#register_connection' do
9
+ let(:db_connection_001) { :production_user_001 }
10
+ let(:db_connection_002) { :production_user_002 }
11
+ let(:db_connection_003) { :production_user_003 }
12
+
13
+ before do
14
+ config.register_connection :db_connection_001
15
+ config.register_connection :db_connection_002
16
+ config.register_connection :db_connection_003
17
+ end
18
+
19
+ describe '#registerd_connection_count' do
20
+ it "returns registerd total connection count" do
21
+ expect(config.registerd_connection_count).to eq 3
22
+ end
23
+ end
24
+
25
+ describe '#fetch' do
26
+ it "returns database connection name by modulo number" do
27
+ expect(config.fetch 0).to eq :db_connection_001
28
+ expect(config.fetch 1).to eq :db_connection_002
29
+ expect(config.fetch 2).to eq :db_connection_003
30
+ end
31
+ end
32
+ end
33
+
34
+ describe '#validate_config!' do
35
+ context "Nothing register connection" do
36
+ it "returns raise" do
37
+ expect { config.validate_config! }.to raise_error "Nothing registerd connections."
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,9 @@
1
+ describe ActiveRecord::Sharding::Error do
2
+ it "is defined class" do
3
+ expect(described_class).to be_a Class
4
+ end
5
+
6
+ it "kind of StandardError" do
7
+ expect(described_class.new).to be_kind_of StandardError
8
+ end
9
+ end
@@ -0,0 +1,90 @@
1
+ describe ActiveRecord::Sharding::Model do
2
+ let!(:model) do
3
+ Class.new(ActiveRecord::Base) do
4
+ def self.name
5
+ "User"
6
+ end
7
+
8
+ def self.sequence_id
9
+ @sequence_id ||= 0
10
+ end
11
+
12
+ class << self
13
+ attr_writer :sequence_id
14
+ end
15
+
16
+ def self.next_sequence_id
17
+ self.sequence_id += 1
18
+ end
19
+
20
+ include ActiveRecord::Sharding::Model
21
+ use_sharding :user, :modulo
22
+ define_sharding_key :id
23
+
24
+ before_put do |attrs|
25
+ attrs[:id] = next_sequence_id unless attrs[:id]
26
+ end
27
+
28
+ define_parent_methods do
29
+ def find_from_all_by_name(name)
30
+ all_shards.map { |m| m.find_by(name: name) }.compact.first
31
+ end
32
+ end
33
+ end
34
+ end
35
+
36
+ let(:alice) { model.put! name: "Alice" }
37
+
38
+ describe ".put!" do
39
+ it "example" do
40
+ expect(alice.persisted?).to be true
41
+ expect(alice.name).to eq "Alice"
42
+ expect(alice.class.name).to match(/User::ShardFor/)
43
+ end
44
+
45
+ context 'next call #put!' do
46
+ let(:bob) { model.put! name: "Bob" }
47
+ let(:alice_connect_db) { alice.class.connection.pool.spec.config[:database] }
48
+ let(:bob_connect_db) { bob.class.connection.pool.spec.config[:database] }
49
+
50
+ it "different connection database" do
51
+ expect(alice_connect_db).not_to eq bob_connect_db
52
+ end
53
+ end
54
+
55
+ context "Not included sharding key in args" do
56
+ before do
57
+ allow(User).to receive(:sharding_key).and_return(:dammy_sharding_key)
58
+ end
59
+
60
+ it "raise ActiveRecord::Sharding::MissingShardingKeyAttribute" do
61
+ expect do
62
+ User.put! name: "foobar"
63
+ end.to raise_error ActiveRecord::Sharding::MissingShardingKeyAttribute
64
+ end
65
+ end
66
+ end
67
+
68
+ describe ".shard_for" do
69
+ before { alice }
70
+
71
+ it "example" do
72
+ user_record = model.shard_for(alice.id).find_by name: "Alice"
73
+ expect(user_record).not_to be nil
74
+ expect(user_record.name).to eq "Alice"
75
+ end
76
+ end
77
+
78
+ describe ".define_parent_methods" do
79
+ before do
80
+ model.put! name: "foo"
81
+ model.put! name: "bar"
82
+ end
83
+
84
+ it "enables to define class methods to parent class" do
85
+ record = model.find_from_all_by_name("foo")
86
+ expect(record).not_to be_nil
87
+ expect(record.name).to eq("foo")
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,22 @@
1
+ describe ActiveRecord::Sharding::ModuloRouter do
2
+ let(:cluster_config) do
3
+ cluster_config = ActiveRecord::Sharding::ClusterConfig.new :user
4
+ cluster_config.register_connection :db_connection_001
5
+ cluster_config.register_connection :db_connection_002
6
+ cluster_config.register_connection :db_connection_003
7
+ cluster_config
8
+ end
9
+
10
+ let(:router) { described_class.new cluster_config }
11
+
12
+ describe '#route' do
13
+ it "returns database connection name by id % cluster nodes" do
14
+ expect(router.route(1)).to eq :db_connection_002
15
+ expect(router.route(2)).to eq :db_connection_003
16
+ expect(router.route(3)).to eq :db_connection_001
17
+ expect(router.route(4)).to eq :db_connection_002
18
+ expect(router.route(5)).to eq :db_connection_003
19
+ expect(router.route(6)).to eq :db_connection_001
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,31 @@
1
+ describe ActiveRecord::Sharding::Model do
2
+ let!(:model) do
3
+ Class.new(ActiveRecord::Base) do
4
+ def self.name
5
+ "User"
6
+ end
7
+
8
+ include ActiveRecord::Sharding::Sequencer
9
+ use_sequencer :user
10
+ end
11
+ end
12
+
13
+ describe '#current_sequence_id' do
14
+ it "returns current sequence id" do
15
+ expect(model.current_sequence_id).to be_a_kind_of Fixnum
16
+ end
17
+ end
18
+ describe '#next_sequence_id' do
19
+ let(:current_id) { model.current_sequence_id }
20
+ let(:next_id) { model.next_sequence_id }
21
+
22
+ it "returns next sequence id" do
23
+ expect(current_id + 1).to eq next_id
24
+ expect(next_id).to be_a_kind_of Fixnum
25
+ end
26
+
27
+ it "next sequence id > current sequence id" do
28
+ expect(current_id).to be < next_id
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,21 @@
1
+ describe ActiveRecord::Sharding::ShardRepository do
2
+ let(:cluster_config) { ActiveRecord::Sharding.config.fetch_cluster_config(:user) }
3
+ let(:shard_repository) { described_class.new cluster_config, User }
4
+
5
+ describe '#fetch' do
6
+ it "returns User class base shard object" do
7
+ expect(shard_repository.fetch(:test_user_001).connection).to be_a ActiveRecord::ConnectionAdapters::AbstractAdapter
8
+ expect(shard_repository.fetch(:test_user_001).table_name).to eq "users"
9
+ expect(shard_repository.fetch(:test_user_001).name).to eq "User::ShardForTestUser001"
10
+ end
11
+ end
12
+
13
+ describe '#all' do
14
+ it "returns all shard class objects" do
15
+ expect(shard_repository.all.count).to eq 3
16
+ expect(shard_repository.all[0]).to eq shard_repository.fetch(:test_user_001)
17
+ expect(shard_repository.all[1]).to eq shard_repository.fetch(:test_user_002)
18
+ expect(shard_repository.all[2]).to eq shard_repository.fetch(:test_user_003)
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,15 @@
1
+ describe ActiveRecord::Sharding do
2
+ it "has a version number" do
3
+ expect(ActiveRecord::Sharding::VERSION).not_to be nil
4
+ end
5
+
6
+ describe ".config" do
7
+ it "returns Config class" do
8
+ expect(ActiveRecord::Sharding.config).to be_a ActiveRecord::Sharding::Config
9
+ end
10
+
11
+ it "returns equal instance, every call" do
12
+ expect(ActiveRecord::Sharding.config).to eq ActiveRecord::Sharding.config
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,51 @@
1
+ base = { "adapter" => "mysql2", "encoding" => "utf8", "pool" => 5, "username" => "root", "password" => "", "host" => "localhost" }
2
+ ActiveRecord::Base.configurations = {
3
+ "test_user_001" => base.merge("database" => "user_001"),
4
+ "test_user_002" => base.merge("database" => "user_002"),
5
+ "test_user_003" => base.merge("database" => "user_003"),
6
+ "test_user_sequencer" => base.merge("database" => "user_sequencer"),
7
+ "test" => base.merge("database" => "default")
8
+ }
9
+ ActiveRecord::Base.establish_connection(:test)
10
+
11
+ ActiveRecord::Sharding.configure do |config|
12
+ config.define_cluster(:user) do |cluster|
13
+ cluster.register_connection(:test_user_001)
14
+ cluster.register_connection(:test_user_002)
15
+ cluster.register_connection(:test_user_003)
16
+ end
17
+
18
+ config.define_sequencer(:user) do |sequencer|
19
+ sequencer.register_connection(:test_user_sequencer)
20
+ sequencer.register_table_name("user_id")
21
+ end
22
+ end
23
+
24
+ class User < ActiveRecord::Base
25
+ def items
26
+ return [] unless id
27
+ Item.shard_for(id).where(user_id: id).all
28
+ end
29
+
30
+ include ActiveRecord::Sharding::Model
31
+ use_sharding :user, :modulo
32
+ define_sharding_key :id
33
+
34
+ include ActiveRecord::Sharding::Sequencer
35
+ use_sequencer :user
36
+
37
+ before_put do |attributes|
38
+ attributes[:id] = next_sequence_id unless attributes[:id]
39
+ end
40
+ end
41
+
42
+ class Item < ActiveRecord::Base
43
+ def user
44
+ return unless user_id
45
+ User.shard_for(user_id).find_by user_id
46
+ end
47
+
48
+ include ActiveRecord::Sharding::Model
49
+ use_sharding :user, :modulo
50
+ define_sharding_key :user_id
51
+ end