activerecord-sharding 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.hound.yml +2 -0
- data/.rspec +3 -0
- data/.rubocop.yml +50 -0
- data/.ruby-version +1 -0
- data/.travis.yml +10 -0
- data/Gemfile +4 -0
- data/README.md +184 -0
- data/Rakefile +6 -0
- data/activerecord-sharding.gemspec +35 -0
- data/bin/benchmark_sequencer.rb +71 -0
- data/bin/console +10 -0
- data/bin/setup +7 -0
- data/lib/active_record/sharding.rb +0 -0
- data/lib/active_record/sharding/abstract_repository.rb +29 -0
- data/lib/active_record/sharding/cluster_config.rb +32 -0
- data/lib/active_record/sharding/config.rb +34 -0
- data/lib/active_record/sharding/database_tasks.rb +234 -0
- data/lib/active_record/sharding/errors.rb +9 -0
- data/lib/active_record/sharding/model.rb +59 -0
- data/lib/active_record/sharding/modulo_router.rb +14 -0
- data/lib/active_record/sharding/railtie.rb +9 -0
- data/lib/active_record/sharding/sequencer.rb +40 -0
- data/lib/active_record/sharding/sequencer_config.rb +26 -0
- data/lib/active_record/sharding/sequencer_repository.rb +22 -0
- data/lib/active_record/sharding/shard_repository.rb +31 -0
- data/lib/active_record/sharding/version.rb +5 -0
- data/lib/activerecord-sharding.rb +30 -0
- data/lib/tasks/activerecord-sharding.rake +88 -0
- data/spec/active_record/sharding/abstract_repository_spec.rb +15 -0
- data/spec/active_record/sharding/cluster_config_spec.rb +41 -0
- data/spec/active_record/sharding/errors_spec.rb +9 -0
- data/spec/active_record/sharding/model_spec.rb +90 -0
- data/spec/active_record/sharding/modulo_router_spec.rb +22 -0
- data/spec/active_record/sharding/sequencer_spec.rb +31 -0
- data/spec/active_record/sharding/shard_repository_spec.rb +21 -0
- data/spec/active_record_sharding_spec.rb +15 -0
- data/spec/models.rb +51 -0
- data/spec/schema.rb +17 -0
- data/spec/spec_helper.rb +63 -0
- data/spec/tasks/activerecord-sharding_spec.rb +74 -0
- metadata +254 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 6161f4f763503556924e2678e30f8b8dd35877dd
|
4
|
+
data.tar.gz: 345928fff20f5bf81e6a85740822b125770ec2a8
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6044b78dcf067016e87bc738c927efd62a8f5406ed8952b3df9d8f032bda50e795af0d71eedbdce12bfd4d2cd177e46d60665f0907ff9e93742f195a23559556
|
7
|
+
data.tar.gz: 329ee41bf7df37ce0302fa8a9338ede775782871197f6f87f99ae589d28c326bae538ff39980f7a087ba31fec4484ab5622dce93c6fe0c1c7897d0e30cc3d6ef
|
data/.gitignore
ADDED
data/.hound.yml
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
AllCops:
|
2
|
+
Exclude:
|
3
|
+
- "vendor/**/*"
|
4
|
+
DisplayCopNames: true
|
5
|
+
|
6
|
+
Style/AsciiComments:
|
7
|
+
Enabled: false
|
8
|
+
|
9
|
+
Style/BracesAroundHashParameters:
|
10
|
+
Enabled: false
|
11
|
+
|
12
|
+
Style/Documentation:
|
13
|
+
Enabled: false
|
14
|
+
|
15
|
+
Style/ExtraSpacing:
|
16
|
+
Enabled: false
|
17
|
+
|
18
|
+
Style/GuardClause:
|
19
|
+
MinBodyLength: 5
|
20
|
+
|
21
|
+
Style/MethodDefParentheses:
|
22
|
+
EnforcedStyle: require_parentheses
|
23
|
+
|
24
|
+
Style/ModuleFunction:
|
25
|
+
Enabled: false
|
26
|
+
|
27
|
+
Style/StringLiterals:
|
28
|
+
EnforcedStyle: double_quotes
|
29
|
+
|
30
|
+
Style/HashSyntax:
|
31
|
+
EnforcedStyle: ruby19_no_mixed_keys
|
32
|
+
Exclude:
|
33
|
+
- "**/*.rake"
|
34
|
+
- "Rakefile"
|
35
|
+
|
36
|
+
Metrics/LineLength:
|
37
|
+
Max: 160
|
38
|
+
Exclude:
|
39
|
+
- "db/migrate/*.rb"
|
40
|
+
|
41
|
+
Lint/AssignmentInCondition:
|
42
|
+
Enabled: false
|
43
|
+
|
44
|
+
Metrics/MethodLength:
|
45
|
+
Max: 12
|
46
|
+
|
47
|
+
FileName:
|
48
|
+
Exclude:
|
49
|
+
- 'lib/activerecord-sharding.rb'
|
50
|
+
- 'spec/tasks/activerecord-sharding_spec.rb'
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.2.3
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,184 @@
|
|
1
|
+
# ActiveRecord::Sharding
|
2
|
+
|
3
|
+
[](https://travis-ci.org/hirocaster/activerecord-sharding) [](https://coveralls.io/github/hirocaster/activerecord-sharding?branch=coveralls) [](https://codeclimate.com/github/hirocaster/activerecord-sharding) [](https://gemnasium.com/hirocaster/activerecord-sharding)
|
4
|
+
|
5
|
+
Simple Database Sharding in ActiveRecord.
|
6
|
+
|
7
|
+
ActiveRecord object distributed multiple databases by modulo.
|
8
|
+
Modulo target is id, generated by sequencer database.
|
9
|
+
|
10
|
+
## Support database
|
11
|
+
|
12
|
+
- MySQL
|
13
|
+
|
14
|
+
## Installation
|
15
|
+
|
16
|
+
Add this line to your application's Gemfile:
|
17
|
+
|
18
|
+
gem 'activerecord-sharding'
|
19
|
+
|
20
|
+
And then execute:
|
21
|
+
|
22
|
+
$ bundle
|
23
|
+
|
24
|
+
Or install it yourself as:
|
25
|
+
|
26
|
+
$ gem install activerecord-sharding
|
27
|
+
|
28
|
+
## Usage
|
29
|
+
|
30
|
+
Add database connections to your application's config/database.yml:
|
31
|
+
|
32
|
+
```yaml
|
33
|
+
default: &default
|
34
|
+
adapter: mysql2
|
35
|
+
encoding: utf8
|
36
|
+
pool: 5
|
37
|
+
username: root
|
38
|
+
password:
|
39
|
+
host: localhost
|
40
|
+
|
41
|
+
user_sequencer:
|
42
|
+
<<: *default
|
43
|
+
database: user_001
|
44
|
+
host: localhost
|
45
|
+
|
46
|
+
user_001:
|
47
|
+
<<: *default
|
48
|
+
database: user_001
|
49
|
+
host: localhost
|
50
|
+
|
51
|
+
user_002:
|
52
|
+
<<: *default
|
53
|
+
database: user_002
|
54
|
+
host: localhost
|
55
|
+
|
56
|
+
user_003:
|
57
|
+
<<: *default
|
58
|
+
database: user_003
|
59
|
+
host: localhost
|
60
|
+
```
|
61
|
+
|
62
|
+
Add this example your application's config/initializers/active_record_sharding.rb:
|
63
|
+
|
64
|
+
```ruby
|
65
|
+
ActiveRecord::Sharding.configure do |config|
|
66
|
+
config.define_sequencer(:user) do |sequencer|
|
67
|
+
sequencer.register_connection(:user_sequencer)
|
68
|
+
sequencer.register_table_name('user_id')
|
69
|
+
end
|
70
|
+
|
71
|
+
config.define_cluster(:user) do |cluster|
|
72
|
+
cluster.register_connection(:user_001)
|
73
|
+
cluster.register_connection(:user_002)
|
74
|
+
cluster.register_connection(:user_003)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
```
|
78
|
+
|
79
|
+
- define user cluster config
|
80
|
+
- define user sequencer config
|
81
|
+
|
82
|
+
### Model
|
83
|
+
|
84
|
+
app/model/user.rb
|
85
|
+
|
86
|
+
```ruby
|
87
|
+
class User < ActiveRecord::Base
|
88
|
+
include ActiveRecord::Sharding::Model
|
89
|
+
use_sharding :user, :modulo # shard name, algorithm
|
90
|
+
define_sharding_key :id
|
91
|
+
|
92
|
+
include ActiveRecord::Sharding::Sequencer
|
93
|
+
use_sequencer :user
|
94
|
+
|
95
|
+
before_put do |attributes|
|
96
|
+
attributes[:id] = next_sequence_id unless attributes[:id]
|
97
|
+
end
|
98
|
+
end
|
99
|
+
```
|
100
|
+
|
101
|
+
|
102
|
+
### Create sequencer databases
|
103
|
+
|
104
|
+
```ruby
|
105
|
+
$ rake active_record:sharding:sequencer:setup
|
106
|
+
```
|
107
|
+
|
108
|
+
### Create cluster dtabases
|
109
|
+
|
110
|
+
```ruby
|
111
|
+
$ rake active_record:sharding:setup
|
112
|
+
```
|
113
|
+
|
114
|
+
and, migrations all cluster databases.
|
115
|
+
|
116
|
+
### in applications
|
117
|
+
|
118
|
+
#### Create
|
119
|
+
|
120
|
+
using `#put!` method.
|
121
|
+
|
122
|
+
```ruby
|
123
|
+
user = User.put! name: 'foobar'
|
124
|
+
```
|
125
|
+
|
126
|
+
returns User new object.
|
127
|
+
|
128
|
+
#### Select Query
|
129
|
+
|
130
|
+
```ruby
|
131
|
+
sharding_key = user.id
|
132
|
+
User.shard_for(sharding_key).where(name: 'foorbar')
|
133
|
+
```
|
134
|
+
|
135
|
+
`sharding_key` is your define_syarding_key.(example is User Object id)
|
136
|
+
|
137
|
+
`#sahrd_for` is returns User class.
|
138
|
+
|
139
|
+
#### Association/Relation
|
140
|
+
|
141
|
+
if use database association/relation in sharding databases.
|
142
|
+
|
143
|
+
Please, don't use ActiveRecord standard associations/relation features(has_may, has_one, belongs_to... etc).because, it using `ActiveRecord::Base.connection`(not sharding databases conneciton).
|
144
|
+
|
145
|
+
Please, manually add association/relation methods.
|
146
|
+
|
147
|
+
Bad sample
|
148
|
+
|
149
|
+
```
|
150
|
+
class User < ActiveRecord::Base
|
151
|
+
has_many :items # connect to not sharding databases(default database)
|
152
|
+
|
153
|
+
include ActiveRecord::Sharding::Model
|
154
|
+
use_sharding :user, :modulo
|
155
|
+
define_sharding_key :id
|
156
|
+
# (snip)
|
157
|
+
end
|
158
|
+
```
|
159
|
+
|
160
|
+
Manually add method
|
161
|
+
|
162
|
+
```
|
163
|
+
class User < ActiveRecord::Base
|
164
|
+
def items
|
165
|
+
return [] unless id
|
166
|
+
Item.shard_for(id).where(user_id: id).all
|
167
|
+
end
|
168
|
+
|
169
|
+
include ActiveRecord::Sharding::Model
|
170
|
+
use_sharding :user, :modulo
|
171
|
+
define_sharding_key :id
|
172
|
+
# (snip)
|
173
|
+
end
|
174
|
+
```
|
175
|
+
|
176
|
+
## Development
|
177
|
+
|
178
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
179
|
+
|
180
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
181
|
+
|
182
|
+
## Contributing
|
183
|
+
|
184
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/hirocaster/activerecord-sharding.
|
data/Rakefile
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "active_record/sharding/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "activerecord-sharding"
|
8
|
+
spec.version = ActiveRecord::Sharding::VERSION
|
9
|
+
spec.authors = ["hirocaster"]
|
10
|
+
spec.email = ["hohtsuka@gmail.com"]
|
11
|
+
spec.summary = "Sharding library for ActiveRecord(MySQL)"
|
12
|
+
spec.description = "Sharding library for ActiveRecord(MySQL)"
|
13
|
+
spec.homepage = "https://github.com/hirocaster/activerecord-sharding"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.required_ruby_version = ">= 2.0"
|
22
|
+
|
23
|
+
spec.add_dependency "activerecord", ">= 4.1.0"
|
24
|
+
|
25
|
+
spec.add_development_dependency "bundler", "~> 1.5"
|
26
|
+
spec.add_development_dependency "rake"
|
27
|
+
spec.add_development_dependency "rspec"
|
28
|
+
spec.add_development_dependency "mysql2"
|
29
|
+
spec.add_development_dependency "pry"
|
30
|
+
spec.add_development_dependency "pry-byebug"
|
31
|
+
spec.add_development_dependency "awesome_print"
|
32
|
+
spec.add_development_dependency "simplecov"
|
33
|
+
spec.add_development_dependency "coveralls"
|
34
|
+
spec.add_development_dependency "codeclimate-test-reporter"
|
35
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "active_record_sharding"
|
5
|
+
|
6
|
+
require_relative "../spec/models"
|
7
|
+
|
8
|
+
require "benchmark"
|
9
|
+
require "pry"
|
10
|
+
|
11
|
+
GENERATE_ID_COUNT = 100_000
|
12
|
+
|
13
|
+
def before_benchmark
|
14
|
+
setup_database_env
|
15
|
+
|
16
|
+
back, $stdout = $stdout, StringIO.new
|
17
|
+
setup_shard_cluster_databases
|
18
|
+
setup_sequence_database
|
19
|
+
$stdout = back
|
20
|
+
|
21
|
+
ActiveRecord::Base.establish_connection(:test)
|
22
|
+
|
23
|
+
unless User.current_sequence_id == 0
|
24
|
+
puts "Fail: Start sequence id is not zero."
|
25
|
+
exit
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def setup_database_env
|
30
|
+
ActiveRecord::Tasks::DatabaseTasks.db_dir = File.expand_path "../../spec", __FILE__
|
31
|
+
ActiveRecord::Tasks::DatabaseTasks.root = File.expand_path "../..", __FILE__
|
32
|
+
ActiveRecord::Tasks::DatabaseTasks.env = "test"
|
33
|
+
end
|
34
|
+
|
35
|
+
def setup_shard_cluster_databases
|
36
|
+
args = { cluster_name: "user" }
|
37
|
+
ActiveRecordSharding::DatabaseTasks.drop_all_databases args
|
38
|
+
ActiveRecordSharding::DatabaseTasks.create_all_databases args
|
39
|
+
ActiveRecordSharding::DatabaseTasks.load_schema_all_databases args
|
40
|
+
end
|
41
|
+
|
42
|
+
def setup_sequence_database
|
43
|
+
sequencer_args = { sequencer_name: "user" }
|
44
|
+
ActiveRecordSharding::DatabaseTasks.drop_sequencer_database sequencer_args
|
45
|
+
ActiveRecordSharding::DatabaseTasks.create_sequencer_database sequencer_args
|
46
|
+
ActiveRecordSharding::DatabaseTasks.create_table_sequencer_database sequencer_args
|
47
|
+
ActiveRecordSharding::DatabaseTasks.insert_initial_record_sequencer_database sequencer_args
|
48
|
+
end
|
49
|
+
|
50
|
+
def after_benchmark
|
51
|
+
if User.current_sequence_id != GENERATE_ID_COUNT
|
52
|
+
puts "Fail: End sequence id is not #{GENERATE_ID_COUNT}."
|
53
|
+
end
|
54
|
+
|
55
|
+
ActiveRecordSharding::DatabaseTasks.drop_all_databases cluster_name: "user"
|
56
|
+
|
57
|
+
sequencer_args = { sequencer_name: "user" }
|
58
|
+
ActiveRecordSharding::DatabaseTasks.drop_sequencer_database sequencer_args
|
59
|
+
end
|
60
|
+
|
61
|
+
before_benchmark
|
62
|
+
|
63
|
+
puts "===== Benchmark sequence id (#{GENERATE_ID_COUNT}) ====="
|
64
|
+
puts Benchmark::CAPTION
|
65
|
+
puts Benchmark.measure {
|
66
|
+
100_000.times do
|
67
|
+
User.next_sequence_id
|
68
|
+
end
|
69
|
+
}
|
70
|
+
|
71
|
+
after_benchmark
|
data/bin/console
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "activerecord-sharding"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
require "pry"
|
10
|
+
Pry.start
|
data/bin/setup
ADDED
File without changes
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module ActiveRecord
|
2
|
+
module Sharding
|
3
|
+
class AbstractRepository
|
4
|
+
private
|
5
|
+
|
6
|
+
def generate_model_for_shard(connection_name)
|
7
|
+
base_class_name = @base_class.name
|
8
|
+
class_name = generate_class_name connection_name
|
9
|
+
|
10
|
+
model = Class.new(base_class) do
|
11
|
+
self.table_name = base_class.table_name
|
12
|
+
|
13
|
+
module_eval <<-RUBY, __FILE__, __LINE__ + 1
|
14
|
+
def self.name
|
15
|
+
"#{base_class_name}::#{class_name}"
|
16
|
+
end
|
17
|
+
RUBY
|
18
|
+
end
|
19
|
+
|
20
|
+
model.class_eval { establish_connection(connection_name) }
|
21
|
+
model
|
22
|
+
end
|
23
|
+
|
24
|
+
def generate_class_name(connection_name) # rubocop:disable Lint/UnusedMethodArgument
|
25
|
+
fail NotImplementedError, "#{self.class.name}.#{__method__} is an abstract method."
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|