activerecord-sharding 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.hound.yml +2 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +50 -0
  6. data/.ruby-version +1 -0
  7. data/.travis.yml +10 -0
  8. data/Gemfile +4 -0
  9. data/README.md +184 -0
  10. data/Rakefile +6 -0
  11. data/activerecord-sharding.gemspec +35 -0
  12. data/bin/benchmark_sequencer.rb +71 -0
  13. data/bin/console +10 -0
  14. data/bin/setup +7 -0
  15. data/lib/active_record/sharding.rb +0 -0
  16. data/lib/active_record/sharding/abstract_repository.rb +29 -0
  17. data/lib/active_record/sharding/cluster_config.rb +32 -0
  18. data/lib/active_record/sharding/config.rb +34 -0
  19. data/lib/active_record/sharding/database_tasks.rb +234 -0
  20. data/lib/active_record/sharding/errors.rb +9 -0
  21. data/lib/active_record/sharding/model.rb +59 -0
  22. data/lib/active_record/sharding/modulo_router.rb +14 -0
  23. data/lib/active_record/sharding/railtie.rb +9 -0
  24. data/lib/active_record/sharding/sequencer.rb +40 -0
  25. data/lib/active_record/sharding/sequencer_config.rb +26 -0
  26. data/lib/active_record/sharding/sequencer_repository.rb +22 -0
  27. data/lib/active_record/sharding/shard_repository.rb +31 -0
  28. data/lib/active_record/sharding/version.rb +5 -0
  29. data/lib/activerecord-sharding.rb +30 -0
  30. data/lib/tasks/activerecord-sharding.rake +88 -0
  31. data/spec/active_record/sharding/abstract_repository_spec.rb +15 -0
  32. data/spec/active_record/sharding/cluster_config_spec.rb +41 -0
  33. data/spec/active_record/sharding/errors_spec.rb +9 -0
  34. data/spec/active_record/sharding/model_spec.rb +90 -0
  35. data/spec/active_record/sharding/modulo_router_spec.rb +22 -0
  36. data/spec/active_record/sharding/sequencer_spec.rb +31 -0
  37. data/spec/active_record/sharding/shard_repository_spec.rb +21 -0
  38. data/spec/active_record_sharding_spec.rb +15 -0
  39. data/spec/models.rb +51 -0
  40. data/spec/schema.rb +17 -0
  41. data/spec/spec_helper.rb +63 -0
  42. data/spec/tasks/activerecord-sharding_spec.rb +74 -0
  43. metadata +254 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 6161f4f763503556924e2678e30f8b8dd35877dd
4
+ data.tar.gz: 345928fff20f5bf81e6a85740822b125770ec2a8
5
+ SHA512:
6
+ metadata.gz: 6044b78dcf067016e87bc738c927efd62a8f5406ed8952b3df9d8f032bda50e795af0d71eedbdce12bfd4d2cd177e46d60665f0907ff9e93742f195a23559556
7
+ data.tar.gz: 329ee41bf7df37ce0302fa8a9338ede775782871197f6f87f99ae589d28c326bae538ff39980f7a087ba31fec4484ab5622dce93c6fe0c1c7897d0e30cc3d6ef
@@ -0,0 +1,11 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ /vendor/
11
+ /log/
@@ -0,0 +1,2 @@
1
+ ruby:
2
+ config_file: .rubocop.yml
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --require spec_helper
2
+ --format documentation
3
+ --color
@@ -0,0 +1,50 @@
1
+ AllCops:
2
+ Exclude:
3
+ - "vendor/**/*"
4
+ DisplayCopNames: true
5
+
6
+ Style/AsciiComments:
7
+ Enabled: false
8
+
9
+ Style/BracesAroundHashParameters:
10
+ Enabled: false
11
+
12
+ Style/Documentation:
13
+ Enabled: false
14
+
15
+ Style/ExtraSpacing:
16
+ Enabled: false
17
+
18
+ Style/GuardClause:
19
+ MinBodyLength: 5
20
+
21
+ Style/MethodDefParentheses:
22
+ EnforcedStyle: require_parentheses
23
+
24
+ Style/ModuleFunction:
25
+ Enabled: false
26
+
27
+ Style/StringLiterals:
28
+ EnforcedStyle: double_quotes
29
+
30
+ Style/HashSyntax:
31
+ EnforcedStyle: ruby19_no_mixed_keys
32
+ Exclude:
33
+ - "**/*.rake"
34
+ - "Rakefile"
35
+
36
+ Metrics/LineLength:
37
+ Max: 160
38
+ Exclude:
39
+ - "db/migrate/*.rb"
40
+
41
+ Lint/AssignmentInCondition:
42
+ Enabled: false
43
+
44
+ Metrics/MethodLength:
45
+ Max: 12
46
+
47
+ FileName:
48
+ Exclude:
49
+ - 'lib/activerecord-sharding.rb'
50
+ - 'spec/tasks/activerecord-sharding_spec.rb'
@@ -0,0 +1 @@
1
+ 2.2.3
@@ -0,0 +1,10 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.2.3
4
+ - 2.1.7
5
+ - 2.0.0-p647
6
+ before_install: gem install bundler -v 1.10.6
7
+ services:
8
+ - mysql
9
+ script:
10
+ - bundle exec rake spec
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "https://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in active_record_sharding.gemspec
4
+ gemspec
@@ -0,0 +1,184 @@
1
+ # ActiveRecord::Sharding
2
+
3
+ [![Build Status](https://travis-ci.org/hirocaster/activerecord-sharding.svg)](https://travis-ci.org/hirocaster/activerecord-sharding) [![Coverage Status](https://coveralls.io/repos/hirocaster/activerecord-sharding/badge.svg?branch=coveralls&service=github)](https://coveralls.io/github/hirocaster/activerecord-sharding?branch=coveralls) [![Code Climate](https://codeclimate.com/github/hirocaster/activerecord-sharding/badges/gpa.svg)](https://codeclimate.com/github/hirocaster/activerecord-sharding) [![Dependency Status](https://gemnasium.com/hirocaster/activerecord-sharding.svg)](https://gemnasium.com/hirocaster/activerecord-sharding)
4
+
5
+ Simple Database Sharding in ActiveRecord.
6
+
7
+ ActiveRecord object distributed multiple databases by modulo.
8
+ Modulo target is id, generated by sequencer database.
9
+
10
+ ## Support database
11
+
12
+ - MySQL
13
+
14
+ ## Installation
15
+
16
+ Add this line to your application's Gemfile:
17
+
18
+ gem 'activerecord-sharding'
19
+
20
+ And then execute:
21
+
22
+ $ bundle
23
+
24
+ Or install it yourself as:
25
+
26
+ $ gem install activerecord-sharding
27
+
28
+ ## Usage
29
+
30
+ Add database connections to your application's config/database.yml:
31
+
32
+ ```yaml
33
+ default: &default
34
+ adapter: mysql2
35
+ encoding: utf8
36
+ pool: 5
37
+ username: root
38
+ password:
39
+ host: localhost
40
+
41
+ user_sequencer:
42
+ <<: *default
43
+ database: user_001
44
+ host: localhost
45
+
46
+ user_001:
47
+ <<: *default
48
+ database: user_001
49
+ host: localhost
50
+
51
+ user_002:
52
+ <<: *default
53
+ database: user_002
54
+ host: localhost
55
+
56
+ user_003:
57
+ <<: *default
58
+ database: user_003
59
+ host: localhost
60
+ ```
61
+
62
+ Add this example your application's config/initializers/active_record_sharding.rb:
63
+
64
+ ```ruby
65
+ ActiveRecord::Sharding.configure do |config|
66
+ config.define_sequencer(:user) do |sequencer|
67
+ sequencer.register_connection(:user_sequencer)
68
+ sequencer.register_table_name('user_id')
69
+ end
70
+
71
+ config.define_cluster(:user) do |cluster|
72
+ cluster.register_connection(:user_001)
73
+ cluster.register_connection(:user_002)
74
+ cluster.register_connection(:user_003)
75
+ end
76
+ end
77
+ ```
78
+
79
+ - define user cluster config
80
+ - define user sequencer config
81
+
82
+ ### Model
83
+
84
+ app/model/user.rb
85
+
86
+ ```ruby
87
+ class User < ActiveRecord::Base
88
+ include ActiveRecord::Sharding::Model
89
+ use_sharding :user, :modulo # shard name, algorithm
90
+ define_sharding_key :id
91
+
92
+ include ActiveRecord::Sharding::Sequencer
93
+ use_sequencer :user
94
+
95
+ before_put do |attributes|
96
+ attributes[:id] = next_sequence_id unless attributes[:id]
97
+ end
98
+ end
99
+ ```
100
+
101
+
102
+ ### Create sequencer databases
103
+
104
+ ```ruby
105
+ $ rake active_record:sharding:sequencer:setup
106
+ ```
107
+
108
+ ### Create cluster dtabases
109
+
110
+ ```ruby
111
+ $ rake active_record:sharding:setup
112
+ ```
113
+
114
+ and, migrations all cluster databases.
115
+
116
+ ### in applications
117
+
118
+ #### Create
119
+
120
+ using `#put!` method.
121
+
122
+ ```ruby
123
+ user = User.put! name: 'foobar'
124
+ ```
125
+
126
+ returns User new object.
127
+
128
+ #### Select Query
129
+
130
+ ```ruby
131
+ sharding_key = user.id
132
+ User.shard_for(sharding_key).where(name: 'foorbar')
133
+ ```
134
+
135
+ `sharding_key` is your define_syarding_key.(example is User Object id)
136
+
137
+ `#sahrd_for` is returns User class.
138
+
139
+ #### Association/Relation
140
+
141
+ if use database association/relation in sharding databases.
142
+
143
+ Please, don't use ActiveRecord standard associations/relation features(has_may, has_one, belongs_to... etc).because, it using `ActiveRecord::Base.connection`(not sharding databases conneciton).
144
+
145
+ Please, manually add association/relation methods.
146
+
147
+ Bad sample
148
+
149
+ ```
150
+ class User < ActiveRecord::Base
151
+ has_many :items # connect to not sharding databases(default database)
152
+
153
+ include ActiveRecord::Sharding::Model
154
+ use_sharding :user, :modulo
155
+ define_sharding_key :id
156
+ # (snip)
157
+ end
158
+ ```
159
+
160
+ Manually add method
161
+
162
+ ```
163
+ class User < ActiveRecord::Base
164
+ def items
165
+ return [] unless id
166
+ Item.shard_for(id).where(user_id: id).all
167
+ end
168
+
169
+ include ActiveRecord::Sharding::Model
170
+ use_sharding :user, :modulo
171
+ define_sharding_key :id
172
+ # (snip)
173
+ end
174
+ ```
175
+
176
+ ## Development
177
+
178
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
179
+
180
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
181
+
182
+ ## Contributing
183
+
184
+ Bug reports and pull requests are welcome on GitHub at https://github.com/hirocaster/activerecord-sharding.
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,35 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path("../lib", __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require "active_record/sharding/version"
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "activerecord-sharding"
8
+ spec.version = ActiveRecord::Sharding::VERSION
9
+ spec.authors = ["hirocaster"]
10
+ spec.email = ["hohtsuka@gmail.com"]
11
+ spec.summary = "Sharding library for ActiveRecord(MySQL)"
12
+ spec.description = "Sharding library for ActiveRecord(MySQL)"
13
+ spec.homepage = "https://github.com/hirocaster/activerecord-sharding"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.required_ruby_version = ">= 2.0"
22
+
23
+ spec.add_dependency "activerecord", ">= 4.1.0"
24
+
25
+ spec.add_development_dependency "bundler", "~> 1.5"
26
+ spec.add_development_dependency "rake"
27
+ spec.add_development_dependency "rspec"
28
+ spec.add_development_dependency "mysql2"
29
+ spec.add_development_dependency "pry"
30
+ spec.add_development_dependency "pry-byebug"
31
+ spec.add_development_dependency "awesome_print"
32
+ spec.add_development_dependency "simplecov"
33
+ spec.add_development_dependency "coveralls"
34
+ spec.add_development_dependency "codeclimate-test-reporter"
35
+ end
@@ -0,0 +1,71 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "active_record_sharding"
5
+
6
+ require_relative "../spec/models"
7
+
8
+ require "benchmark"
9
+ require "pry"
10
+
11
+ GENERATE_ID_COUNT = 100_000
12
+
13
+ def before_benchmark
14
+ setup_database_env
15
+
16
+ back, $stdout = $stdout, StringIO.new
17
+ setup_shard_cluster_databases
18
+ setup_sequence_database
19
+ $stdout = back
20
+
21
+ ActiveRecord::Base.establish_connection(:test)
22
+
23
+ unless User.current_sequence_id == 0
24
+ puts "Fail: Start sequence id is not zero."
25
+ exit
26
+ end
27
+ end
28
+
29
+ def setup_database_env
30
+ ActiveRecord::Tasks::DatabaseTasks.db_dir = File.expand_path "../../spec", __FILE__
31
+ ActiveRecord::Tasks::DatabaseTasks.root = File.expand_path "../..", __FILE__
32
+ ActiveRecord::Tasks::DatabaseTasks.env = "test"
33
+ end
34
+
35
+ def setup_shard_cluster_databases
36
+ args = { cluster_name: "user" }
37
+ ActiveRecordSharding::DatabaseTasks.drop_all_databases args
38
+ ActiveRecordSharding::DatabaseTasks.create_all_databases args
39
+ ActiveRecordSharding::DatabaseTasks.load_schema_all_databases args
40
+ end
41
+
42
+ def setup_sequence_database
43
+ sequencer_args = { sequencer_name: "user" }
44
+ ActiveRecordSharding::DatabaseTasks.drop_sequencer_database sequencer_args
45
+ ActiveRecordSharding::DatabaseTasks.create_sequencer_database sequencer_args
46
+ ActiveRecordSharding::DatabaseTasks.create_table_sequencer_database sequencer_args
47
+ ActiveRecordSharding::DatabaseTasks.insert_initial_record_sequencer_database sequencer_args
48
+ end
49
+
50
+ def after_benchmark
51
+ if User.current_sequence_id != GENERATE_ID_COUNT
52
+ puts "Fail: End sequence id is not #{GENERATE_ID_COUNT}."
53
+ end
54
+
55
+ ActiveRecordSharding::DatabaseTasks.drop_all_databases cluster_name: "user"
56
+
57
+ sequencer_args = { sequencer_name: "user" }
58
+ ActiveRecordSharding::DatabaseTasks.drop_sequencer_database sequencer_args
59
+ end
60
+
61
+ before_benchmark
62
+
63
+ puts "===== Benchmark sequence id (#{GENERATE_ID_COUNT}) ====="
64
+ puts Benchmark::CAPTION
65
+ puts Benchmark.measure {
66
+ 100_000.times do
67
+ User.next_sequence_id
68
+ end
69
+ }
70
+
71
+ after_benchmark
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "activerecord-sharding"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ require "pry"
10
+ Pry.start
@@ -0,0 +1,7 @@
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+
5
+ bundle install
6
+
7
+ # Do any other automated setup that you need to do here
File without changes
@@ -0,0 +1,29 @@
1
+ module ActiveRecord
2
+ module Sharding
3
+ class AbstractRepository
4
+ private
5
+
6
+ def generate_model_for_shard(connection_name)
7
+ base_class_name = @base_class.name
8
+ class_name = generate_class_name connection_name
9
+
10
+ model = Class.new(base_class) do
11
+ self.table_name = base_class.table_name
12
+
13
+ module_eval <<-RUBY, __FILE__, __LINE__ + 1
14
+ def self.name
15
+ "#{base_class_name}::#{class_name}"
16
+ end
17
+ RUBY
18
+ end
19
+
20
+ model.class_eval { establish_connection(connection_name) }
21
+ model
22
+ end
23
+
24
+ def generate_class_name(connection_name) # rubocop:disable Lint/UnusedMethodArgument
25
+ fail NotImplementedError, "#{self.class.name}.#{__method__} is an abstract method."
26
+ end
27
+ end
28
+ end
29
+ end