activerecord-sharding 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.hound.yml +2 -0
- data/.rspec +3 -0
- data/.rubocop.yml +50 -0
- data/.ruby-version +1 -0
- data/.travis.yml +10 -0
- data/Gemfile +4 -0
- data/README.md +184 -0
- data/Rakefile +6 -0
- data/activerecord-sharding.gemspec +35 -0
- data/bin/benchmark_sequencer.rb +71 -0
- data/bin/console +10 -0
- data/bin/setup +7 -0
- data/lib/active_record/sharding.rb +0 -0
- data/lib/active_record/sharding/abstract_repository.rb +29 -0
- data/lib/active_record/sharding/cluster_config.rb +32 -0
- data/lib/active_record/sharding/config.rb +34 -0
- data/lib/active_record/sharding/database_tasks.rb +234 -0
- data/lib/active_record/sharding/errors.rb +9 -0
- data/lib/active_record/sharding/model.rb +59 -0
- data/lib/active_record/sharding/modulo_router.rb +14 -0
- data/lib/active_record/sharding/railtie.rb +9 -0
- data/lib/active_record/sharding/sequencer.rb +40 -0
- data/lib/active_record/sharding/sequencer_config.rb +26 -0
- data/lib/active_record/sharding/sequencer_repository.rb +22 -0
- data/lib/active_record/sharding/shard_repository.rb +31 -0
- data/lib/active_record/sharding/version.rb +5 -0
- data/lib/activerecord-sharding.rb +30 -0
- data/lib/tasks/activerecord-sharding.rake +88 -0
- data/spec/active_record/sharding/abstract_repository_spec.rb +15 -0
- data/spec/active_record/sharding/cluster_config_spec.rb +41 -0
- data/spec/active_record/sharding/errors_spec.rb +9 -0
- data/spec/active_record/sharding/model_spec.rb +90 -0
- data/spec/active_record/sharding/modulo_router_spec.rb +22 -0
- data/spec/active_record/sharding/sequencer_spec.rb +31 -0
- data/spec/active_record/sharding/shard_repository_spec.rb +21 -0
- data/spec/active_record_sharding_spec.rb +15 -0
- data/spec/models.rb +51 -0
- data/spec/schema.rb +17 -0
- data/spec/spec_helper.rb +63 -0
- data/spec/tasks/activerecord-sharding_spec.rb +74 -0
- metadata +254 -0
@@ -0,0 +1,32 @@
|
|
1
|
+
module ActiveRecord
|
2
|
+
module Sharding
|
3
|
+
class ClusterConfig
|
4
|
+
attr_reader :name
|
5
|
+
|
6
|
+
def initialize(name)
|
7
|
+
@name = name
|
8
|
+
@connection_registry = []
|
9
|
+
end
|
10
|
+
|
11
|
+
def register_connection(connection_name)
|
12
|
+
@connection_registry << connection_name
|
13
|
+
end
|
14
|
+
|
15
|
+
def fetch(modulo_key)
|
16
|
+
@connection_registry[modulo_key]
|
17
|
+
end
|
18
|
+
|
19
|
+
def registerd_connection_count
|
20
|
+
@connection_registry.count
|
21
|
+
end
|
22
|
+
|
23
|
+
def validate_config!
|
24
|
+
fail "Nothing registerd connections." if registerd_connection_count == 0
|
25
|
+
end
|
26
|
+
|
27
|
+
def connections
|
28
|
+
@connection_registry
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module ActiveRecord
|
2
|
+
module Sharding
|
3
|
+
class Config
|
4
|
+
attr_reader :cluster_configs, :sequencer_configs
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@cluster_configs = {}
|
8
|
+
@sequencer_configs = {}
|
9
|
+
end
|
10
|
+
|
11
|
+
def define_cluster(cluster_name, &block)
|
12
|
+
cluster_config = ClusterConfig.new(cluster_name)
|
13
|
+
cluster_config.instance_eval(&block)
|
14
|
+
cluster_config.validate_config!
|
15
|
+
@cluster_configs[cluster_name] = cluster_config
|
16
|
+
end
|
17
|
+
|
18
|
+
def fetch_cluster_config(cluster_name)
|
19
|
+
@cluster_configs.fetch cluster_name
|
20
|
+
end
|
21
|
+
|
22
|
+
def define_sequencer(sequencer_name, &block)
|
23
|
+
sequencer_config = SequencerConfig.new sequencer_name
|
24
|
+
sequencer_config.instance_eval(&block)
|
25
|
+
sequencer_config.validate_config!
|
26
|
+
@sequencer_configs[sequencer_name] = sequencer_config
|
27
|
+
end
|
28
|
+
|
29
|
+
def fetch_sequencer_config(sequencer_name)
|
30
|
+
@sequencer_configs.fetch sequencer_name
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,234 @@
|
|
1
|
+
module ActiveRecord
|
2
|
+
module Sharding
|
3
|
+
module DatabaseTasks
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def info
|
7
|
+
puts "All clusters registered to activerecord-sharding"
|
8
|
+
puts
|
9
|
+
clusters.each do |cluster|
|
10
|
+
puts "= Cluster: #{cluster.name} ="
|
11
|
+
cluster.connections.each do |name|
|
12
|
+
puts "- #{name}"
|
13
|
+
end
|
14
|
+
puts
|
15
|
+
end
|
16
|
+
puts_sequencers
|
17
|
+
end
|
18
|
+
|
19
|
+
def puts_sequencers
|
20
|
+
return unless sequencers
|
21
|
+
|
22
|
+
puts "All sequencers registered to activerecord-sharding"
|
23
|
+
puts
|
24
|
+
sequencers.each do |sequencer|
|
25
|
+
puts "= Sequencer: #{sequencer.name} ="
|
26
|
+
puts "- Connection:#{sequencer.connection_name} Table:#{sequencer.table_name}"
|
27
|
+
puts
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def ar5?
|
32
|
+
ActiveRecord::VERSION::MAJOR == 5
|
33
|
+
end
|
34
|
+
|
35
|
+
def ar4?
|
36
|
+
ActiveRecord::VERSION::MAJOR == 4
|
37
|
+
end
|
38
|
+
|
39
|
+
def ar42?
|
40
|
+
ar4? && ActiveRecord::VERSION::MINOR == 2
|
41
|
+
end
|
42
|
+
|
43
|
+
def ar41?
|
44
|
+
ar4? && ActiveRecord::VERSION::MINOR == 1
|
45
|
+
end
|
46
|
+
|
47
|
+
def ar417_above?
|
48
|
+
ar41? && ActiveRecord::VERSION::TINY > 7
|
49
|
+
end
|
50
|
+
|
51
|
+
def clusters
|
52
|
+
ActiveRecord::Sharding.config.cluster_configs.values
|
53
|
+
end
|
54
|
+
|
55
|
+
def cluster_names
|
56
|
+
ActiveRecord::Sharding.config.cluster_configs.keys
|
57
|
+
end
|
58
|
+
|
59
|
+
def sequencer_names
|
60
|
+
ActiveRecord::Sharding.config.sequencer_configs.keys
|
61
|
+
end
|
62
|
+
|
63
|
+
def fetch_cluster_config(cluster_name)
|
64
|
+
ActiveRecord::Sharding.config.fetch_cluster_config cluster_name
|
65
|
+
end
|
66
|
+
|
67
|
+
def sequencers
|
68
|
+
ActiveRecord::Sharding.config.sequencer_configs.values
|
69
|
+
end
|
70
|
+
|
71
|
+
def fetch_sequencer_config(sequencer_name)
|
72
|
+
ActiveRecord::Sharding.config.fetch_sequencer_config sequencer_name
|
73
|
+
end
|
74
|
+
|
75
|
+
def to_rake_task(task_name)
|
76
|
+
Rake::Task[task_name]
|
77
|
+
end
|
78
|
+
|
79
|
+
module TasksForMultipleClusters
|
80
|
+
def invoke_task_for_all_clusters(task_name)
|
81
|
+
cluster_names.each do |cluster_name|
|
82
|
+
invoke_task task_name, cluster_name
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def invoke_task(name, cluster_name)
|
87
|
+
task_name = "active_record:sharding:#{name}"
|
88
|
+
to_rake_task(task_name).invoke cluster_name.to_s
|
89
|
+
to_rake_task(task_name).reenable
|
90
|
+
end
|
91
|
+
|
92
|
+
def invoke_task_for_all_sequencers(task_name)
|
93
|
+
sequencer_names.each do |sequencer_name|
|
94
|
+
invoke_task_for_sequencer task_name, sequencer_name
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def invoke_task_for_sequencer(name, sequencer_name)
|
99
|
+
task_name = "active_record:sharding:sequencer:#{name}"
|
100
|
+
to_rake_task(task_name).invoke sequencer_name.to_s
|
101
|
+
to_rake_task(task_name).reenable
|
102
|
+
end
|
103
|
+
end
|
104
|
+
extend TasksForMultipleClusters
|
105
|
+
|
106
|
+
module TaskOrganizerForSingleClusterTask
|
107
|
+
def create_all_databases(args)
|
108
|
+
exec_task_for_all_databases "create", args
|
109
|
+
end
|
110
|
+
|
111
|
+
def drop_all_databases(args)
|
112
|
+
exec_task_for_all_databases "drop", args
|
113
|
+
end
|
114
|
+
|
115
|
+
def load_schema_all_databases(args)
|
116
|
+
exec_task_for_all_databases "load_schema", args
|
117
|
+
end
|
118
|
+
|
119
|
+
private
|
120
|
+
|
121
|
+
def exec_task_for_all_databases(task_name, args)
|
122
|
+
cluster_name = cluster_name_or_error task_name, args
|
123
|
+
cluster = cluster_or_error cluster_name
|
124
|
+
cluster.connections.each do |connection_name|
|
125
|
+
__send__ task_name, connection_name.to_s
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def cluster_name_or_error(name, args)
|
130
|
+
unless cluster_name = args[:cluster_name]
|
131
|
+
$stderr.puts <<-MSG
|
132
|
+
Missing cluster_name. Find cluster_name via `rake active_record:sharding:info` then call `rake "active_record:sharding:#{name}[$cluster_name]"`.
|
133
|
+
MSG
|
134
|
+
exit
|
135
|
+
end
|
136
|
+
cluster_name
|
137
|
+
end
|
138
|
+
|
139
|
+
def cluster_or_error(cluster_name)
|
140
|
+
fetch_cluster_config cluster_name.to_sym
|
141
|
+
rescue KeyError
|
142
|
+
$stderr.puts %(cluster name "#{cluster_name}" not found.)
|
143
|
+
exit
|
144
|
+
end
|
145
|
+
end
|
146
|
+
extend TaskOrganizerForSingleClusterTask
|
147
|
+
|
148
|
+
module TasksForSingleConnection
|
149
|
+
def create(connection_name)
|
150
|
+
configuration = ActiveRecord::Base.configurations[connection_name]
|
151
|
+
ActiveRecord::Tasks::DatabaseTasks.create(configuration)
|
152
|
+
ActiveRecord::Base.establish_connection(configuration)
|
153
|
+
end
|
154
|
+
|
155
|
+
def drop(connection_name)
|
156
|
+
configuration = ActiveRecord::Base.configurations[connection_name]
|
157
|
+
ActiveRecord::Tasks::DatabaseTasks.drop configuration
|
158
|
+
end
|
159
|
+
|
160
|
+
def execute(connection_name, sql)
|
161
|
+
configuration = ActiveRecord::Base.configurations[connection_name]
|
162
|
+
ActiveRecord::Base.establish_connection(configuration).connection.execute sql
|
163
|
+
end
|
164
|
+
|
165
|
+
def load_schema(connection_name)
|
166
|
+
configuration = ActiveRecord::Base.configurations[connection_name]
|
167
|
+
|
168
|
+
case
|
169
|
+
when ar5?
|
170
|
+
ActiveRecord::Tasks::DatabaseTasks.load_schema configuration, :ruby
|
171
|
+
when ar42? || ar417_above?
|
172
|
+
ActiveRecord::Tasks::DatabaseTasks.load_schema_for configuration, :ruby
|
173
|
+
when ar41?
|
174
|
+
ActiveRecord::Base.establish_connection configuration
|
175
|
+
ActiveRecord::Tasks::DatabaseTasks.load_schema :ruby
|
176
|
+
else
|
177
|
+
fail "This version of ActiveRecord is not supported: v#{ActiveRecord::VERSION::STRING}"
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
extend TasksForSingleConnection
|
182
|
+
|
183
|
+
module TasksForSingleSequencerTask
|
184
|
+
def create_sequencer_database(args)
|
185
|
+
exec_task_for_sequencer_database "create", args
|
186
|
+
end
|
187
|
+
|
188
|
+
def drop_sequencer_database(args)
|
189
|
+
exec_task_for_sequencer_database "drop", args
|
190
|
+
end
|
191
|
+
|
192
|
+
def create_table_sequencer_database(args)
|
193
|
+
sequencer = sequencer_or_error "create_table", args
|
194
|
+
create_table_sql = "CREATE TABLE #{sequencer.table_name} (id BIGINT unsigned NOT NULL DEFAULT 0) ENGINE=MyISAM"
|
195
|
+
execute sequencer.connection_name.to_s, create_table_sql
|
196
|
+
end
|
197
|
+
|
198
|
+
def insert_initial_record_sequencer_database(args)
|
199
|
+
sequencer = sequencer_or_error "insert_initial_record", args
|
200
|
+
insert_initial_record_sql = "INSERT INTO #{sequencer.table_name} VALUES (0)"
|
201
|
+
execute sequencer.connection_name.to_s, insert_initial_record_sql
|
202
|
+
end
|
203
|
+
|
204
|
+
private
|
205
|
+
|
206
|
+
def exec_task_for_sequencer_database(task_name, args)
|
207
|
+
sequencer = sequencer_or_error task_name, args
|
208
|
+
__send__ task_name, sequencer.connection_name.to_s
|
209
|
+
end
|
210
|
+
|
211
|
+
def sequencer_or_error(task_name, args)
|
212
|
+
sequencer_name = sequencer_name_or_error task_name, args
|
213
|
+
fetch_sequencer_config sequencer_name.to_sym
|
214
|
+
rescue KeyError
|
215
|
+
$stderr.puts %(sequencer name "#{sequencer_name}" not found.)
|
216
|
+
exit
|
217
|
+
end
|
218
|
+
|
219
|
+
def sequencer_name_or_error(task_name, args)
|
220
|
+
unless sequencer_name = args[:sequencer_name]
|
221
|
+
# rubocop:disable Metrics/LineLength
|
222
|
+
$stderr.puts <<-MSG
|
223
|
+
Missing sequencer_name. Find sequencer_name via `rake active_record:sharding:info` then call `rake "active_record:sharding:sequencer#{task_name}[$sequencer_name]"`.
|
224
|
+
MSG
|
225
|
+
exit
|
226
|
+
# rubocop:enable Metrics/LineLength
|
227
|
+
end
|
228
|
+
sequencer_name
|
229
|
+
end
|
230
|
+
end
|
231
|
+
extend TasksForSingleSequencerTask
|
232
|
+
end # module DatabaseTasks
|
233
|
+
end
|
234
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require "active_support/concern"
|
2
|
+
|
3
|
+
module ActiveRecord
|
4
|
+
module Sharding
|
5
|
+
module Model
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
included do
|
9
|
+
class_attribute :cluster_router, instance_writer: false
|
10
|
+
class_attribute :shard_repository, instance_writer: false
|
11
|
+
class_attribute :sharding_key, instance_writer: false
|
12
|
+
end
|
13
|
+
|
14
|
+
module ClassMethods
|
15
|
+
def use_sharding(name, algorithm = :modulo)
|
16
|
+
config = ActiveRecord::Sharding.config.fetch_cluster_config name
|
17
|
+
if algorithm == :modulo
|
18
|
+
self.cluster_router = ActiveRecord::Sharding::ModuloRouter.new config
|
19
|
+
end
|
20
|
+
self.shard_repository = ActiveRecord::Sharding::ShardRepository.new config, self
|
21
|
+
self.abstract_class = true
|
22
|
+
end
|
23
|
+
|
24
|
+
def define_sharding_key(column)
|
25
|
+
self.sharding_key = column.to_sym
|
26
|
+
end
|
27
|
+
|
28
|
+
def before_put(&block)
|
29
|
+
@before_put_callback = block
|
30
|
+
end
|
31
|
+
|
32
|
+
def put!(attributes)
|
33
|
+
fail "`sharding_key` is not defined. Use `define_sharding_key`." unless sharding_key
|
34
|
+
|
35
|
+
@before_put_callback.call(attributes) if @before_put_callback
|
36
|
+
|
37
|
+
if key = attributes[sharding_key] || attributes[sharding_key.to_s]
|
38
|
+
shard_for(key).create!(attributes)
|
39
|
+
else
|
40
|
+
fail ActiveRecord::Sharding::MissingShardingKeyAttribute
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def shard_for(key)
|
45
|
+
connection_name = cluster_router.route key
|
46
|
+
shard_repository.fetch connection_name
|
47
|
+
end
|
48
|
+
|
49
|
+
def all_shards
|
50
|
+
shard_repository.all
|
51
|
+
end
|
52
|
+
|
53
|
+
def define_parent_methods(&block)
|
54
|
+
instance_eval(&block)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module ActiveRecord
|
2
|
+
module Sharding
|
3
|
+
class ModuloRouter
|
4
|
+
def initialize(cluster_config)
|
5
|
+
@cluster_config = cluster_config
|
6
|
+
end
|
7
|
+
|
8
|
+
def route(id)
|
9
|
+
modulo_key = id % @cluster_config.registerd_connection_count
|
10
|
+
@cluster_config.fetch modulo_key
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require "active_support/concern"
|
2
|
+
|
3
|
+
module ActiveRecord
|
4
|
+
module Sharding
|
5
|
+
module Sequencer
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
included do
|
9
|
+
class_attribute :sequencer_repository, instance_writer: false
|
10
|
+
class_attribute :sequencer_name, instance_writer: false
|
11
|
+
class_attribute :sequencer_config, instance_writer: false
|
12
|
+
end
|
13
|
+
|
14
|
+
module ClassMethods
|
15
|
+
def use_sequencer(name)
|
16
|
+
self.sequencer_name = name
|
17
|
+
self.sequencer_config = ActiveRecord::Sharding.config.fetch_sequencer_config name
|
18
|
+
self.sequencer_repository = ActiveRecord::Sharding::SequencerRepository.new sequencer_config, self
|
19
|
+
self.abstract_class = true
|
20
|
+
end
|
21
|
+
|
22
|
+
def current_sequence_id
|
23
|
+
execute_sql "id"
|
24
|
+
end
|
25
|
+
|
26
|
+
def next_sequence_id
|
27
|
+
execute_sql "id +1"
|
28
|
+
end
|
29
|
+
|
30
|
+
def execute_sql(last_insert_id_args)
|
31
|
+
connection = sequencer_repository.fetch(sequencer_name).connection
|
32
|
+
connection.execute "UPDATE `#{sequencer_config.table_name}` SET id = LAST_INSERT_ID(#{last_insert_id_args})"
|
33
|
+
res = connection.execute "SELECT LAST_INSERT_ID()"
|
34
|
+
new_id = res.first.first.to_i
|
35
|
+
new_id
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module ActiveRecord
|
2
|
+
module Sharding
|
3
|
+
class SequencerConfig
|
4
|
+
attr_reader :name, :table_name, :connection_name
|
5
|
+
|
6
|
+
def initialize(name)
|
7
|
+
@name = name
|
8
|
+
@table_name = nil
|
9
|
+
@connection_name = nil
|
10
|
+
end
|
11
|
+
|
12
|
+
def register_connection(connection_name)
|
13
|
+
@connection_name = connection_name
|
14
|
+
end
|
15
|
+
|
16
|
+
def register_table_name(table_name)
|
17
|
+
@table_name = table_name
|
18
|
+
end
|
19
|
+
|
20
|
+
def validate_config!
|
21
|
+
fail "Nothing connection. Please call register_connection" if @connection_name.blank?
|
22
|
+
fail "Nothing table_name. Please call register_table_name" if @table_name.blank?
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|