activerecord-shard_for 0.2.1 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +3 -273
- data/lib/activerecord/shard_for/model.rb +10 -0
- data/lib/activerecord/shard_for/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cad523e3ec520b0697848ece57f5703c5d049ea0
|
4
|
+
data.tar.gz: cf3f36c9de0afd5f6f42addc95d9f29081a5d51a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 18da70aea7e423466d39cd5c677b469320cc18e57f475e60fa82bde5c4370de58bf3c000f8e91c38371ead208842ea91cc848fa3998d257b1bb48171bece222a
|
7
|
+
data.tar.gz: 35261193e6150eefd81f8f323912254edc6448493a701dfaf952edfe3c75bdd8be1ba8c864d7d2e6cf57d415f98fb017de328dd8c545699920b419003d81302f
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# CHANGELOG for activerecord-shard_for
|
2
2
|
|
3
|
+
## 0.3.0
|
4
|
+
|
5
|
+
- Add using syntax. [#8](https://github.com/yuemori/activerecord-shard_for/pull/8)
|
6
|
+
|
3
7
|
## 0.2.1
|
4
8
|
|
5
9
|
- Fix raise MissingDistkeyAttribute before callback. [#6](https://github.com/yuemori/activerecord-shard_for/pull/6)
|
data/README.md
CHANGED
@@ -41,7 +41,9 @@ Or install it yourself as:
|
|
41
41
|
|
42
42
|
$ gem install activerecord-shard_for
|
43
43
|
|
44
|
-
|
44
|
+
# Getting Started
|
45
|
+
|
46
|
+
More example to see [wiki](https://github.com/yuemori/activerecord-shard_for/wiki).
|
45
47
|
|
46
48
|
Add additional database connection config to database.yml.
|
47
49
|
|
@@ -103,278 +105,6 @@ alice.save!
|
|
103
105
|
User.all_shards.flat_map {|m| m.find_by(name: 'alice') }.compact
|
104
106
|
```
|
105
107
|
|
106
|
-
When you want to execute queries in all nodes in parallel, use .all_shards_in_parallel. It returns `ActiveRecord::ShardFor::AllShardsInParallel` and it offers some collection actions which runs in parallel. It is aliased to .parallel.
|
107
|
-
|
108
|
-
```ruby
|
109
|
-
User.all_shards_in_parallel.map(&count) #=> 1
|
110
|
-
User.parallel.flat_map {|m| m.where(age: 1) }.size #=> 1
|
111
|
-
```
|
112
|
-
|
113
|
-
Sometimes you want to generates distkey value before validation. Since activerecord-shard_for generates sub class of your models, AR's callback is usesless for this usecase, so activerecord-shard_for offers its own callback method.
|
114
|
-
|
115
|
-
```ruby
|
116
|
-
class AccessToken < ActiveRecord::Base
|
117
|
-
include ActiveRecord::ShardFor::Model
|
118
|
-
use_cluster :access_token
|
119
|
-
def_distkey :token
|
120
|
-
|
121
|
-
validates :token, presence: true
|
122
|
-
|
123
|
-
def self.generate_token
|
124
|
-
SecureRandom.uuid
|
125
|
-
end
|
126
|
-
|
127
|
-
before_put do |attributes|
|
128
|
-
unless attributes[:token] || attributes['token']
|
129
|
-
attributes[:token] = generate_token
|
130
|
-
end
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
access_token = AccessToken.put!
|
135
|
-
access_token.token #=> a generated token
|
136
|
-
```
|
137
|
-
|
138
|
-
If you want to range sharding, `Range` object set to cluster key.
|
139
|
-
|
140
|
-
```ruby
|
141
|
-
ActiveRecord::ShardFor.configure do |config|
|
142
|
-
config.define_cluster(:user) do |cluster|
|
143
|
-
# unique identifier, connection name
|
144
|
-
cluster.register(0...100, :production_user_001)
|
145
|
-
cluster.register(100...200, :production_user_002)
|
146
|
-
cluster.register(200...300, :production_user_003)
|
147
|
-
cluster.register(300..Float::INFINITY, :production_user_004)
|
148
|
-
end
|
149
|
-
end
|
150
|
-
|
151
|
-
class User < ActiveRecord::Base
|
152
|
-
include ActiveRecord::ShardFor::Model
|
153
|
-
use_cluster :user, :distkey
|
154
|
-
def_distkey :id
|
155
|
-
|
156
|
-
def self.generate_unique_id
|
157
|
-
# Implement to generate unique id
|
158
|
-
end
|
159
|
-
|
160
|
-
before_put do |attributes|
|
161
|
-
attributes[:id] = generate_unique_id unless attributes[:id]
|
162
|
-
end
|
163
|
-
end
|
164
|
-
```
|
165
|
-
|
166
|
-
## Sharding with Replication
|
167
|
-
|
168
|
-
activerecord-shard_for also supports replication.
|
169
|
-
|
170
|
-
In case you have 2 shards in cluster and each shard have read replica.
|
171
|
-
|
172
|
-
- db-user-101 --replicated--> db-user-102
|
173
|
-
- db-user-201 --replicated--> db-user-202
|
174
|
-
|
175
|
-
Your database connection configuration might be like this:
|
176
|
-
|
177
|
-
```yaml
|
178
|
-
# database.yml
|
179
|
-
production_user_001:
|
180
|
-
adapter: mysql2
|
181
|
-
username: user_writable
|
182
|
-
host: db-user-101
|
183
|
-
production_user_002:
|
184
|
-
adapter: mysql2
|
185
|
-
username: user_writable
|
186
|
-
host: db-user-201
|
187
|
-
production_user_readonly_001:
|
188
|
-
adapter: mysql2
|
189
|
-
username: user_readonly
|
190
|
-
host: db-user-102
|
191
|
-
production_user_readonly_002:
|
192
|
-
adapter: mysql2
|
193
|
-
username: user_writable
|
194
|
-
host: db-user-202
|
195
|
-
```
|
196
|
-
|
197
|
-
Your initializer for activerecord-shard_for might be like this:
|
198
|
-
|
199
|
-
```ruby
|
200
|
-
ActiveRecord::ShardFor.configure do |config|
|
201
|
-
config.define_cluster(:user) do |cluster|
|
202
|
-
cluster.register(0, :production_user_001)
|
203
|
-
cluster.register(1, :production_user_002)
|
204
|
-
end
|
205
|
-
|
206
|
-
config.define_cluster(:user_readonly) do |cluster|
|
207
|
-
# give same key of master
|
208
|
-
cluster.register(0, :production_user_readonly_001)
|
209
|
-
cluster.register(1, :production_user_readonly_002)
|
210
|
-
end
|
211
|
-
end
|
212
|
-
```
|
213
|
-
|
214
|
-
You can split read/write by defining AR model class for each connection:
|
215
|
-
|
216
|
-
```ruby
|
217
|
-
class User < ActiveRecord::Base
|
218
|
-
include ActiveRecord::ShardFor::Model
|
219
|
-
use_cluster :user, :hash_modulo
|
220
|
-
def_distkey :email
|
221
|
-
end
|
222
|
-
|
223
|
-
class UserReadonly < ActiveRecord::Base
|
224
|
-
self.table_name = 'users'
|
225
|
-
|
226
|
-
include ActiveRecord::ShardFor::Model
|
227
|
-
use_cluster :user_readonly, :hash_modulo
|
228
|
-
def_distkey :email
|
229
|
-
end
|
230
|
-
|
231
|
-
User.put!(name: 'Alice', email: 'alice@example.com')
|
232
|
-
UserReadonly.get('alice@example.com')
|
233
|
-
```
|
234
|
-
|
235
|
-
If you want to switch specific shard to another shard in another cluster, define mapping between each model:
|
236
|
-
|
237
|
-
```ruby
|
238
|
-
class User < ActiveRecord::Base
|
239
|
-
include ActiveRecord::ShardFor::Model
|
240
|
-
use_cluster :user, :hash_modulo
|
241
|
-
def_distkey :email
|
242
|
-
|
243
|
-
replicates_with slave: :UserReadonly
|
244
|
-
end
|
245
|
-
|
246
|
-
class UserReadonly < ActiveRecord::Base
|
247
|
-
self.table_name = 'users'
|
248
|
-
|
249
|
-
include ActiveRecord::ShardFor::Model
|
250
|
-
use_cluster :user_readonly, :hash_modulo
|
251
|
-
def_distkey :email
|
252
|
-
|
253
|
-
replicates_with master: :User
|
254
|
-
end
|
255
|
-
```
|
256
|
-
|
257
|
-
You can switch to another model which have connection to the shard by calling .switch:
|
258
|
-
|
259
|
-
```ruby
|
260
|
-
UserReadonly.all_shards do |readonly|
|
261
|
-
target_ids = readonly.where(age: 0).pluck(:id)
|
262
|
-
readonly.switch(:master) do |writable|
|
263
|
-
writable.where(id: target_ids).delete_all
|
264
|
-
end
|
265
|
-
end
|
266
|
-
```
|
267
|
-
|
268
|
-
## Plugin of connection router
|
269
|
-
|
270
|
-
If you need to advanced connection routing, implement router class and register this.
|
271
|
-
|
272
|
-
### Embeded
|
273
|
-
|
274
|
-
Embeded routers:
|
275
|
-
|
276
|
-
|name|class|description|
|
277
|
-
|:---:|:---:|:---|
|
278
|
-
|:hash_modulo|[HashModuloRouter](https://github.com/yuemori/activerecord-shard_for/blob/master/lib/activerecord/shard_for/hash_modulo_router.rb)|use `hash(key) mod connection_count`|
|
279
|
-
|:distkey|[DistkeyRouter](https://github.com/yuemori/activerecord-shard_for/blob/master/lib/activerecord/shard_for/distkey_router.rb)|use `distkey` at it is|
|
280
|
-
|
281
|
-
Connection Routers specific with cluster in AR model.
|
282
|
-
|
283
|
-
```ruby
|
284
|
-
class User < ActiveRecord::Base
|
285
|
-
include ActiveRecord::ShardFor::Model
|
286
|
-
use_cluster :user, :hash_modulo # use hash_modulo
|
287
|
-
def_distkey :email
|
288
|
-
end
|
289
|
-
|
290
|
-
class Character < ActiveRecord::Base
|
291
|
-
include ActiveRecord::ShardFor::Model
|
292
|
-
use_cluster :character, :distkey # use distkey at it is
|
293
|
-
def_distkey :shard_no
|
294
|
-
end
|
295
|
-
```
|
296
|
-
|
297
|
-
### Implement
|
298
|
-
|
299
|
-
Reference a interface to [HashModuloRouter](https://github.com/yuemori/activerecord-shard_for/blob/master/lib/activerecord/shard_for/hash_modulo_router.rb), [DistkeyRouter](https://github.com/yuemori/activerecord-shard_for/blob/master/lib/activerecord/shard_for/distkey_router.rb) and [ConnectionRouter](https://github.com/yuemori/activerecord-shard_for/blob/master/lib/activerecord/shard_for/connection_router.rb).
|
300
|
-
|
301
|
-
Example, simple modulo router:
|
302
|
-
|
303
|
-
```ruby
|
304
|
-
class SimpleModuloRouter < ActiveRecord::ShardFor::ConnectionRouter
|
305
|
-
def route(key)
|
306
|
-
key.to_i % connection_count
|
307
|
-
end
|
308
|
-
end
|
309
|
-
```
|
310
|
-
|
311
|
-
Your initializer for activerecord-shard_for might be like this:
|
312
|
-
|
313
|
-
```ruby
|
314
|
-
ActiveRecord::ShardFor.configure do |config|
|
315
|
-
config.register_connection_router(:modulo, SimpleModuloRouter)
|
316
|
-
end
|
317
|
-
```
|
318
|
-
|
319
|
-
And specify router in your AR model.
|
320
|
-
|
321
|
-
```ruby
|
322
|
-
class User < ActiveRecord::Base
|
323
|
-
include ActiveRecord::ShardFor::Model
|
324
|
-
use_cluster :user, :modulo
|
325
|
-
def_distkey :id
|
326
|
-
|
327
|
-
def self.generate_unique_id
|
328
|
-
# Implement to generate unique id
|
329
|
-
end
|
330
|
-
|
331
|
-
before_put do |attributes|
|
332
|
-
attributes[:id] = generate_unique_id unless attributes[:id]
|
333
|
-
end
|
334
|
-
end
|
335
|
-
```
|
336
|
-
|
337
|
-
## Advanced
|
338
|
-
|
339
|
-
More example, sharding key is String:
|
340
|
-
|
341
|
-
```ruby
|
342
|
-
ActiveRecord::ShardFor.configure do |config|
|
343
|
-
config.define_cluster(:user) do |cluster|
|
344
|
-
# unique identifier, connection name
|
345
|
-
cluster.register('a'..'z', :production_user_001)
|
346
|
-
cluster.register('A'..'Z', :production_user_002)
|
347
|
-
cluster.register('0'..'9', :production_user_003)
|
348
|
-
end
|
349
|
-
end
|
350
|
-
|
351
|
-
class InitialStringRouter < ActiveRecord::ShardFor::ConnectionRouter
|
352
|
-
def route(key)
|
353
|
-
key.to_s.first
|
354
|
-
end
|
355
|
-
end
|
356
|
-
|
357
|
-
ActiveRecord::ShardFor.configure do |config|
|
358
|
-
config.register_connection_router(:initial_string, InitialStringRouter)
|
359
|
-
end
|
360
|
-
|
361
|
-
class User < ActiveRecord::Base
|
362
|
-
include ActiveRecord::ShardFor::Model
|
363
|
-
use_cluster :user, :initial_string
|
364
|
-
def_distkey :id
|
365
|
-
|
366
|
-
def self.generate_unique_id
|
367
|
-
SecureRandom.uuid
|
368
|
-
end
|
369
|
-
|
370
|
-
before_put do |attributes|
|
371
|
-
attributes[:id] = generate_unique_id unless attributes[:id]
|
372
|
-
end
|
373
|
-
end
|
374
|
-
|
375
|
-
```
|
376
|
-
|
377
|
-
|
378
108
|
## Contributing with ActiveRecord::ShardFor
|
379
109
|
|
380
110
|
Contributors are welcome! This is what you need to setup your Octopus development environment:
|
@@ -23,6 +23,16 @@ module ActiveRecord
|
|
23
23
|
self.abstract_class = true
|
24
24
|
end
|
25
25
|
|
26
|
+
# Returns a generated model class of included model which specific connection.
|
27
|
+
# @param [Object] shard_key key of a shard connection
|
28
|
+
# @yield [Class] generated model class which key of shard connection
|
29
|
+
# @return [Class] generated model class which key of shard connection
|
30
|
+
def using(shard_key)
|
31
|
+
model = shard_repository.fetch_by_key(shard_key)
|
32
|
+
yield model if block_given?
|
33
|
+
model
|
34
|
+
end
|
35
|
+
|
26
36
|
# Returns a generated model class of included model class which has proper
|
27
37
|
# connection config for the shard for given key.
|
28
38
|
# @param [String] key A value of distkey
|