activerecord-shard_for 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +3 -273
- data/lib/activerecord/shard_for/model.rb +10 -0
- data/lib/activerecord/shard_for/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cad523e3ec520b0697848ece57f5703c5d049ea0
|
4
|
+
data.tar.gz: cf3f36c9de0afd5f6f42addc95d9f29081a5d51a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 18da70aea7e423466d39cd5c677b469320cc18e57f475e60fa82bde5c4370de58bf3c000f8e91c38371ead208842ea91cc848fa3998d257b1bb48171bece222a
|
7
|
+
data.tar.gz: 35261193e6150eefd81f8f323912254edc6448493a701dfaf952edfe3c75bdd8be1ba8c864d7d2e6cf57d415f98fb017de328dd8c545699920b419003d81302f
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# CHANGELOG for activerecord-shard_for
|
2
2
|
|
3
|
+
## 0.3.0
|
4
|
+
|
5
|
+
- Add using syntax. [#8](https://github.com/yuemori/activerecord-shard_for/pull/8)
|
6
|
+
|
3
7
|
## 0.2.1
|
4
8
|
|
5
9
|
- Fix raise MissingDistkeyAttribute before callback. [#6](https://github.com/yuemori/activerecord-shard_for/pull/6)
|
data/README.md
CHANGED
@@ -41,7 +41,9 @@ Or install it yourself as:
|
|
41
41
|
|
42
42
|
$ gem install activerecord-shard_for
|
43
43
|
|
44
|
-
|
44
|
+
# Getting Started
|
45
|
+
|
46
|
+
More example to see [wiki](https://github.com/yuemori/activerecord-shard_for/wiki).
|
45
47
|
|
46
48
|
Add additional database connection config to database.yml.
|
47
49
|
|
@@ -103,278 +105,6 @@ alice.save!
|
|
103
105
|
User.all_shards.flat_map {|m| m.find_by(name: 'alice') }.compact
|
104
106
|
```
|
105
107
|
|
106
|
-
When you want to execute queries in all nodes in parallel, use .all_shards_in_parallel. It returns `ActiveRecord::ShardFor::AllShardsInParallel` and it offers some collection actions which runs in parallel. It is aliased to .parallel.
|
107
|
-
|
108
|
-
```ruby
|
109
|
-
User.all_shards_in_parallel.map(&count) #=> 1
|
110
|
-
User.parallel.flat_map {|m| m.where(age: 1) }.size #=> 1
|
111
|
-
```
|
112
|
-
|
113
|
-
Sometimes you want to generates distkey value before validation. Since activerecord-shard_for generates sub class of your models, AR's callback is usesless for this usecase, so activerecord-shard_for offers its own callback method.
|
114
|
-
|
115
|
-
```ruby
|
116
|
-
class AccessToken < ActiveRecord::Base
|
117
|
-
include ActiveRecord::ShardFor::Model
|
118
|
-
use_cluster :access_token
|
119
|
-
def_distkey :token
|
120
|
-
|
121
|
-
validates :token, presence: true
|
122
|
-
|
123
|
-
def self.generate_token
|
124
|
-
SecureRandom.uuid
|
125
|
-
end
|
126
|
-
|
127
|
-
before_put do |attributes|
|
128
|
-
unless attributes[:token] || attributes['token']
|
129
|
-
attributes[:token] = generate_token
|
130
|
-
end
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
access_token = AccessToken.put!
|
135
|
-
access_token.token #=> a generated token
|
136
|
-
```
|
137
|
-
|
138
|
-
If you want to range sharding, `Range` object set to cluster key.
|
139
|
-
|
140
|
-
```ruby
|
141
|
-
ActiveRecord::ShardFor.configure do |config|
|
142
|
-
config.define_cluster(:user) do |cluster|
|
143
|
-
# unique identifier, connection name
|
144
|
-
cluster.register(0...100, :production_user_001)
|
145
|
-
cluster.register(100...200, :production_user_002)
|
146
|
-
cluster.register(200...300, :production_user_003)
|
147
|
-
cluster.register(300..Float::INFINITY, :production_user_004)
|
148
|
-
end
|
149
|
-
end
|
150
|
-
|
151
|
-
class User < ActiveRecord::Base
|
152
|
-
include ActiveRecord::ShardFor::Model
|
153
|
-
use_cluster :user, :distkey
|
154
|
-
def_distkey :id
|
155
|
-
|
156
|
-
def self.generate_unique_id
|
157
|
-
# Implement to generate unique id
|
158
|
-
end
|
159
|
-
|
160
|
-
before_put do |attributes|
|
161
|
-
attributes[:id] = generate_unique_id unless attributes[:id]
|
162
|
-
end
|
163
|
-
end
|
164
|
-
```
|
165
|
-
|
166
|
-
## Sharding with Replication
|
167
|
-
|
168
|
-
activerecord-shard_for also supports replication.
|
169
|
-
|
170
|
-
In case you have 2 shards in cluster and each shard have read replica.
|
171
|
-
|
172
|
-
- db-user-101 --replicated--> db-user-102
|
173
|
-
- db-user-201 --replicated--> db-user-202
|
174
|
-
|
175
|
-
Your database connection configuration might be like this:
|
176
|
-
|
177
|
-
```yaml
|
178
|
-
# database.yml
|
179
|
-
production_user_001:
|
180
|
-
adapter: mysql2
|
181
|
-
username: user_writable
|
182
|
-
host: db-user-101
|
183
|
-
production_user_002:
|
184
|
-
adapter: mysql2
|
185
|
-
username: user_writable
|
186
|
-
host: db-user-201
|
187
|
-
production_user_readonly_001:
|
188
|
-
adapter: mysql2
|
189
|
-
username: user_readonly
|
190
|
-
host: db-user-102
|
191
|
-
production_user_readonly_002:
|
192
|
-
adapter: mysql2
|
193
|
-
username: user_writable
|
194
|
-
host: db-user-202
|
195
|
-
```
|
196
|
-
|
197
|
-
Your initializer for activerecord-shard_for might be like this:
|
198
|
-
|
199
|
-
```ruby
|
200
|
-
ActiveRecord::ShardFor.configure do |config|
|
201
|
-
config.define_cluster(:user) do |cluster|
|
202
|
-
cluster.register(0, :production_user_001)
|
203
|
-
cluster.register(1, :production_user_002)
|
204
|
-
end
|
205
|
-
|
206
|
-
config.define_cluster(:user_readonly) do |cluster|
|
207
|
-
# give same key of master
|
208
|
-
cluster.register(0, :production_user_readonly_001)
|
209
|
-
cluster.register(1, :production_user_readonly_002)
|
210
|
-
end
|
211
|
-
end
|
212
|
-
```
|
213
|
-
|
214
|
-
You can split read/write by defining AR model class for each connection:
|
215
|
-
|
216
|
-
```ruby
|
217
|
-
class User < ActiveRecord::Base
|
218
|
-
include ActiveRecord::ShardFor::Model
|
219
|
-
use_cluster :user, :hash_modulo
|
220
|
-
def_distkey :email
|
221
|
-
end
|
222
|
-
|
223
|
-
class UserReadonly < ActiveRecord::Base
|
224
|
-
self.table_name = 'users'
|
225
|
-
|
226
|
-
include ActiveRecord::ShardFor::Model
|
227
|
-
use_cluster :user_readonly, :hash_modulo
|
228
|
-
def_distkey :email
|
229
|
-
end
|
230
|
-
|
231
|
-
User.put!(name: 'Alice', email: 'alice@example.com')
|
232
|
-
UserReadonly.get('alice@example.com')
|
233
|
-
```
|
234
|
-
|
235
|
-
If you want to switch specific shard to another shard in another cluster, define mapping between each model:
|
236
|
-
|
237
|
-
```ruby
|
238
|
-
class User < ActiveRecord::Base
|
239
|
-
include ActiveRecord::ShardFor::Model
|
240
|
-
use_cluster :user, :hash_modulo
|
241
|
-
def_distkey :email
|
242
|
-
|
243
|
-
replicates_with slave: :UserReadonly
|
244
|
-
end
|
245
|
-
|
246
|
-
class UserReadonly < ActiveRecord::Base
|
247
|
-
self.table_name = 'users'
|
248
|
-
|
249
|
-
include ActiveRecord::ShardFor::Model
|
250
|
-
use_cluster :user_readonly, :hash_modulo
|
251
|
-
def_distkey :email
|
252
|
-
|
253
|
-
replicates_with master: :User
|
254
|
-
end
|
255
|
-
```
|
256
|
-
|
257
|
-
You can switch to another model which have connection to the shard by calling .switch:
|
258
|
-
|
259
|
-
```ruby
|
260
|
-
UserReadonly.all_shards do |readonly|
|
261
|
-
target_ids = readonly.where(age: 0).pluck(:id)
|
262
|
-
readonly.switch(:master) do |writable|
|
263
|
-
writable.where(id: target_ids).delete_all
|
264
|
-
end
|
265
|
-
end
|
266
|
-
```
|
267
|
-
|
268
|
-
## Plugin of connection router
|
269
|
-
|
270
|
-
If you need to advanced connection routing, implement router class and register this.
|
271
|
-
|
272
|
-
### Embeded
|
273
|
-
|
274
|
-
Embeded routers:
|
275
|
-
|
276
|
-
|name|class|description|
|
277
|
-
|:---:|:---:|:---|
|
278
|
-
|:hash_modulo|[HashModuloRouter](https://github.com/yuemori/activerecord-shard_for/blob/master/lib/activerecord/shard_for/hash_modulo_router.rb)|use `hash(key) mod connection_count`|
|
279
|
-
|:distkey|[DistkeyRouter](https://github.com/yuemori/activerecord-shard_for/blob/master/lib/activerecord/shard_for/distkey_router.rb)|use `distkey` at it is|
|
280
|
-
|
281
|
-
Connection Routers specific with cluster in AR model.
|
282
|
-
|
283
|
-
```ruby
|
284
|
-
class User < ActiveRecord::Base
|
285
|
-
include ActiveRecord::ShardFor::Model
|
286
|
-
use_cluster :user, :hash_modulo # use hash_modulo
|
287
|
-
def_distkey :email
|
288
|
-
end
|
289
|
-
|
290
|
-
class Character < ActiveRecord::Base
|
291
|
-
include ActiveRecord::ShardFor::Model
|
292
|
-
use_cluster :character, :distkey # use distkey at it is
|
293
|
-
def_distkey :shard_no
|
294
|
-
end
|
295
|
-
```
|
296
|
-
|
297
|
-
### Implement
|
298
|
-
|
299
|
-
Reference a interface to [HashModuloRouter](https://github.com/yuemori/activerecord-shard_for/blob/master/lib/activerecord/shard_for/hash_modulo_router.rb), [DistkeyRouter](https://github.com/yuemori/activerecord-shard_for/blob/master/lib/activerecord/shard_for/distkey_router.rb) and [ConnectionRouter](https://github.com/yuemori/activerecord-shard_for/blob/master/lib/activerecord/shard_for/connection_router.rb).
|
300
|
-
|
301
|
-
Example, simple modulo router:
|
302
|
-
|
303
|
-
```ruby
|
304
|
-
class SimpleModuloRouter < ActiveRecord::ShardFor::ConnectionRouter
|
305
|
-
def route(key)
|
306
|
-
key.to_i % connection_count
|
307
|
-
end
|
308
|
-
end
|
309
|
-
```
|
310
|
-
|
311
|
-
Your initializer for activerecord-shard_for might be like this:
|
312
|
-
|
313
|
-
```ruby
|
314
|
-
ActiveRecord::ShardFor.configure do |config|
|
315
|
-
config.register_connection_router(:modulo, SimpleModuloRouter)
|
316
|
-
end
|
317
|
-
```
|
318
|
-
|
319
|
-
And specify router in your AR model.
|
320
|
-
|
321
|
-
```ruby
|
322
|
-
class User < ActiveRecord::Base
|
323
|
-
include ActiveRecord::ShardFor::Model
|
324
|
-
use_cluster :user, :modulo
|
325
|
-
def_distkey :id
|
326
|
-
|
327
|
-
def self.generate_unique_id
|
328
|
-
# Implement to generate unique id
|
329
|
-
end
|
330
|
-
|
331
|
-
before_put do |attributes|
|
332
|
-
attributes[:id] = generate_unique_id unless attributes[:id]
|
333
|
-
end
|
334
|
-
end
|
335
|
-
```
|
336
|
-
|
337
|
-
## Advanced
|
338
|
-
|
339
|
-
More example, sharding key is String:
|
340
|
-
|
341
|
-
```ruby
|
342
|
-
ActiveRecord::ShardFor.configure do |config|
|
343
|
-
config.define_cluster(:user) do |cluster|
|
344
|
-
# unique identifier, connection name
|
345
|
-
cluster.register('a'..'z', :production_user_001)
|
346
|
-
cluster.register('A'..'Z', :production_user_002)
|
347
|
-
cluster.register('0'..'9', :production_user_003)
|
348
|
-
end
|
349
|
-
end
|
350
|
-
|
351
|
-
class InitialStringRouter < ActiveRecord::ShardFor::ConnectionRouter
|
352
|
-
def route(key)
|
353
|
-
key.to_s.first
|
354
|
-
end
|
355
|
-
end
|
356
|
-
|
357
|
-
ActiveRecord::ShardFor.configure do |config|
|
358
|
-
config.register_connection_router(:initial_string, InitialStringRouter)
|
359
|
-
end
|
360
|
-
|
361
|
-
class User < ActiveRecord::Base
|
362
|
-
include ActiveRecord::ShardFor::Model
|
363
|
-
use_cluster :user, :initial_string
|
364
|
-
def_distkey :id
|
365
|
-
|
366
|
-
def self.generate_unique_id
|
367
|
-
SecureRandom.uuid
|
368
|
-
end
|
369
|
-
|
370
|
-
before_put do |attributes|
|
371
|
-
attributes[:id] = generate_unique_id unless attributes[:id]
|
372
|
-
end
|
373
|
-
end
|
374
|
-
|
375
|
-
```
|
376
|
-
|
377
|
-
|
378
108
|
## Contributing with ActiveRecord::ShardFor
|
379
109
|
|
380
110
|
Contributors are welcome! This is what you need to setup your Octopus development environment:
|
@@ -23,6 +23,16 @@ module ActiveRecord
|
|
23
23
|
self.abstract_class = true
|
24
24
|
end
|
25
25
|
|
26
|
+
# Returns a generated model class of included model which specific connection.
|
27
|
+
# @param [Object] shard_key key of a shard connection
|
28
|
+
# @yield [Class] generated model class which key of shard connection
|
29
|
+
# @return [Class] generated model class which key of shard connection
|
30
|
+
def using(shard_key)
|
31
|
+
model = shard_repository.fetch_by_key(shard_key)
|
32
|
+
yield model if block_given?
|
33
|
+
model
|
34
|
+
end
|
35
|
+
|
26
36
|
# Returns a generated model class of included model class which has proper
|
27
37
|
# connection config for the shard for given key.
|
28
38
|
# @param [String] key A value of distkey
|