acts_as_shardable 0.7.0 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +12 -4
- data/lib/acts_as_shardable.rb +17 -192
- data/lib/acts_as_shardable/attribute_methods.rb +14 -0
- data/lib/acts_as_shardable/shard.rb +72 -0
- data/lib/acts_as_shardable/version.rb +1 -1
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: f0eac1d75f9a6c51a14d30662973d19a2eb191c2b02aae5cfbe2fd0564dc45d2
|
4
|
+
data.tar.gz: 501940171fa843435ce04ba2240f305fea243d738d1513b737e69d3292536035
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3c5fc8e35e934d3fbd2aefcbdc24c8715bf9e13b25c98500ccd1eb7ec7b1a4aa62743b0381f67c7e2f10585dabd91de9b58f537a3ea2c0b70c1e7faa5e083c51
|
7
|
+
data.tar.gz: bd35953eb6fff25b02757df5948e6995aa2374b72062293708fd418d79ef44e9ff91295519e912f67b6eff010106b29927f3ed03b06940136efa26c250c5d175
|
data/README.md
CHANGED
@@ -18,17 +18,25 @@ Or install it yourself as:
|
|
18
18
|
|
19
19
|
## Usage
|
20
20
|
|
21
|
-
|
21
|
+
Examples
|
22
22
|
|
23
|
-
|
23
|
+
### Built-in sharding method(mod)
|
24
24
|
|
25
25
|
```ruby
|
26
26
|
class Mod2Model < ActiveRecord::Base
|
27
|
-
acts_as_shardable by: :hash_id, mod: 2
|
27
|
+
acts_as_shardable by: :hash_id, using: :mod, args: { mod: 2 }
|
28
28
|
end
|
29
29
|
```
|
30
30
|
|
31
|
-
|
31
|
+
### Customized sharding method
|
32
|
+
|
33
|
+
```ruby
|
34
|
+
class Mod2Model < ActiveRecord::Base
|
35
|
+
acts_as_shardable by: :hash_id, using: ->(hash_id) { hash_id % 2 }
|
36
|
+
end
|
37
|
+
```
|
38
|
+
|
39
|
+
### Migrations
|
32
40
|
|
33
41
|
```ruby
|
34
42
|
class CreateMod2Models < ActiveRecord::Migration
|
data/lib/acts_as_shardable.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
require 'active_record'
|
2
2
|
require "acts_as_shardable/version"
|
3
|
+
require "acts_as_shardable/attribute_methods"
|
4
|
+
require "acts_as_shardable/shard"
|
3
5
|
|
4
6
|
module ActsAsShardable
|
5
7
|
|
@@ -10,207 +12,30 @@ module ActsAsShardable
|
|
10
12
|
Mutex.new
|
11
13
|
end
|
12
14
|
|
13
|
-
def acts_as_shardable(by:,
|
14
|
-
class_attribute :shard_method, :
|
15
|
+
def acts_as_shardable(by:, using:, args: {})
|
16
|
+
class_attribute :shard_method, :shard_method_using, :shard_args, :module_name, :base_table_name
|
17
|
+
|
18
|
+
case using
|
19
|
+
when Symbol
|
20
|
+
# built-in functions
|
21
|
+
when Proc
|
22
|
+
# customized functions
|
23
|
+
else
|
24
|
+
raise ArgumentError, "unknown sharding function, `using` must be a symbol or proc"
|
25
|
+
end
|
15
26
|
|
16
27
|
mutex.synchronize do
|
17
28
|
self.shard_method = by
|
18
|
-
self.
|
29
|
+
self.shard_method_using = using
|
30
|
+
self.shard_args = args
|
19
31
|
self.module_name = self.name.deconstantize.safe_constantize || Object
|
20
32
|
self.base_table_name = self.name.demodulize.pluralize.underscore
|
21
33
|
self.table_name = "#{self.base_table_name}_0000"
|
22
34
|
self.validates self.shard_method.to_sym, presence: true
|
23
|
-
|
24
|
-
# Updates the associated record with values matching those of the instance attributes.
|
25
|
-
# Returns the number of affected rows.
|
26
|
-
define_method :_update_record do |attribute_names = self.attribute_names|
|
27
|
-
attribute_names = keys_for_partial_write if self.class.base_class.partial_writes
|
28
|
-
|
29
|
-
was, is = changes[self.class.base_class.shard_method]
|
30
|
-
|
31
|
-
if was
|
32
|
-
# shard_column changed
|
33
|
-
table_was = self.class.base_class.sharding(was).table_name
|
34
|
-
table_is = self.class.base_class.sharding(is).table_name
|
35
|
-
raise WrongShardingError, "Please move from #{table_was} to #{table_is} manually."
|
36
|
-
else
|
37
|
-
# shard_column not changing
|
38
|
-
if locking_enabled?
|
39
|
-
lock_col = self.class.base_class.locking_column
|
40
|
-
previous_lock_value = read_attribute(lock_col)
|
41
|
-
self[lock_col] = previous_lock_value + 1
|
42
|
-
|
43
|
-
attribute_names += [lock_col].compact
|
44
|
-
attribute_names.uniq!
|
45
|
-
|
46
|
-
begin
|
47
|
-
relation = shard.unscoped
|
48
|
-
|
49
|
-
affected_rows = relation.where(
|
50
|
-
self.class.primary_key => id,
|
51
|
-
lock_col => previous_lock_value,
|
52
|
-
).update_all(
|
53
|
-
Hash[attributes_for_update(attribute_names).map do |name|
|
54
|
-
[name, _read_attribute(name)]
|
55
|
-
end]
|
56
|
-
)
|
57
|
-
|
58
|
-
unless affected_rows == 1
|
59
|
-
raise ActiveRecord::StaleObjectError.new(self, "update")
|
60
|
-
end
|
61
|
-
|
62
|
-
affected_rows
|
63
|
-
|
64
|
-
# If something went wrong, revert the version.
|
65
|
-
rescue Exception
|
66
|
-
send(lock_col + '=', previous_lock_value)
|
67
|
-
raise
|
68
|
-
end
|
69
|
-
else
|
70
|
-
if self.respond_to?(:attributes_with_values_for_update, true)
|
71
|
-
attributes_values = attributes_with_values_for_update(attribute_names)
|
72
|
-
else
|
73
|
-
attribute_names = attributes_for_update(attribute_names)
|
74
|
-
attributes_values = {}
|
75
|
-
arel_table = shard.arel_table
|
76
|
-
|
77
|
-
attribute_names.each do |name|
|
78
|
-
attributes_values[arel_table[name]] = typecasted_attribute_value(name)
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
if attributes_values.empty?
|
83
|
-
0
|
84
|
-
else
|
85
|
-
if ActiveRecord::VERSION::MAJOR == 5 && ActiveRecord::VERSION::MINOR >= 1
|
86
|
-
shard.unscoped._update_record(attributes_values, self.class.base_class.primary_key => id_in_database)
|
87
|
-
else
|
88
|
-
shard.unscoped._update_record(attributes_values, id, id_was)
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
|
-
private :_update_record
|
96
|
-
|
97
|
-
|
98
|
-
define_method :touch do |*names|
|
99
|
-
raise ActiveRecordError, "cannot touch on a new record object" unless persisted?
|
100
|
-
|
101
|
-
attributes = timestamp_attributes_for_update_in_model
|
102
|
-
attributes.concat(names)
|
103
|
-
|
104
|
-
unless attributes.empty?
|
105
|
-
current_time = current_time_from_proper_timezone
|
106
|
-
changes = {}
|
107
|
-
|
108
|
-
attributes.each do |column|
|
109
|
-
column = column.to_s
|
110
|
-
changes[column] = write_attribute(column, current_time)
|
111
|
-
end
|
112
|
-
|
113
|
-
if locking_enabled?
|
114
|
-
if self.respond_to?(:increment_lock)
|
115
|
-
changes[self.class.base_class.locking_column] = increment_lock
|
116
|
-
else
|
117
|
-
lock_col = self.class.base_class.locking_column
|
118
|
-
previous_lock_value = read_attribute(lock_col)
|
119
|
-
self[lock_col] = previous_lock_value + 1
|
120
|
-
changes[lock_col] = self[lock_col]
|
121
|
-
end
|
122
|
-
end
|
123
|
-
|
124
|
-
clear_attribute_changes(changes.keys)
|
125
|
-
primary_key = self.class.primary_key
|
126
|
-
shard.unscoped.where(primary_key => self[primary_key]).update_all(changes) == 1
|
127
|
-
else
|
128
|
-
true
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
# Creates a record with values matching those of the instance attributes
|
133
|
-
# and returns its id.
|
134
|
-
define_method :_create_record do |attribute_names = self.attribute_names|
|
135
|
-
_run_create_callbacks {
|
136
|
-
attribute_names = keys_for_partial_write if self.class.base_class.partial_writes
|
137
|
-
attribute_names |= [self.class.base_class.locking_column] if locking_enabled?
|
138
|
-
|
139
|
-
if self.respond_to?(:attributes_with_values_for_create, true)
|
140
|
-
attributes_values = attributes_with_values_for_create(attribute_names)
|
141
|
-
else
|
142
|
-
attributes_values = arel_attributes_with_values_for_create(attribute_names)
|
143
|
-
end
|
144
|
-
|
145
|
-
if shard.respond_to?(:_insert_record)
|
146
|
-
new_id = shard.unscoped._insert_record attributes_values
|
147
|
-
else
|
148
|
-
new_id = shard.unscoped.insert attributes_values
|
149
|
-
end
|
150
|
-
|
151
|
-
self.id ||= new_id if self.class.base_class.primary_key
|
152
|
-
|
153
|
-
@new_record = false
|
154
|
-
id
|
155
|
-
}
|
156
|
-
end
|
157
|
-
|
158
|
-
private :_create_record
|
159
|
-
|
160
|
-
define_method :shard do
|
161
|
-
shard_value = self.send(self.class.base_class.shard_method)
|
162
|
-
self.class.base_class.sharding(shard_value)
|
163
|
-
end
|
164
|
-
|
165
|
-
private :shard
|
166
|
-
|
167
|
-
self.class.send :define_method, :sharding do |column|
|
168
|
-
i = column.to_i % shard_mod
|
169
|
-
klass = "#{base_class.name.demodulize}_%04d" % i
|
170
|
-
@@sharding_class ||= {}
|
171
|
-
@@sharding_class[klass] ||= mutex.synchronize do
|
172
|
-
if module_name.const_defined?(klass, false)
|
173
|
-
module_name.const_get(klass, false)
|
174
|
-
else
|
175
|
-
Class.new(base_class) do
|
176
|
-
self.table_name = ("#{base_class.base_table_name}_%04d" % i)
|
177
|
-
|
178
|
-
if base_class.respond_to?(:protobuf_message)
|
179
|
-
self.protobuf_message base_class.protobuf_message
|
180
|
-
|
181
|
-
# Create a .to_proto method on XXX::ActiveRecord_Relation
|
182
|
-
self.const_get('ActiveRecord_Relation', false).class_exec do
|
183
|
-
def to_proto(*args)
|
184
|
-
msg_class = base_class.name.demodulize.pluralize
|
185
|
-
module_name = base_class.name.deconstantize.constantize
|
186
|
-
module_name::Messages.const_get(msg_class, false).new(msg_class.underscore => map { |r| r.to_proto(*args) })
|
187
|
-
end
|
188
|
-
end
|
189
|
-
end
|
190
|
-
|
191
|
-
end.tap { |k| module_name.const_set(klass, k) }
|
192
|
-
end
|
193
|
-
end
|
194
|
-
end
|
195
|
-
|
196
|
-
define_method :reload do |options = nil|
|
197
|
-
clear_aggregation_cache
|
198
|
-
clear_association_cache
|
199
|
-
shard.connection.clear_query_cache
|
200
|
-
|
201
|
-
fresh_object =
|
202
|
-
if options && options[:lock]
|
203
|
-
shard.unscoped { shard.lock(options[:lock]).find(id) }
|
204
|
-
else
|
205
|
-
shard.unscoped { shard.find(id) }
|
206
|
-
end
|
207
|
-
|
208
|
-
@attributes = fresh_object.instance_variable_get('@attributes')
|
209
|
-
@new_record = false
|
210
|
-
fresh_object
|
211
|
-
end
|
212
35
|
end
|
213
36
|
|
37
|
+
include AttributeMethods
|
38
|
+
include Shard
|
214
39
|
end
|
215
40
|
|
216
41
|
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module ActsAsShardable
|
2
|
+
module AttributeMethods
|
3
|
+
private
|
4
|
+
|
5
|
+
# @overload
|
6
|
+
def _assign_attribute(k, v)
|
7
|
+
if persisted? && k.to_s == self.class.base_class.shard_method.to_s
|
8
|
+
raise WrongShardingError, "The sharding key #{k} can't be change"
|
9
|
+
end
|
10
|
+
|
11
|
+
super
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
module ActsAsShardable
|
2
|
+
module Shard
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
included do
|
6
|
+
alias :_class :class
|
7
|
+
|
8
|
+
def shard
|
9
|
+
if defined? @attributes
|
10
|
+
shard_value = self.attributes[_class.base_class.shard_method.to_s]
|
11
|
+
if shard_value
|
12
|
+
_class.base_class.sharding(shard_value)
|
13
|
+
else
|
14
|
+
_class
|
15
|
+
end
|
16
|
+
else
|
17
|
+
_class
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
alias :class :shard
|
22
|
+
end
|
23
|
+
|
24
|
+
class_methods do
|
25
|
+
|
26
|
+
def sharding(column)
|
27
|
+
i = case self.shard_method_using
|
28
|
+
when Proc
|
29
|
+
self.shard_method_using.call(column)
|
30
|
+
when Symbol
|
31
|
+
self.send(self.shard_method_using, column, self.shard_args)
|
32
|
+
end
|
33
|
+
|
34
|
+
constantize_class(i)
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def mod(current, mod:)
|
40
|
+
current % mod
|
41
|
+
end
|
42
|
+
|
43
|
+
def constantize_class(i)
|
44
|
+
class_name = "#{base_class.name.demodulize}_%04d" % i
|
45
|
+
@@sharding_class ||= {}
|
46
|
+
@@sharding_class[class_name] ||= mutex.synchronize do
|
47
|
+
if module_name.const_defined?(class_name, false)
|
48
|
+
module_name.const_get(class_name, false)
|
49
|
+
else
|
50
|
+
Class.new(base_class) do
|
51
|
+
self.table_name = ("#{base_class.base_table_name}_%04d" % i)
|
52
|
+
|
53
|
+
if base_class.respond_to?(:protobuf_message)
|
54
|
+
self.protobuf_message base_class.protobuf_message
|
55
|
+
|
56
|
+
# Create a .to_proto method on XXX::ActiveRecord_Relation
|
57
|
+
self.const_get('ActiveRecord_Relation', false).class_exec do
|
58
|
+
def to_proto(*args)
|
59
|
+
msg_class = base_class.name.demodulize.pluralize
|
60
|
+
module_name = base_class.name.deconstantize.constantize
|
61
|
+
module_name::Messages.const_get(msg_class, false).new(msg_class.underscore => map { |r| r.to_proto(*args) })
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
end.tap { |k| module_name.const_set(class_name, k) }
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: acts_as_shardable
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- scorix
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-09-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -97,6 +97,8 @@ files:
|
|
97
97
|
- bin/console
|
98
98
|
- bin/setup
|
99
99
|
- lib/acts_as_shardable.rb
|
100
|
+
- lib/acts_as_shardable/attribute_methods.rb
|
101
|
+
- lib/acts_as_shardable/shard.rb
|
100
102
|
- lib/acts_as_shardable/version.rb
|
101
103
|
homepage: https://github.com/scorix
|
102
104
|
licenses:
|
@@ -119,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
119
121
|
version: '0'
|
120
122
|
requirements: []
|
121
123
|
rubyforge_project:
|
122
|
-
rubygems_version: 2.
|
124
|
+
rubygems_version: 2.7.3
|
123
125
|
signing_key:
|
124
126
|
specification_version: 4
|
125
127
|
summary: Let subclasses of ActiveRecord::Base to be shardable.
|