global_uid 3.7.1 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/global_uid.rb +35 -8
- data/lib/global_uid/active_record_extension.rb +11 -17
- data/lib/global_uid/allocator.rb +51 -0
- data/lib/global_uid/base.rb +35 -160
- data/lib/global_uid/configuration.rb +66 -0
- data/lib/global_uid/error_tracker.rb +7 -0
- data/lib/global_uid/migration_extension.rb +14 -6
- data/lib/global_uid/server.rb +121 -0
- data/lib/global_uid/test_support.rb +44 -0
- metadata +53 -7
- data/lib/global_uid/server_variables.rb +0 -30
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bc0bc131f043570b57245aa1b807b4f7e74dd87aefc17db5b403d2d7e615f5e5
|
|
4
|
+
data.tar.gz: a0048d1ebec4af921f0fee12db3ade0e37bfcc5af282f94e61f197fdb182ff17
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: df66f3a63d4b00c946e76f41cb18817e3790ba229be243ba1abf6cbc555cec1d778316782afdb87ac9769d47cef151b41e3a3b0f47b8640b0ccc6fb9e5f0dcbe
|
|
7
|
+
data.tar.gz: 987393d901a7d49897a4362c58a88d851695af84fdc137c307781acdc3b21dd00537608d241814d7293e4d12e3c26fe3ad96dfa085228ecbcff6d6edfb69fe1f
|
data/lib/global_uid.rb
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
require "global_uid/base"
|
|
3
|
+
require "global_uid/allocator"
|
|
4
|
+
require "global_uid/server"
|
|
5
|
+
require "global_uid/configuration"
|
|
6
|
+
require "global_uid/error_tracker"
|
|
3
7
|
require "global_uid/active_record_extension"
|
|
4
8
|
require "global_uid/has_and_belongs_to_many_builder_extension"
|
|
5
9
|
require "global_uid/migration_extension"
|
|
@@ -9,8 +13,36 @@ module GlobalUid
|
|
|
9
13
|
class NoServersAvailableException < StandardError ; end
|
|
10
14
|
class ConnectionTimeoutException < StandardError ; end
|
|
11
15
|
class TimeoutException < StandardError ; end
|
|
16
|
+
class InvalidIncrementException < StandardError ; end
|
|
12
17
|
|
|
13
|
-
|
|
18
|
+
def self.configuration
|
|
19
|
+
@configuration ||= GlobalUid::Configuration.new
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.configure
|
|
23
|
+
yield configuration if block_given?
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def self.disable!
|
|
27
|
+
self.configuration.disabled = true
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.enable!
|
|
31
|
+
self.configuration.disabled = false
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.enabled?
|
|
35
|
+
!self.disabled?
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def self.disabled?
|
|
39
|
+
self.configuration.disabled
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# @private
|
|
43
|
+
def self.reset_configuration
|
|
44
|
+
@configuration = nil
|
|
45
|
+
end
|
|
14
46
|
end
|
|
15
47
|
|
|
16
48
|
ActiveRecord::Base.send(:include, GlobalUid::ActiveRecordExtension)
|
|
@@ -26,10 +58,5 @@ if ActiveRecord::VERSION::MAJOR >= 5
|
|
|
26
58
|
ActiveRecord::InternalMetadata.disable_global_uid
|
|
27
59
|
end
|
|
28
60
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
if ActiveRecord::VERSION::MAJOR >= 4
|
|
34
|
-
ActiveRecord::SchemaDumper.send(:prepend, GlobalUid::SchemaDumperExtension)
|
|
35
|
-
end
|
|
61
|
+
ActiveRecord::Associations::Builder::HasAndBelongsToMany.send(:include, GlobalUid::HasAndBelongsToManyBuilderExtension)
|
|
62
|
+
ActiveRecord::SchemaDumper.send(:prepend, GlobalUid::SchemaDumperExtension)
|
|
@@ -8,20 +8,10 @@ module GlobalUid
|
|
|
8
8
|
end
|
|
9
9
|
|
|
10
10
|
def global_uid_before_create
|
|
11
|
-
return if GlobalUid
|
|
11
|
+
return if GlobalUid.disabled?
|
|
12
12
|
return if self.class.global_uid_disabled
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
realtime = Benchmark::realtime do
|
|
16
|
-
global_uid = self.class.generate_uid
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
if GlobalUid::Base.global_uid_options[:dry_run]
|
|
20
|
-
ActiveRecord::Base.logger.info("GlobalUid dry-run: #{self.class.name}\t#{global_uid}\t#{"%.4f" % realtime}")
|
|
21
|
-
return
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
self.id = global_uid
|
|
14
|
+
self.id = self.class.generate_uid
|
|
25
15
|
end
|
|
26
16
|
|
|
27
17
|
module ClassMethods
|
|
@@ -37,12 +27,16 @@ module GlobalUid
|
|
|
37
27
|
@global_uid_disabled
|
|
38
28
|
end
|
|
39
29
|
|
|
40
|
-
def generate_uid
|
|
41
|
-
GlobalUid::Base.
|
|
30
|
+
def generate_uid
|
|
31
|
+
GlobalUid::Base.with_servers do |server|
|
|
32
|
+
return server.allocate(self)
|
|
33
|
+
end
|
|
42
34
|
end
|
|
43
35
|
|
|
44
|
-
def generate_many_uids(count
|
|
45
|
-
GlobalUid::Base.
|
|
36
|
+
def generate_many_uids(count)
|
|
37
|
+
GlobalUid::Base.with_servers do |server|
|
|
38
|
+
return Array(server.allocate(self, count: count))
|
|
39
|
+
end
|
|
46
40
|
end
|
|
47
41
|
|
|
48
42
|
def disable_global_uid
|
|
@@ -54,7 +48,7 @@ module GlobalUid
|
|
|
54
48
|
end
|
|
55
49
|
|
|
56
50
|
def global_uid_table
|
|
57
|
-
GlobalUid::Base.id_table_from_name(self.table_name)
|
|
51
|
+
@_global_uid_table ||= GlobalUid::Base.id_table_from_name(self.table_name)
|
|
58
52
|
end
|
|
59
53
|
end
|
|
60
54
|
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
module GlobalUid
|
|
2
|
+
class Allocator
|
|
3
|
+
attr_reader :recent_allocations, :max_window_size, :incrementing_by, :connection, :table_name
|
|
4
|
+
|
|
5
|
+
def initialize(incrementing_by:, connection:, table_name:)
|
|
6
|
+
@recent_allocations = []
|
|
7
|
+
@max_window_size = 5
|
|
8
|
+
@incrementing_by = incrementing_by
|
|
9
|
+
@connection = connection
|
|
10
|
+
@table_name = table_name
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def allocate_one
|
|
14
|
+
identifier = connection.insert("REPLACE INTO #{table_name} (stub) VALUES ('a')")
|
|
15
|
+
allocate(identifier)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def allocate_many(count:)
|
|
19
|
+
return [] unless count > 0
|
|
20
|
+
|
|
21
|
+
increment_by = connection.select_value("SELECT @@auto_increment_increment")
|
|
22
|
+
|
|
23
|
+
start_id = connection.insert("REPLACE INTO #{table_name} (stub) VALUES " + (["('a')"] * count).join(','))
|
|
24
|
+
identifiers = start_id.step(start_id + (count - 1) * increment_by, increment_by).to_a
|
|
25
|
+
identifiers.each { |identifier| allocate(identifier) }
|
|
26
|
+
identifiers
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def allocate(identifier)
|
|
32
|
+
recent_allocations.shift if recent_allocations.size >= max_window_size
|
|
33
|
+
recent_allocations << identifier
|
|
34
|
+
|
|
35
|
+
if !valid_allocation?
|
|
36
|
+
db_increment = connection.select_value("SELECT @@auto_increment_increment")
|
|
37
|
+
message = "Configured: '#{incrementing_by}', Found: '#{db_increment}' on '#{connection.current_database}'. Recently allocated IDs: #{recent_allocations} using table '#{table_name}'"
|
|
38
|
+
GlobalUid::Base.alert(InvalidIncrementException.new(message))
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
identifier
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def valid_allocation?
|
|
45
|
+
recent_allocations[1..-1].all? do |identifier|
|
|
46
|
+
(identifier > recent_allocations[0]) &&
|
|
47
|
+
(identifier - recent_allocations[0]) % incrementing_by == 0
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
data/lib/global_uid/base.rb
CHANGED
|
@@ -6,17 +6,6 @@ require "timeout"
|
|
|
6
6
|
|
|
7
7
|
module GlobalUid
|
|
8
8
|
class Base
|
|
9
|
-
GLOBAL_UID_DEFAULTS = {
|
|
10
|
-
:connection_timeout => 3,
|
|
11
|
-
:connection_retry => 10.minutes,
|
|
12
|
-
:notifier => Proc.new { |exception, message| ActiveRecord::Base.logger.error("GlobalUID error: #{exception} #{message}") },
|
|
13
|
-
:query_timeout => 10,
|
|
14
|
-
:increment_by => 5, # This will define the maximum number of servers that you can have
|
|
15
|
-
:disabled => false,
|
|
16
|
-
:per_process_affinity => true,
|
|
17
|
-
:dry_run => false
|
|
18
|
-
}
|
|
19
|
-
|
|
20
9
|
def self.servers
|
|
21
10
|
# Thread local storage is inheritted on `fork`, include the pid
|
|
22
11
|
Thread.current["global_uid_servers_#{$$}"]
|
|
@@ -26,180 +15,66 @@ module GlobalUid
|
|
|
26
15
|
Thread.current["global_uid_servers_#{$$}"] = s
|
|
27
16
|
end
|
|
28
17
|
|
|
29
|
-
def self.
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
`stub` char(1) NOT NULL DEFAULT '',
|
|
39
|
-
PRIMARY KEY (`id`),
|
|
40
|
-
UNIQUE KEY `stub` (`stub`)
|
|
41
|
-
) #{engine_stmt}")
|
|
42
|
-
|
|
43
|
-
# prime the pump on each server
|
|
44
|
-
connection.execute("INSERT IGNORE INTO `#{id_table_name}` VALUES(#{start_id}, 'a')")
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
def self.drop_uid_tables(id_table_name, options={})
|
|
49
|
-
with_connections do |connection|
|
|
50
|
-
connection.execute("DROP TABLE IF EXISTS `#{id_table_name}`")
|
|
51
|
-
end
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
def self.new_connection(name, connection_timeout, offset, increment_by)
|
|
55
|
-
raise "No id server '#{name}' configured in database.yml" unless ActiveRecord::Base.configurations.to_h.has_key?(name)
|
|
56
|
-
config = ActiveRecord::Base.configurations.to_h[name]
|
|
57
|
-
c = config.symbolize_keys
|
|
58
|
-
|
|
59
|
-
raise "No global_uid support for adapter #{c[:adapter]}" if c[:adapter] != 'mysql2'
|
|
60
|
-
|
|
61
|
-
begin
|
|
62
|
-
Timeout.timeout(connection_timeout, ConnectionTimeoutException) do
|
|
63
|
-
ActiveRecord::Base.mysql2_connection(config)
|
|
64
|
-
end
|
|
65
|
-
rescue ConnectionTimeoutException => e
|
|
66
|
-
notify e, "Timed out establishing a connection to #{name}"
|
|
67
|
-
nil
|
|
68
|
-
rescue Exception => e
|
|
69
|
-
notify e, "establishing a connection to #{name}: #{e.message}"
|
|
70
|
-
nil
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def self.init_server_info(options)
|
|
75
|
-
id_servers = self.global_uid_servers
|
|
76
|
-
|
|
77
|
-
raise "You haven't configured any id servers" if id_servers.nil? or id_servers.empty?
|
|
78
|
-
raise "More servers configured than increment_by: #{id_servers.size} > #{options[:increment_by]} -- this will create duplicate IDs." if id_servers.size > options[:increment_by]
|
|
79
|
-
|
|
80
|
-
offset = 1
|
|
81
|
-
|
|
82
|
-
id_servers.map do |name, i|
|
|
83
|
-
info = {}
|
|
84
|
-
info[:cx] = nil
|
|
85
|
-
info[:name] = name
|
|
86
|
-
info[:retry_at] = nil
|
|
87
|
-
info[:offset] = offset
|
|
88
|
-
info[:rand] = rand
|
|
89
|
-
info[:new?] = true
|
|
90
|
-
offset +=1
|
|
91
|
-
info
|
|
92
|
-
end
|
|
18
|
+
def self.init_server_info
|
|
19
|
+
GlobalUid.configuration.id_servers.map do |name|
|
|
20
|
+
GlobalUid::Server.new(name,
|
|
21
|
+
increment_by: GlobalUid.configuration.increment_by,
|
|
22
|
+
connection_retry: GlobalUid.configuration.connection_retry,
|
|
23
|
+
connection_timeout: GlobalUid.configuration.connection_timeout,
|
|
24
|
+
query_timeout: GlobalUid.configuration.query_timeout
|
|
25
|
+
)
|
|
26
|
+
end.shuffle # so each process uses a random server
|
|
93
27
|
end
|
|
94
28
|
|
|
95
29
|
def self.disconnect!
|
|
30
|
+
servers.each(&:disconnect!) unless servers.nil?
|
|
96
31
|
self.servers = nil
|
|
97
32
|
end
|
|
98
33
|
|
|
99
|
-
def self.
|
|
100
|
-
|
|
101
|
-
|
|
34
|
+
def self.with_servers
|
|
35
|
+
self.servers ||= init_server_info
|
|
36
|
+
servers = self.servers.each(&:connect)
|
|
102
37
|
|
|
103
|
-
if
|
|
104
|
-
|
|
105
|
-
# sorting here sets up each process to have affinity to a particular server.
|
|
106
|
-
self.servers = self.servers.sort_by { |s| s[:rand] }
|
|
38
|
+
if GlobalUid.configuration.connection_shuffling?
|
|
39
|
+
servers.shuffle! # subsequent requests are made against different servers
|
|
107
40
|
end
|
|
108
41
|
|
|
109
|
-
self.servers.each do |info|
|
|
110
|
-
next if info[:cx]
|
|
111
|
-
|
|
112
|
-
if info[:new?] || ( info[:retry_at] && Time.now > info[:retry_at] )
|
|
113
|
-
info[:new?] = false
|
|
114
|
-
|
|
115
|
-
connection = new_connection(info[:name], connection_timeout, info[:offset], increment_by)
|
|
116
|
-
info[:cx] = connection
|
|
117
|
-
info[:retry_at] = Time.now + options[:connection_retry] if connection.nil?
|
|
118
|
-
end
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
self.servers
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
def self.with_connections(options = {})
|
|
125
|
-
options = self.global_uid_options.merge(options)
|
|
126
|
-
servers = setup_connections!(options)
|
|
127
|
-
|
|
128
|
-
if !options[:per_process_affinity]
|
|
129
|
-
servers = servers.sort_by { rand } #yes, I know it's not true random.
|
|
130
|
-
end
|
|
131
|
-
|
|
132
|
-
raise NoServersAvailableException if servers.empty?
|
|
133
|
-
|
|
134
42
|
errors = []
|
|
135
|
-
servers.each do |
|
|
43
|
+
servers.each do |server|
|
|
136
44
|
begin
|
|
137
|
-
yield
|
|
45
|
+
yield server if server.active?
|
|
138
46
|
rescue TimeoutException, Exception => e
|
|
139
|
-
|
|
47
|
+
GlobalUid.configuration.notifier.call(e)
|
|
140
48
|
errors << e
|
|
141
|
-
|
|
142
|
-
|
|
49
|
+
server.disconnect!
|
|
50
|
+
server.update_retry_at(1.minute)
|
|
143
51
|
end
|
|
144
52
|
end
|
|
145
53
|
|
|
146
54
|
# in the case where all servers are gone, put everyone back in.
|
|
147
|
-
if servers.all?
|
|
148
|
-
servers.each do |
|
|
149
|
-
|
|
55
|
+
if servers.all?(&:disconnected?)
|
|
56
|
+
servers.each do |server|
|
|
57
|
+
server.update_retry_at(0)
|
|
150
58
|
end
|
|
151
|
-
|
|
59
|
+
message = errors.empty? ? "" : "Errors hit: #{errors.map(&:to_s).join(', ')}"
|
|
60
|
+
exception = NoServersAvailableException.new(message)
|
|
61
|
+
GlobalUid.configuration.notifier.call(exception)
|
|
62
|
+
raise exception
|
|
152
63
|
end
|
|
153
64
|
|
|
154
|
-
servers
|
|
65
|
+
servers
|
|
155
66
|
end
|
|
156
67
|
|
|
157
|
-
def self.
|
|
158
|
-
|
|
159
|
-
self.global_uid_options[:notifier].call(exception, message)
|
|
160
|
-
end
|
|
161
|
-
end
|
|
162
|
-
|
|
163
|
-
def self.get_connections(options = {})
|
|
164
|
-
with_connections {}
|
|
165
|
-
end
|
|
166
|
-
|
|
167
|
-
def self.get_uid_for_class(klass, options = {})
|
|
168
|
-
with_connections do |connection|
|
|
169
|
-
Timeout.timeout(self.global_uid_options[:query_timeout], TimeoutException) do
|
|
170
|
-
id = connection.insert("REPLACE INTO #{klass.global_uid_table} (stub) VALUES ('a')")
|
|
171
|
-
return id
|
|
172
|
-
end
|
|
173
|
-
end
|
|
174
|
-
raise NoServersAvailableException, "All global UID servers are gone!"
|
|
68
|
+
def self.id_table_from_name(name)
|
|
69
|
+
"#{name}_ids".to_sym
|
|
175
70
|
end
|
|
176
71
|
|
|
177
|
-
def self.
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
start_id = connection.insert("REPLACE INTO #{klass.global_uid_table} (stub) VALUES " + (["('a')"] * count).join(','))
|
|
183
|
-
return start_id.step(start_id + (count-1) * increment_by, increment_by).to_a
|
|
184
|
-
end
|
|
72
|
+
def self.alert(exception)
|
|
73
|
+
if GlobalUid.configuration.suppress_increment_exceptions?
|
|
74
|
+
GlobalUid.configuration.notifier.call(exception)
|
|
75
|
+
else
|
|
76
|
+
raise exception
|
|
185
77
|
end
|
|
186
|
-
raise NoServersAvailableException, "All global UID servers are gone!"
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
def self.global_uid_options=(options)
|
|
190
|
-
@global_uid_options = GLOBAL_UID_DEFAULTS.merge(options.symbolize_keys)
|
|
191
|
-
end
|
|
192
|
-
|
|
193
|
-
def self.global_uid_options
|
|
194
|
-
@global_uid_options
|
|
195
|
-
end
|
|
196
|
-
|
|
197
|
-
def self.global_uid_servers
|
|
198
|
-
self.global_uid_options[:id_servers]
|
|
199
|
-
end
|
|
200
|
-
|
|
201
|
-
def self.id_table_from_name(name)
|
|
202
|
-
"#{name}_ids".to_sym
|
|
203
78
|
end
|
|
204
79
|
end
|
|
205
80
|
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
module GlobalUid
|
|
2
|
+
class Configuration
|
|
3
|
+
|
|
4
|
+
attr_accessor :connection_timeout
|
|
5
|
+
attr_accessor :connection_retry
|
|
6
|
+
attr_accessor :notifier
|
|
7
|
+
attr_accessor :query_timeout
|
|
8
|
+
attr_accessor :increment_by
|
|
9
|
+
attr_accessor :disabled
|
|
10
|
+
attr_accessor :connection_shuffling
|
|
11
|
+
attr_accessor :suppress_increment_exceptions
|
|
12
|
+
attr_accessor :storage_engine
|
|
13
|
+
|
|
14
|
+
alias_method :connection_shuffling?, :connection_shuffling
|
|
15
|
+
alias_method :suppress_increment_exceptions?, :suppress_increment_exceptions
|
|
16
|
+
|
|
17
|
+
# Set defaults
|
|
18
|
+
def initialize
|
|
19
|
+
# Timeout (in seconds) for connecting to a global UID server
|
|
20
|
+
@connection_timeout = 3
|
|
21
|
+
|
|
22
|
+
# Duration (in seconds) to wait before attempting another connection to UID server
|
|
23
|
+
@connection_retry = 600 # 10 minutes
|
|
24
|
+
|
|
25
|
+
# An object that will be notified in case of a UID server failure.
|
|
26
|
+
# The object is expected to respond to `#call`, which will be passed one argument of type `Exception`.
|
|
27
|
+
@notifier = GlobalUid::ErrorTracker.new
|
|
28
|
+
|
|
29
|
+
# Timeout (in seconds) for retrieving a global UID from a server before moving to the next server
|
|
30
|
+
@query_timeout = 10
|
|
31
|
+
|
|
32
|
+
# Used for validation, compared with the value on the alloc servers to prevent allocation of duplicate IDs
|
|
33
|
+
# NB: The value configured here does not dictate the value on your alloc server and must remain in
|
|
34
|
+
# sync with the value of auto_increment_increment in the database.
|
|
35
|
+
@increment_by = 5
|
|
36
|
+
|
|
37
|
+
# Disable GlobalUid entirely
|
|
38
|
+
@disabled = false
|
|
39
|
+
|
|
40
|
+
# The same allocation server is used each time `with_servers` is called
|
|
41
|
+
@connection_shuffling = false
|
|
42
|
+
|
|
43
|
+
# Suppress configuration validation, allowing updates to auto_increment_increment while alloc servers in use.
|
|
44
|
+
# The InvalidIncrementException will be swallowed and logged when suppressed
|
|
45
|
+
@suppress_increment_exceptions = false
|
|
46
|
+
|
|
47
|
+
# The name of the alloc DB servers, defined in your database.yml
|
|
48
|
+
# e.g. ["id_server_1", "id_server_2"]
|
|
49
|
+
@id_servers = []
|
|
50
|
+
|
|
51
|
+
# The storage engine used during GloblaUid table creation
|
|
52
|
+
# Supported and tested: InnoDB, MyISAM
|
|
53
|
+
@storage_engine = "MyISAM"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def id_servers=(value)
|
|
57
|
+
raise "More servers configured than increment_by: #{value.size} > #{increment_by} -- this will create duplicate IDs." if value.size > increment_by
|
|
58
|
+
@id_servers = value
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def id_servers
|
|
62
|
+
raise "You haven't configured any id servers" if @id_servers.empty?
|
|
63
|
+
@id_servers
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -3,10 +3,10 @@ module GlobalUid
|
|
|
3
3
|
module MigrationExtension
|
|
4
4
|
|
|
5
5
|
def create_table(name, options = {}, &blk)
|
|
6
|
-
uid_enabled =
|
|
6
|
+
uid_enabled = GlobalUid.enabled? && options[:use_global_uid] != false
|
|
7
7
|
|
|
8
|
-
# rules for stripping out auto_increment -- enabled
|
|
9
|
-
remove_auto_increment = uid_enabled && !
|
|
8
|
+
# rules for stripping out auto_increment -- enabled and not a "PK-less" table
|
|
9
|
+
remove_auto_increment = uid_enabled && !(options[:id] == false)
|
|
10
10
|
|
|
11
11
|
options.merge!(:id => false) if remove_auto_increment
|
|
12
12
|
|
|
@@ -17,15 +17,23 @@ module GlobalUid
|
|
|
17
17
|
|
|
18
18
|
if uid_enabled
|
|
19
19
|
id_table_name = options[:global_uid_table] || GlobalUid::Base.id_table_from_name(name)
|
|
20
|
-
GlobalUid::Base.
|
|
20
|
+
GlobalUid::Base.with_servers do |server|
|
|
21
|
+
server.create_uid_table!(
|
|
22
|
+
name: id_table_name,
|
|
23
|
+
uid_type: options[:uid_type],
|
|
24
|
+
start_id: options[:start_id]
|
|
25
|
+
)
|
|
26
|
+
end
|
|
21
27
|
end
|
|
22
28
|
|
|
23
29
|
end
|
|
24
30
|
|
|
25
31
|
def drop_table(name, options = {})
|
|
26
|
-
if
|
|
32
|
+
if GlobalUid.enabled? && options[:use_global_uid] == true
|
|
27
33
|
id_table_name = options[:global_uid_table] || GlobalUid::Base.id_table_from_name(name)
|
|
28
|
-
GlobalUid::Base.
|
|
34
|
+
GlobalUid::Base.with_servers do |server|
|
|
35
|
+
server.drop_uid_table!(name: id_table_name)
|
|
36
|
+
end
|
|
29
37
|
end
|
|
30
38
|
super(name, options)
|
|
31
39
|
end
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
module GlobalUid
|
|
2
|
+
class Server
|
|
3
|
+
|
|
4
|
+
attr_accessor :connection, :name
|
|
5
|
+
|
|
6
|
+
def initialize(name, increment_by:, connection_retry:, connection_timeout:, query_timeout:)
|
|
7
|
+
@connection = nil
|
|
8
|
+
@name = name
|
|
9
|
+
@retry_at = nil
|
|
10
|
+
@allocators = {}
|
|
11
|
+
@increment_by = increment_by
|
|
12
|
+
@connection_retry = connection_retry
|
|
13
|
+
@connection_timeout = connection_timeout
|
|
14
|
+
@query_timeout = query_timeout
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def connect
|
|
18
|
+
return @connection if active? || !retry_connection?
|
|
19
|
+
@connection = mysql2_connection(name)
|
|
20
|
+
|
|
21
|
+
begin
|
|
22
|
+
validate_connection_increment if active?
|
|
23
|
+
rescue InvalidIncrementException => e
|
|
24
|
+
GlobalUid.configuration.notifier.call(e)
|
|
25
|
+
disconnect!
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
@connection
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def active?
|
|
32
|
+
!disconnected?
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def disconnected?
|
|
36
|
+
@connection.nil?
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def update_retry_at(seconds)
|
|
40
|
+
@retry_at = Time.now + seconds
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def disconnect!
|
|
44
|
+
@connection = nil
|
|
45
|
+
@allocators = {}
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def create_uid_table!(name:, uid_type: nil, start_id: nil)
|
|
49
|
+
uid_type ||= "bigint(21) UNSIGNED"
|
|
50
|
+
start_id ||= 1
|
|
51
|
+
|
|
52
|
+
connection.execute("CREATE TABLE IF NOT EXISTS `#{name}` (
|
|
53
|
+
`id` #{uid_type} NOT NULL AUTO_INCREMENT,
|
|
54
|
+
`stub` char(1) NOT NULL DEFAULT '',
|
|
55
|
+
PRIMARY KEY (`id`),
|
|
56
|
+
UNIQUE KEY `stub` (`stub`)
|
|
57
|
+
) ENGINE=#{GlobalUid.configuration.storage_engine}")
|
|
58
|
+
|
|
59
|
+
# prime the pump on each server
|
|
60
|
+
connection.execute("INSERT IGNORE INTO `#{name}` VALUES(#{start_id}, 'a')")
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def drop_uid_table!(name:)
|
|
64
|
+
connection.execute("DROP TABLE IF EXISTS `#{name}`")
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def allocate(klass, count: 1)
|
|
68
|
+
# TODO: Replace Timeout.timeout with DB level timeout
|
|
69
|
+
# Timeout.timeout is unpredictable
|
|
70
|
+
Timeout.timeout(query_timeout, TimeoutException) do
|
|
71
|
+
if count == 1
|
|
72
|
+
allocator(klass).allocate_one
|
|
73
|
+
else
|
|
74
|
+
allocator(klass).allocate_many(count: count)
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
attr_accessor :connection_retry, :connection_timeout, :retry_at, :increment_by, :query_timeout, :allocators
|
|
82
|
+
|
|
83
|
+
def allocator(klass)
|
|
84
|
+
table_name = klass.global_uid_table
|
|
85
|
+
@allocators[table_name] ||= Allocator.new(incrementing_by: increment_by, connection: connection, table_name: table_name)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def retry_connection?
|
|
89
|
+
return Time.now > retry_at if retry_at
|
|
90
|
+
|
|
91
|
+
update_retry_at(connection_retry)
|
|
92
|
+
true
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def mysql2_connection(name)
|
|
96
|
+
raise "No id server '#{name}' configured in database.yml" unless ActiveRecord::Base.configurations.to_h.has_key?(name)
|
|
97
|
+
config = ActiveRecord::Base.configurations.to_h[name]
|
|
98
|
+
c = config.symbolize_keys
|
|
99
|
+
|
|
100
|
+
raise "No global_uid support for adapter #{c[:adapter]}" if c[:adapter] != 'mysql2'
|
|
101
|
+
|
|
102
|
+
Timeout.timeout(connection_timeout, ConnectionTimeoutException) do
|
|
103
|
+
ActiveRecord::Base.mysql2_connection(config)
|
|
104
|
+
end
|
|
105
|
+
rescue ConnectionTimeoutException => e
|
|
106
|
+
GlobalUid.configuration.notifier.call(ConnectionTimeoutException.new("Timed out establishing a connection to #{name}"))
|
|
107
|
+
nil
|
|
108
|
+
rescue Exception => e
|
|
109
|
+
GlobalUid.configuration.notifier.call(StandardError.new("establishing a connection to #{name}: #{e.message}"))
|
|
110
|
+
nil
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def validate_connection_increment
|
|
114
|
+
db_increment = connection.select_value("SELECT @@auto_increment_increment")
|
|
115
|
+
|
|
116
|
+
if db_increment != increment_by
|
|
117
|
+
GlobalUid::Base.alert(InvalidIncrementException.new("Configured: '#{increment_by}', Found: '#{db_increment}' on '#{connection.current_database}'"))
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
module GlobalUid
|
|
2
|
+
module TestSupport
|
|
3
|
+
# Tables should be created through the MigrationExtension but
|
|
4
|
+
# if you want to manually create and drop the '_id' tables,
|
|
5
|
+
# you can do so via this module
|
|
6
|
+
class << self
|
|
7
|
+
def create_uid_tables(tables: [], uid_type: nil, start_id: nil)
|
|
8
|
+
return if GlobalUid.disabled?
|
|
9
|
+
|
|
10
|
+
GlobalUid::Base.with_servers do |server|
|
|
11
|
+
tables.each do |table|
|
|
12
|
+
server.create_uid_table!(
|
|
13
|
+
name: GlobalUid::Base.id_table_from_name(table),
|
|
14
|
+
uid_type: uid_type,
|
|
15
|
+
start_id: start_id
|
|
16
|
+
)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def drop_uid_tables(tables: [])
|
|
22
|
+
return if GlobalUid.disabled?
|
|
23
|
+
|
|
24
|
+
GlobalUid::Base.with_servers do |server|
|
|
25
|
+
tables.each do |table|
|
|
26
|
+
server.drop_uid_table!(
|
|
27
|
+
name: GlobalUid::Base.id_table_from_name(table)
|
|
28
|
+
)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def recreate_uid_tables(tables: [], uid_type: nil, start_id: nil)
|
|
34
|
+
return if GlobalUid.disabled?
|
|
35
|
+
|
|
36
|
+
drop_uid_tables(tables: tables)
|
|
37
|
+
create_uid_tables(tables: tables, uid_type: nil, start_id: start_id)
|
|
38
|
+
|
|
39
|
+
# Reset the servers, clearing any allocations from memory
|
|
40
|
+
GlobalUid::Base.disconnect!
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
metadata
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: global_uid
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 4.0.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Benjamin Quorning
|
|
8
8
|
- Gabe Martin-Dempesy
|
|
9
9
|
- Pierre Schambacher
|
|
10
10
|
- Ben Osheroff
|
|
11
|
-
autorequire:
|
|
11
|
+
autorequire:
|
|
12
12
|
bindir: bin
|
|
13
13
|
cert_chain: []
|
|
14
|
-
date: 2020-
|
|
14
|
+
date: 2020-11-12 00:00:00.000000000 Z
|
|
15
15
|
dependencies:
|
|
16
16
|
- !ruby/object:Gem::Dependency
|
|
17
17
|
name: activerecord
|
|
@@ -117,6 +117,20 @@ dependencies:
|
|
|
117
117
|
- - ">="
|
|
118
118
|
- !ruby/object:Gem::Version
|
|
119
119
|
version: '0'
|
|
120
|
+
- !ruby/object:Gem::Dependency
|
|
121
|
+
name: minitest-line
|
|
122
|
+
requirement: !ruby/object:Gem::Requirement
|
|
123
|
+
requirements:
|
|
124
|
+
- - ">="
|
|
125
|
+
- !ruby/object:Gem::Version
|
|
126
|
+
version: '0'
|
|
127
|
+
type: :development
|
|
128
|
+
prerelease: false
|
|
129
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
130
|
+
requirements:
|
|
131
|
+
- - ">="
|
|
132
|
+
- !ruby/object:Gem::Version
|
|
133
|
+
version: '0'
|
|
120
134
|
- !ruby/object:Gem::Dependency
|
|
121
135
|
name: mocha
|
|
122
136
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -131,6 +145,20 @@ dependencies:
|
|
|
131
145
|
- - ">="
|
|
132
146
|
- !ruby/object:Gem::Version
|
|
133
147
|
version: '0'
|
|
148
|
+
- !ruby/object:Gem::Dependency
|
|
149
|
+
name: benchmark-ips
|
|
150
|
+
requirement: !ruby/object:Gem::Requirement
|
|
151
|
+
requirements:
|
|
152
|
+
- - ">="
|
|
153
|
+
- !ruby/object:Gem::Version
|
|
154
|
+
version: '0'
|
|
155
|
+
type: :development
|
|
156
|
+
prerelease: false
|
|
157
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
158
|
+
requirements:
|
|
159
|
+
- - ">="
|
|
160
|
+
- !ruby/object:Gem::Version
|
|
161
|
+
version: '0'
|
|
134
162
|
- !ruby/object:Gem::Dependency
|
|
135
163
|
name: bump
|
|
136
164
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -159,6 +187,20 @@ dependencies:
|
|
|
159
187
|
- - ">="
|
|
160
188
|
- !ruby/object:Gem::Version
|
|
161
189
|
version: '0'
|
|
190
|
+
- !ruby/object:Gem::Dependency
|
|
191
|
+
name: pry
|
|
192
|
+
requirement: !ruby/object:Gem::Requirement
|
|
193
|
+
requirements:
|
|
194
|
+
- - ">="
|
|
195
|
+
- !ruby/object:Gem::Version
|
|
196
|
+
version: '0'
|
|
197
|
+
type: :development
|
|
198
|
+
prerelease: false
|
|
199
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
200
|
+
requirements:
|
|
201
|
+
- - ">="
|
|
202
|
+
- !ruby/object:Gem::Version
|
|
203
|
+
version: '0'
|
|
162
204
|
description: GUIDs for sharded models
|
|
163
205
|
email:
|
|
164
206
|
- bquorning@zendesk.com
|
|
@@ -170,16 +212,20 @@ extra_rdoc_files: []
|
|
|
170
212
|
files:
|
|
171
213
|
- lib/global_uid.rb
|
|
172
214
|
- lib/global_uid/active_record_extension.rb
|
|
215
|
+
- lib/global_uid/allocator.rb
|
|
173
216
|
- lib/global_uid/base.rb
|
|
217
|
+
- lib/global_uid/configuration.rb
|
|
218
|
+
- lib/global_uid/error_tracker.rb
|
|
174
219
|
- lib/global_uid/has_and_belongs_to_many_builder_extension.rb
|
|
175
220
|
- lib/global_uid/migration_extension.rb
|
|
176
221
|
- lib/global_uid/schema_dumper_extension.rb
|
|
177
|
-
- lib/global_uid/
|
|
222
|
+
- lib/global_uid/server.rb
|
|
223
|
+
- lib/global_uid/test_support.rb
|
|
178
224
|
homepage: https://github.com/zendesk/global_uid
|
|
179
225
|
licenses:
|
|
180
226
|
- MIT
|
|
181
227
|
metadata: {}
|
|
182
|
-
post_install_message:
|
|
228
|
+
post_install_message:
|
|
183
229
|
rdoc_options: []
|
|
184
230
|
require_paths:
|
|
185
231
|
- lib
|
|
@@ -194,8 +240,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
194
240
|
- !ruby/object:Gem::Version
|
|
195
241
|
version: '0'
|
|
196
242
|
requirements: []
|
|
197
|
-
rubygems_version: 3.
|
|
198
|
-
signing_key:
|
|
243
|
+
rubygems_version: 3.0.3
|
|
244
|
+
signing_key:
|
|
199
245
|
specification_version: 4
|
|
200
246
|
summary: GUID
|
|
201
247
|
test_files: []
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# This module is good for testing and development, not so much for production.
|
|
3
|
-
# Please note that this is unreliable -- if you lose your CX to the server
|
|
4
|
-
# and auto-reconnect, you will be utterly hosed. Much better to dedicate a server
|
|
5
|
-
# or two to the cause, and set their auto_increment_increment globally.
|
|
6
|
-
#
|
|
7
|
-
# You can include this module in tests like this:
|
|
8
|
-
# GlobalUid::Base.extend(GlobalUid::ServerVariables)
|
|
9
|
-
#
|
|
10
|
-
module GlobalUid
|
|
11
|
-
module ServerVariables
|
|
12
|
-
|
|
13
|
-
def self.extended(base)
|
|
14
|
-
base.singleton_class.send(:alias_method, :new_connection_without_server_variables, :new_connection)
|
|
15
|
-
base.singleton_class.send(:alias_method, :new_connection, :new_connection_with_server_variables)
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def new_connection_with_server_variables(name, connection_timeout, offset, increment_by)
|
|
19
|
-
con = new_connection_without_server_variables(name, connection_timeout, offset, increment_by)
|
|
20
|
-
|
|
21
|
-
if con
|
|
22
|
-
con.execute("set @@auto_increment_increment = #{increment_by}")
|
|
23
|
-
con.execute("set @@auto_increment_offset = #{offset}")
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
con
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
end
|
|
30
|
-
end
|