switchman 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Rakefile +30 -0
- data/app/models/switchman/shard.rb +502 -0
- data/db/migrate/20130328212039_create_switchman_shards.rb +9 -0
- data/db/migrate/20130328224244_create_default_shard.rb +9 -0
- data/lib/switchman.rb +9 -0
- data/lib/switchman/active_record/abstract_adapter.rb +11 -0
- data/lib/switchman/active_record/association.rb +108 -0
- data/lib/switchman/active_record/attribute_methods.rb +104 -0
- data/lib/switchman/active_record/base.rb +95 -0
- data/lib/switchman/active_record/calculations.rb +63 -0
- data/lib/switchman/active_record/connection_handler.rb +147 -0
- data/lib/switchman/active_record/connection_pool.rb +117 -0
- data/lib/switchman/active_record/finder_methods.rb +25 -0
- data/lib/switchman/active_record/log_subscriber.rb +43 -0
- data/lib/switchman/active_record/postgresql_adapter.rb +13 -0
- data/lib/switchman/active_record/query_cache.rb +12 -0
- data/lib/switchman/active_record/query_methods.rb +184 -0
- data/lib/switchman/active_record/relation.rb +69 -0
- data/lib/switchman/cache_extensions.rb +12 -0
- data/lib/switchman/connection_pool_proxy.rb +62 -0
- data/lib/switchman/database_server.rb +197 -0
- data/lib/switchman/default_shard.rb +28 -0
- data/lib/switchman/engine.rb +91 -0
- data/lib/switchman/r_spec_helper.rb +124 -0
- data/lib/switchman/shackles.rb +34 -0
- data/lib/switchman/test_helper.rb +65 -0
- data/lib/switchman/version.rb +3 -0
- data/spec/dummy/Rakefile +7 -0
- data/spec/dummy/app/models/appendage.rb +24 -0
- data/spec/dummy/app/models/digit.rb +9 -0
- data/spec/dummy/app/models/feature.rb +5 -0
- data/spec/dummy/app/models/mirror_user.rb +5 -0
- data/spec/dummy/app/models/user.rb +23 -0
- data/spec/dummy/config.ru +4 -0
- data/spec/dummy/config/application.rb +59 -0
- data/spec/dummy/config/boot.rb +10 -0
- data/spec/dummy/config/database.yml +17 -0
- data/spec/dummy/config/database.yml.example +25 -0
- data/spec/dummy/config/environment.rb +5 -0
- data/spec/dummy/config/environments/development.rb +37 -0
- data/spec/dummy/config/environments/production.rb +67 -0
- data/spec/dummy/config/environments/test.rb +37 -0
- data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/spec/dummy/config/initializers/secret_token.rb +7 -0
- data/spec/dummy/config/initializers/session_store.rb +8 -0
- data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
- data/spec/dummy/config/routes.rb +8 -0
- data/spec/dummy/db/migrate/20130403132607_create_users.rb +10 -0
- data/spec/dummy/db/migrate/20130411202442_create_appendages.rb +10 -0
- data/spec/dummy/db/migrate/20130411202551_create_mirror_users.rb +9 -0
- data/spec/dummy/db/migrate/20131022202028_create_digits.rb +10 -0
- data/spec/dummy/db/migrate/20131206172923_create_features.rb +12 -0
- data/spec/dummy/db/schema.rb +57 -0
- data/spec/dummy/log/development.log +504 -0
- data/spec/dummy/log/test.log +29907 -0
- data/spec/dummy/script/rails +6 -0
- data/spec/dummy/tmp/cache/2E2/830/shard%2F2 +0 -0
- data/spec/dummy/tmp/cache/2E3/840/shard%2F3 +0 -0
- data/spec/dummy/tmp/cache/313/970/shard%2F30 +0 -0
- data/spec/dummy/tmp/cache/314/980/shard%2F31 +0 -0
- data/spec/dummy/tmp/cache/316/980/shard%2F15 +1 -0
- data/spec/dummy/tmp/cache/316/9D0/shard%2F60 +0 -0
- data/spec/dummy/tmp/cache/317/990/shard%2F16 +0 -0
- data/spec/dummy/tmp/cache/317/9C0/shard%2F43 +1 -0
- data/spec/dummy/tmp/cache/317/9E0/shard%2F61 +0 -0
- data/spec/dummy/tmp/cache/318/9A0/shard%2F17 +0 -0
- data/spec/dummy/tmp/cache/318/9D0/shard%2F44 +0 -0
- data/spec/dummy/tmp/cache/318/9F0/shard%2F62 +1 -0
- data/spec/dummy/tmp/cache/319/9E0/shard%2F45 +0 -0
- data/spec/dummy/tmp/cache/319/9F0/shard%2F54 +1 -0
- data/spec/dummy/tmp/cache/319/A10/shard%2F72 +1 -0
- data/spec/dummy/tmp/cache/319/A30/shard%2F90 +0 -0
- data/spec/dummy/tmp/cache/31B/9E0/shard%2F29 +1 -0
- data/spec/dummy/tmp/cache/321/AA0/shard%2F89 +0 -0
- data/spec/dummy/tmp/cache/322/AC0/shard%2F99 +1 -0
- data/spec/dummy/tmp/cache/344/D70/shard%2F103 +1 -0
- data/spec/dummy/tmp/cache/345/D80/shard%2F104 +0 -0
- data/spec/dummy/tmp/cache/345/DB0/shard%2F131 +1 -0
- data/spec/dummy/tmp/cache/345/DC0/shard%2F140 +0 -0
- data/spec/dummy/tmp/cache/346/D90/shard%2F105 +0 -0
- data/spec/dummy/tmp/cache/346/DB0/shard%2F123 +0 -0
- data/spec/dummy/tmp/cache/346/DD0/shard%2F222 +1 -0
- data/spec/dummy/tmp/cache/346/DE0/shard%2F150 +0 -0
- data/spec/dummy/tmp/cache/346/DF0/shard%2F240 +1 -0
- data/spec/dummy/tmp/cache/347/DA0/shard%2F106 +1 -0
- data/spec/dummy/tmp/cache/347/DC0/shard%2F124 +0 -0
- data/spec/dummy/tmp/cache/347/DC0/shard%2F205 +1 -0
- data/spec/dummy/tmp/cache/347/E10/shard%2F250 +1 -0
- data/spec/dummy/tmp/cache/348/DF0/shard%2F143 +1 -0
- data/spec/dummy/tmp/cache/348/DF0/shard%2F224 +1 -0
- data/spec/dummy/tmp/cache/348/E10/shard%2F161 +1 -0
- data/spec/dummy/tmp/cache/349/DD0/shard%2F117 +1 -0
- data/spec/dummy/tmp/cache/349/E40/shard%2F180 +1 -0
- data/spec/dummy/tmp/cache/34A/DF0/shard%2F127 +1 -0
- data/spec/dummy/tmp/cache/34A/DF0/shard%2F208 +1 -0
- data/spec/dummy/tmp/cache/34A/E10/shard%2F145 +1 -0
- data/spec/dummy/tmp/cache/34A/E60/shard%2F190 +1 -0
- data/spec/dummy/tmp/cache/34B/E30/shard%2F155 +1 -0
- data/spec/dummy/tmp/cache/34D/E30/shard%2F139 +0 -0
- data/spec/dummy/tmp/cache/34E/E50/shard%2F149 +0 -0
- data/spec/dummy/tmp/cache/353/EF0/shard%2F199 +1 -0
- data/spec/dummy/tmp/cache/3A4/E90/shard%2F10003 +1 -0
- data/spec/dummy/tmp/cache/3A5/ED0/shard%2F10031 +1 -0
- data/spec/dummy/tmp/cache/3A9/EF0/shard%2F10017 +1 -0
- data/spec/lib/active_record/association_spec.rb +305 -0
- data/spec/lib/active_record/attribute_methods_spec.rb +108 -0
- data/spec/lib/active_record/base_spec.rb +66 -0
- data/spec/lib/active_record/calculations_spec.rb +119 -0
- data/spec/lib/active_record/connection_handler_spec.rb +45 -0
- data/spec/lib/active_record/connection_pool_spec.rb +23 -0
- data/spec/lib/active_record/finder_methods_spec.rb +29 -0
- data/spec/lib/active_record/query_cache_spec.rb +20 -0
- data/spec/lib/active_record/query_methods_spec.rb +130 -0
- data/spec/lib/active_record/relation_spec.rb +38 -0
- data/spec/lib/cache_extensions_spec.rb +27 -0
- data/spec/lib/connection_pool_proxy_spec.rb +13 -0
- data/spec/lib/database_server_spec.rb +154 -0
- data/spec/lib/shackles_spec.rb +147 -0
- data/spec/models/shard_spec.rb +382 -0
- data/spec/spec_helper.rb +32 -0
- metadata +344 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: cc14f7783fcf89a84b17781a64cc331aa29f0447
|
4
|
+
data.tar.gz: 7921d4093ae4c1194b03724fa10013d9c71dbcae
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ec8355813399e1a4d0f01c48c299498735ee1bd913fe2083624e161d26e32533c9d5a0695f0ddfa726648f63292fd9a5e1f58ecbdacfc6d0f70274d148cae52b
|
7
|
+
data.tar.gz: b7f54166cf2d7eab085cfe199bd47be4d1566860de6805a67bc3614e86bc04eebef4d80b892b2c5a26a71d5604d5b80a62efb260deee6dbee6f7a8f5190acf34
|
data/Rakefile
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
begin
|
3
|
+
require 'bundler/setup'
|
4
|
+
rescue LoadError
|
5
|
+
puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
|
6
|
+
end
|
7
|
+
begin
|
8
|
+
require 'rdoc/task'
|
9
|
+
rescue LoadError
|
10
|
+
require 'rdoc/rdoc'
|
11
|
+
require 'rake/rdoctask'
|
12
|
+
RDoc::Task = Rake::RDocTask
|
13
|
+
end
|
14
|
+
|
15
|
+
RDoc::Task.new(:rdoc) do |rdoc|
|
16
|
+
rdoc.rdoc_dir = 'rdoc'
|
17
|
+
rdoc.title = 'Switchman'
|
18
|
+
rdoc.options << '--line-numbers'
|
19
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
20
|
+
end
|
21
|
+
|
22
|
+
APP_RAKEFILE = File.expand_path("../spec/dummy/Rakefile", __FILE__)
|
23
|
+
load 'rails/tasks/engine.rake'
|
24
|
+
|
25
|
+
Bundler::GemHelper.install_tasks
|
26
|
+
|
27
|
+
require 'rspec/core/rake_task'
|
28
|
+
RSpec::Core::RakeTask.new
|
29
|
+
|
30
|
+
task :default => :spec
|
@@ -0,0 +1,502 @@
|
|
1
|
+
require_dependency 'switchman/database_server'
|
2
|
+
require_dependency 'switchman/default_shard'
|
3
|
+
|
4
|
+
module Switchman
|
5
|
+
class Shard < ::ActiveRecord::Base
|
6
|
+
# ten trillion possible ids per shard. yup.
|
7
|
+
IDS_PER_SHARD = 10_000_000_000_000
|
8
|
+
|
9
|
+
CATEGORIES =
|
10
|
+
{
|
11
|
+
# special cased to mean all other models
|
12
|
+
:default => nil,
|
13
|
+
# special cased to not allow activating a shard other than the default
|
14
|
+
:unsharded => [Shard]
|
15
|
+
}
|
16
|
+
private_constant :CATEGORIES
|
17
|
+
|
18
|
+
attr_accessible :name, :database_server, :default
|
19
|
+
|
20
|
+
# only allow one default
|
21
|
+
validates_uniqueness_of :default, :if => lambda { |s| s.default? }
|
22
|
+
|
23
|
+
after_save :clear_cache
|
24
|
+
|
25
|
+
|
26
|
+
class << self
|
27
|
+
def categories
|
28
|
+
CATEGORIES.keys
|
29
|
+
end
|
30
|
+
|
31
|
+
def default(reload = false)
|
32
|
+
if !@default || reload
|
33
|
+
# Have to create a dummy object so that several key methods still work
|
34
|
+
# (it's easier to do this in one place here, and just assume that sharding
|
35
|
+
# is up and running everywhere else). This includes for looking up the
|
36
|
+
# default shard itself. This also needs to be a local so that this method
|
37
|
+
# can be re-entrant
|
38
|
+
default = DefaultShard.new
|
39
|
+
|
40
|
+
# the first time we need a dummy dummy for re-entrancy to avoid looping on ourselves
|
41
|
+
@default ||= default
|
42
|
+
|
43
|
+
# Now find the actual record, if it exists; rescue the fake default if the table doesn't exist
|
44
|
+
@default = Shard.find_by_default(true) || default rescue default
|
45
|
+
end
|
46
|
+
@default
|
47
|
+
end
|
48
|
+
|
49
|
+
def current(category = :default)
|
50
|
+
active_shards[category] || Shard.default
|
51
|
+
end
|
52
|
+
|
53
|
+
def activate(shards)
|
54
|
+
old_shards = activate!(shards)
|
55
|
+
yield
|
56
|
+
ensure
|
57
|
+
active_shards.merge!(old_shards)
|
58
|
+
end
|
59
|
+
|
60
|
+
def activate!(shards)
|
61
|
+
old_shards = {}
|
62
|
+
shards.each do |category, shard|
|
63
|
+
next if category == :unsharded
|
64
|
+
old_shards[category] = active_shards[category]
|
65
|
+
active_shards[category] = shard
|
66
|
+
end
|
67
|
+
old_shards
|
68
|
+
end
|
69
|
+
|
70
|
+
def lookup(id)
|
71
|
+
id_i = id.to_i
|
72
|
+
return current if id_i == current.id || id == 'self'
|
73
|
+
return default if id_i == default.id || id.nil? || id == 'default'
|
74
|
+
id = id_i
|
75
|
+
raise ArgumentError if id == 0
|
76
|
+
|
77
|
+
cached_shards[id] ||= Shard.default.activate do
|
78
|
+
# can't simply cache the AR object since Shard has a custom serializer
|
79
|
+
# that calls this method
|
80
|
+
attributes = Rails.cache.fetch(['shard', id].join('/')) do
|
81
|
+
shard = find_by_id(id)
|
82
|
+
shard.try(:attributes) || :nil
|
83
|
+
end
|
84
|
+
if attributes == :nil
|
85
|
+
nil
|
86
|
+
else
|
87
|
+
shard = Shard.new
|
88
|
+
shard.assign_attributes(attributes, :without_protection => true)
|
89
|
+
shard.instance_variable_set(:@new_record, false)
|
90
|
+
# connection info doesn't exist in database.yml;
|
91
|
+
# pretend the shard doesn't exist either
|
92
|
+
shard = nil unless shard.database_server
|
93
|
+
shard
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def clear_cache
|
99
|
+
@cached_shards = {}
|
100
|
+
end
|
101
|
+
|
102
|
+
# options
|
103
|
+
# :parallel - true/false to execute in parallel, or a integer of how many
|
104
|
+
# sub-processes per database server. Note that parallel
|
105
|
+
# invocation currently uses forking, so should be used sparingly
|
106
|
+
# because errors are not raised, and you cannot get results back
|
107
|
+
def with_each_shard(scope = nil, categories = nil, options = {})
|
108
|
+
unless default.is_a?(Shard)
|
109
|
+
return Array(yield)
|
110
|
+
end
|
111
|
+
|
112
|
+
parallel = case options[:parallel]
|
113
|
+
when true
|
114
|
+
1
|
115
|
+
when false, nil
|
116
|
+
0
|
117
|
+
else
|
118
|
+
options[:parallel]
|
119
|
+
end
|
120
|
+
scope ||= Shard.order("database_server_id IS NOT NULL, database_server_id, id")
|
121
|
+
|
122
|
+
if parallel > 0
|
123
|
+
if scope.class == ::ActiveRecord::NamedScope::Scope
|
124
|
+
# still need a post-uniq, cause the default database server could be NULL or Rails.env in the db
|
125
|
+
database_servers = scope.reorder('database_server_id').select(:database_server_id).uniq.
|
126
|
+
map(&:database_server).compact.uniq
|
127
|
+
scopes = Hash[database_servers.map do |server|
|
128
|
+
server_scope = server.shards(scope)
|
129
|
+
if parallel == 1
|
130
|
+
subscopes = [server_scope]
|
131
|
+
else
|
132
|
+
subscopes = []
|
133
|
+
total = server_scope.count
|
134
|
+
ranges = []
|
135
|
+
server_scope.find_ids_in_ranges(:batch_size => (total.to_f / parallel).ceil) do |min, max|
|
136
|
+
ranges << [min, max]
|
137
|
+
end
|
138
|
+
# create a half-open range on the last one
|
139
|
+
ranges.last[1] = nil
|
140
|
+
ranges.each do |min, max|
|
141
|
+
subscope = server_scope.where("id>=?", min)
|
142
|
+
subscope = subscope.where("id<=?", max) if max
|
143
|
+
subscopes << subscope
|
144
|
+
end
|
145
|
+
end
|
146
|
+
[server, subscopes]
|
147
|
+
end]
|
148
|
+
else
|
149
|
+
scopes = scope.group_by(&:database_server)
|
150
|
+
if parallel > 1
|
151
|
+
scopes = Hash[scopes.map do |(server, shards)|
|
152
|
+
[server, shards.in_groups(parallel, false).compact]
|
153
|
+
end]
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
fd_to_name_map = {}
|
158
|
+
fds = []
|
159
|
+
pids = []
|
160
|
+
exception_pipe = IO.pipe
|
161
|
+
scopes.each do |server, subscopes|
|
162
|
+
if subscopes.first.class != ::ActiveRecord::NamedScope::Scope && subscopes.first.class != Array
|
163
|
+
subscopes = [subscopes]
|
164
|
+
end
|
165
|
+
# only one process; don't bother forking
|
166
|
+
if scopes.length == 1 && subscopes.length == 1
|
167
|
+
exception_pipe.first.close
|
168
|
+
exception_pipe.last.close
|
169
|
+
return with_each_shard(subscopes.first, categories) { yield }
|
170
|
+
end
|
171
|
+
subscopes.each_with_index do |subscope, idx|
|
172
|
+
details = Open4.pfork4(lambda do
|
173
|
+
begin
|
174
|
+
::ActiveRecord::Base.clear_all_connections!
|
175
|
+
with_each_shard(subscope, categories) { yield }
|
176
|
+
rescue Exception => e
|
177
|
+
exception_pipe.last.write(Marshal.dump(e))
|
178
|
+
exception_pipe.last.flush
|
179
|
+
exit 1
|
180
|
+
end
|
181
|
+
end)
|
182
|
+
# don't care about writing to stdin
|
183
|
+
details[1].close
|
184
|
+
fds.concat details[2..3]
|
185
|
+
pids << details[0]
|
186
|
+
if subscopes.length > 1
|
187
|
+
name = "#{server.id} #{idx + 1}"
|
188
|
+
else
|
189
|
+
name = server.id
|
190
|
+
end
|
191
|
+
fd_to_name_map[details[2]] = name
|
192
|
+
fd_to_name_map[details[3]] = name
|
193
|
+
end
|
194
|
+
end
|
195
|
+
exception_pipe.last.close
|
196
|
+
|
197
|
+
while !fds.empty?
|
198
|
+
ready, _ = IO.select(fds)
|
199
|
+
ready.each do |fd|
|
200
|
+
if fd.eof?
|
201
|
+
fd.close
|
202
|
+
fds.delete(fd)
|
203
|
+
next
|
204
|
+
end
|
205
|
+
line = fd.readline
|
206
|
+
puts "#{fd_to_name_map[fd]}: #{line}"
|
207
|
+
end
|
208
|
+
end
|
209
|
+
pids.each { |pid| Process.waitpid2(pid) }
|
210
|
+
# I'm not sure why, but we have to do this
|
211
|
+
::ActiveRecord::Base.clear_all_connections!
|
212
|
+
# check for an exception; we only re-raise the first one
|
213
|
+
# (all the sub-processes shared the same pipe, so we only
|
214
|
+
# have to check the one)
|
215
|
+
begin
|
216
|
+
exception = Marshal.load exception_pipe.first
|
217
|
+
raise exception
|
218
|
+
rescue EOFError
|
219
|
+
# No exceptions
|
220
|
+
ensure
|
221
|
+
exception_pipe.first.close
|
222
|
+
end
|
223
|
+
return
|
224
|
+
end
|
225
|
+
|
226
|
+
categories ||= []
|
227
|
+
|
228
|
+
previous_shard = nil
|
229
|
+
close_connections_if_needed = lambda do |shard|
|
230
|
+
# prune the prior connection unless it happened to be the same
|
231
|
+
if previous_shard && shard != previous_shard &&
|
232
|
+
(shard.database_server != previous_shard.database_server || !previous_shard.database_server.shareable?)
|
233
|
+
previous_shard.activate do
|
234
|
+
if ::ActiveRecord::Base.connected? && ::ActiveRecord::Base.connection.open_transactions == 0
|
235
|
+
::ActiveRecord::Base.connection_pool.current_pool.disconnect!
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
result = []
|
242
|
+
scope.each do |shard|
|
243
|
+
# shard references a database server that isn't configured in this environment
|
244
|
+
next unless shard.database_server
|
245
|
+
close_connections_if_needed.call(shard)
|
246
|
+
shard.activate(*categories) do
|
247
|
+
result.concat Array(yield)
|
248
|
+
end
|
249
|
+
previous_shard = shard
|
250
|
+
end
|
251
|
+
close_connections_if_needed.call(Shard.current)
|
252
|
+
result
|
253
|
+
end
|
254
|
+
|
255
|
+
def partition_by_shard(array, partition_proc = nil)
|
256
|
+
shard_arrays = {}
|
257
|
+
array.each do |object|
|
258
|
+
partition_object = partition_proc ? partition_proc.call(object) : object
|
259
|
+
case partition_object
|
260
|
+
when Shard
|
261
|
+
shard = partition_object
|
262
|
+
when ::ActiveRecord::Base
|
263
|
+
if partition_object.respond_to?(:associated_shards)
|
264
|
+
partition_object.associated_shards.each do |a_shard|
|
265
|
+
shard_arrays[a_shard] ||= []
|
266
|
+
shard_arrays[a_shard] << object
|
267
|
+
end
|
268
|
+
next
|
269
|
+
else
|
270
|
+
shard = partition_object.shard
|
271
|
+
end
|
272
|
+
when Fixnum, /^\d+$/, /^(\d+)~(\d+)$/
|
273
|
+
local_id, shard = Shard.local_id_for(partition_object)
|
274
|
+
local_id ||= partition_object
|
275
|
+
object = local_id if !partition_proc
|
276
|
+
end
|
277
|
+
shard ||= Shard.current
|
278
|
+
shard_arrays[shard] ||= []
|
279
|
+
shard_arrays[shard] << object
|
280
|
+
end
|
281
|
+
# TODO: use with_each_shard (or vice versa) to get
|
282
|
+
# connection management and parallelism benefits
|
283
|
+
shard_arrays.inject([]) do |results, (shard, objects)|
|
284
|
+
results.concat shard.activate { Array(yield objects) }
|
285
|
+
end
|
286
|
+
end
|
287
|
+
|
288
|
+
# converts an AR object, integral id, string id, or string short-global-id to a
|
289
|
+
# integral id. nil if it can't be interpreted
|
290
|
+
def integral_id_for(any_id)
|
291
|
+
case any_id
|
292
|
+
when ::ActiveRecord::Base
|
293
|
+
any_id.id
|
294
|
+
when /^(\d+)~(\d+)$/
|
295
|
+
local_id = $2.to_i
|
296
|
+
# doesn't make sense to have a double-global id
|
297
|
+
return nil if local_id > IDS_PER_SHARD
|
298
|
+
$1.to_i * IDS_PER_SHARD + local_id
|
299
|
+
when Fixnum, /^\d+$/
|
300
|
+
any_id.to_i
|
301
|
+
else
|
302
|
+
nil
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
# takes an id-ish, and returns a local id and the shard it's
|
307
|
+
# local to. [nil, nil] if it can't be interpreted. [id, nil]
|
308
|
+
# if it's already a local ID
|
309
|
+
def local_id_for(any_id)
|
310
|
+
id = integral_id_for(any_id)
|
311
|
+
return [nil, nil] unless id
|
312
|
+
if id < IDS_PER_SHARD
|
313
|
+
[id, nil]
|
314
|
+
elsif shard = lookup(id / IDS_PER_SHARD)
|
315
|
+
[id % IDS_PER_SHARD, shard]
|
316
|
+
else
|
317
|
+
[nil, nil]
|
318
|
+
end
|
319
|
+
end
|
320
|
+
|
321
|
+
# takes an id-ish, and returns an integral id relative to
|
322
|
+
# target_shard. returns any_id itself if it can't be interpreted
|
323
|
+
def relative_id_for(any_id, source_shard, target_shard)
|
324
|
+
local_id, shard = local_id_for(any_id)
|
325
|
+
return any_id unless local_id
|
326
|
+
shard ||= source_shard
|
327
|
+
return local_id if shard == target_shard
|
328
|
+
shard.global_id_for(local_id)
|
329
|
+
end
|
330
|
+
|
331
|
+
# takes an id-ish, and returns a shortened global
|
332
|
+
# string id if global, and itself if local.
|
333
|
+
# returns any_id itself if it can't be interpreted
|
334
|
+
def short_id_for(any_id)
|
335
|
+
local_id, shard = local_id_for(any_id)
|
336
|
+
return any_id unless local_id
|
337
|
+
return local_id unless shard
|
338
|
+
"#{shard.id}~#{local_id}"
|
339
|
+
end
|
340
|
+
|
341
|
+
# takes an id-ish, and returns an integral global id.
|
342
|
+
# returns nil if it can't be interpreted
|
343
|
+
def global_id_for(any_id, source_shard = nil)
|
344
|
+
id = integral_id_for(any_id)
|
345
|
+
return any_id unless id
|
346
|
+
if id >= IDS_PER_SHARD
|
347
|
+
id
|
348
|
+
else
|
349
|
+
source_shard ||= Shard.current
|
350
|
+
source_shard.global_id_for(id)
|
351
|
+
end
|
352
|
+
end
|
353
|
+
|
354
|
+
def shard_for(any_id, source_shard = nil)
|
355
|
+
_, shard = local_id_for(any_id)
|
356
|
+
shard || source_shard || Shard.current
|
357
|
+
end
|
358
|
+
|
359
|
+
private
|
360
|
+
# in-process caching
|
361
|
+
def cached_shards
|
362
|
+
@cached_shards ||= []
|
363
|
+
end
|
364
|
+
|
365
|
+
def add_to_cache(shard)
|
366
|
+
cached_shards[shard.id] = shard
|
367
|
+
end
|
368
|
+
|
369
|
+
def remove_from_cache(shard)
|
370
|
+
cached_shards.delete(shard.id)
|
371
|
+
end
|
372
|
+
|
373
|
+
def active_shards
|
374
|
+
Thread.current[:active_shards] ||= {}
|
375
|
+
end
|
376
|
+
end
|
377
|
+
|
378
|
+
def name
|
379
|
+
read_attribute(:name) || default_name
|
380
|
+
end
|
381
|
+
|
382
|
+
def name=(name)
|
383
|
+
write_attribute(:name, @name = name)
|
384
|
+
remove_instance_variable(:@name) if name == nil
|
385
|
+
end
|
386
|
+
|
387
|
+
def database_server
|
388
|
+
@database_server ||= DatabaseServer.find(self.database_server_id)
|
389
|
+
end
|
390
|
+
|
391
|
+
def database_server=(database_server)
|
392
|
+
self.database_server_id = database_server.id
|
393
|
+
@database_server = database_server
|
394
|
+
end
|
395
|
+
|
396
|
+
def description
|
397
|
+
[database_server.id, name].compact.join(':')
|
398
|
+
end
|
399
|
+
|
400
|
+
# Shards are always on the default shard
|
401
|
+
def shard
|
402
|
+
Shard.default
|
403
|
+
end
|
404
|
+
|
405
|
+
def activate(*categories, &block)
|
406
|
+
shards = hashify_categories(categories)
|
407
|
+
Shard.activate(shards, &block)
|
408
|
+
end
|
409
|
+
|
410
|
+
# for use from console ONLY
|
411
|
+
def activate!(*categories)
|
412
|
+
shards = hashify_categories(categories)
|
413
|
+
Shard.activate!(shards)
|
414
|
+
nil
|
415
|
+
end
|
416
|
+
|
417
|
+
# custom serialization, since shard is self-referential
|
418
|
+
def _dump(depth)
|
419
|
+
self.id.to_s
|
420
|
+
end
|
421
|
+
|
422
|
+
def self._load(str)
|
423
|
+
lookup(str.to_i)
|
424
|
+
end
|
425
|
+
|
426
|
+
def drop_database
|
427
|
+
return unless read_attribute(:name)
|
428
|
+
begin
|
429
|
+
adapter = self.database_server.config[:adapter]
|
430
|
+
sharding_config = Switchman.config || {}
|
431
|
+
drop_statement = sharding_config[adapter].try(:[], :drop_statement)
|
432
|
+
drop_statement ||= sharding_config[:drop_statement]
|
433
|
+
if drop_statement
|
434
|
+
drop_statement = Array(drop_statement).dup.
|
435
|
+
map { |statement| statement.gsub('%{db_name}', self.name) }
|
436
|
+
end
|
437
|
+
|
438
|
+
case adapter
|
439
|
+
when 'mysql', 'mysql2'
|
440
|
+
self.activate do
|
441
|
+
::Shackles.activate(:deploy) do
|
442
|
+
drop_statement ||= "DROP DATABASE #{self.name}"
|
443
|
+
Array(drop_statement).each do |stmt|
|
444
|
+
::ActiveRecord::Base.connection.execute(stmt)
|
445
|
+
end
|
446
|
+
end
|
447
|
+
end
|
448
|
+
when 'postgresql'
|
449
|
+
self.activate do
|
450
|
+
::Shackles.activate(:deploy) do
|
451
|
+
# Shut up, Postgres!
|
452
|
+
conn = ::ActiveRecord::Base.connection
|
453
|
+
old_proc = conn.raw_connection.set_notice_processor {}
|
454
|
+
begin
|
455
|
+
drop_statement ||= "DROP SCHEMA #{self.name} CASCADE"
|
456
|
+
Array(drop_statement).each do |stmt|
|
457
|
+
::ActiveRecord::Base.connection.execute(stmt)
|
458
|
+
end
|
459
|
+
ensure
|
460
|
+
conn.raw_connection.set_notice_processor(&old_proc) if old_proc
|
461
|
+
end
|
462
|
+
end
|
463
|
+
end
|
464
|
+
when 'sqlite3'
|
465
|
+
File.delete(self.name) unless self.name == ':memory:'
|
466
|
+
end
|
467
|
+
rescue
|
468
|
+
logger.info "Drop failed: #{$!}"
|
469
|
+
end
|
470
|
+
end
|
471
|
+
|
472
|
+
# takes an id local to this shard, and returns a global id
|
473
|
+
def global_id_for(local_id)
|
474
|
+
return nil unless local_id
|
475
|
+
local_id + self.id * IDS_PER_SHARD
|
476
|
+
end
|
477
|
+
|
478
|
+
private
|
479
|
+
|
480
|
+
def clear_cache
|
481
|
+
Shard.default.activate do
|
482
|
+
Rails.cache.delete(['shard', id].join('/'))
|
483
|
+
end
|
484
|
+
end
|
485
|
+
|
486
|
+
def default_name
|
487
|
+
unless instance_variable_defined?(:@name)
|
488
|
+
# protect against re-entrancy
|
489
|
+
@name = nil
|
490
|
+
@name = database_server.shard_name(self)
|
491
|
+
end
|
492
|
+
@name
|
493
|
+
end
|
494
|
+
|
495
|
+
def hashify_categories(categories)
|
496
|
+
categories = categories.flatten
|
497
|
+
categories << :default if categories.empty?
|
498
|
+
Hash[*categories.map{ |category| [category, self] }.flatten]
|
499
|
+
end
|
500
|
+
|
501
|
+
end
|
502
|
+
end
|