KellyMahan-memcachedb-client 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +3 -0
- data/LICENSE.txt +28 -0
- data/README.rdoc +28 -0
- data/Rakefile +26 -0
- data/lib/continuum_db.rb +77 -0
- data/lib/memcache_db.rb +896 -0
- data/lib/memcache_util_db.rb +102 -0
- data/test/test_mem_cache_db.rb +950 -0
- metadata +69 -0
data/History.txt
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
Copyright 2005-2009 Bob Cottrell, Eric Hodel, Mike Perham, Kelly Mahan.
|
2
|
+
All rights reserved.
|
3
|
+
|
4
|
+
Redistribution and use in source and binary forms, with or without
|
5
|
+
modification, are permitted provided that the following conditions
|
6
|
+
are met:
|
7
|
+
|
8
|
+
1. Redistributions of source code must retain the above copyright
|
9
|
+
notice, this list of conditions and the following disclaimer.
|
10
|
+
2. Redistributions in binary form must reproduce the above copyright
|
11
|
+
notice, this list of conditions and the following disclaimer in the
|
12
|
+
documentation and/or other materials provided with the distribution.
|
13
|
+
3. Neither the names of the authors nor the names of their contributors
|
14
|
+
may be used to endorse or promote products derived from this software
|
15
|
+
without specific prior written permission.
|
16
|
+
|
17
|
+
THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
|
18
|
+
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
19
|
+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
20
|
+
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
|
21
|
+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
22
|
+
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
|
23
|
+
OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
|
24
|
+
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
25
|
+
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
26
|
+
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
27
|
+
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
28
|
+
|
data/README.rdoc
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
= memcachedb-client
|
2
|
+
|
3
|
+
A pure ruby library for accessing memcachedb. Forked from http://github.com/mperham/memcache-client
|
4
|
+
|
5
|
+
Source:
|
6
|
+
|
7
|
+
http://github.com/KellyMahan/memcachedb-client
|
8
|
+
|
9
|
+
== Installing memcachedb-client
|
10
|
+
|
11
|
+
Just install the gem:
|
12
|
+
|
13
|
+
$ sudo gem install memcachedb-client
|
14
|
+
|
15
|
+
== Using memcachedb-client
|
16
|
+
|
17
|
+
With one server:
|
18
|
+
|
19
|
+
CACHE = MemCacheDb.new 'localhost:21201', :namespace => 'my_namespace'
|
20
|
+
|
21
|
+
Or with multiple servers:
|
22
|
+
|
23
|
+
CACHE = MemCacheDb.new %w[one.example.com:21201 two.example.com:21201],
|
24
|
+
:namespace => 'my_namespace'
|
25
|
+
|
26
|
+
See MemCacheDb.new for details. Please note memcachedb-client is not thread-safe
|
27
|
+
by default. You should create a separate instance for each thread in your
|
28
|
+
process.
|
data/Rakefile
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# vim: syntax=Ruby
|
2
|
+
require 'rubygems'
|
3
|
+
require 'rake/rdoctask'
|
4
|
+
require 'rake/testtask'
|
5
|
+
|
6
|
+
task :gem do
|
7
|
+
sh "gem build memcachedb-client.gemspec"
|
8
|
+
end
|
9
|
+
|
10
|
+
task :install => [:gem] do
|
11
|
+
sh "sudo gem install memcachedb-client-*.gem"
|
12
|
+
end
|
13
|
+
|
14
|
+
Rake::RDocTask.new do |rd|
|
15
|
+
rd.main = "README.rdoc"
|
16
|
+
rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
|
17
|
+
rd.rdoc_dir = 'doc'
|
18
|
+
end
|
19
|
+
|
20
|
+
Rake::TestTask.new
|
21
|
+
|
22
|
+
task :default => :test
|
23
|
+
|
24
|
+
task :rcov do
|
25
|
+
`rcov -Ilib test/*.rb`
|
26
|
+
end
|
data/lib/continuum_db.rb
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
module ContinuumDb
|
2
|
+
POINTS_PER_SERVER = 160 # this is the default in libmemcached
|
3
|
+
|
4
|
+
class << self
|
5
|
+
|
6
|
+
begin
|
7
|
+
require 'inline'
|
8
|
+
inline do |builder|
|
9
|
+
builder.c <<-EOM
|
10
|
+
int binary_search(VALUE ary, unsigned int r) {
|
11
|
+
int upper = RARRAY_LEN(ary) - 1;
|
12
|
+
int lower = 0;
|
13
|
+
int idx = 0;
|
14
|
+
ID value = rb_intern("value");
|
15
|
+
|
16
|
+
while (lower <= upper) {
|
17
|
+
idx = (lower + upper) / 2;
|
18
|
+
|
19
|
+
VALUE continuumValue = rb_funcall(RARRAY_PTR(ary)[idx], value, 0);
|
20
|
+
unsigned int l = NUM2UINT(continuumValue);
|
21
|
+
if (l == r) {
|
22
|
+
return idx;
|
23
|
+
}
|
24
|
+
else if (l > r) {
|
25
|
+
upper = idx - 1;
|
26
|
+
}
|
27
|
+
else {
|
28
|
+
lower = idx + 1;
|
29
|
+
}
|
30
|
+
}
|
31
|
+
return upper;
|
32
|
+
}
|
33
|
+
EOM
|
34
|
+
end
|
35
|
+
rescue Exception => e
|
36
|
+
puts "Unable to generate native code, falling back to Ruby: #{e.message}"
|
37
|
+
|
38
|
+
# slow but pure ruby version
|
39
|
+
# Find the closest index in Continuum with value <= the given value
|
40
|
+
def binary_search(ary, value, &block)
|
41
|
+
upper = ary.size - 1
|
42
|
+
lower = 0
|
43
|
+
idx = 0
|
44
|
+
|
45
|
+
while(lower <= upper) do
|
46
|
+
idx = (lower + upper) / 2
|
47
|
+
comp = ary[idx].value <=> value
|
48
|
+
|
49
|
+
if comp == 0
|
50
|
+
return idx
|
51
|
+
elsif comp > 0
|
52
|
+
upper = idx - 1
|
53
|
+
else
|
54
|
+
lower = idx + 1
|
55
|
+
end
|
56
|
+
end
|
57
|
+
return upper
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
|
64
|
+
class Entry
|
65
|
+
attr_reader :value
|
66
|
+
attr_reader :server
|
67
|
+
|
68
|
+
def initialize(val, srv)
|
69
|
+
@value = val
|
70
|
+
@server = srv
|
71
|
+
end
|
72
|
+
|
73
|
+
def inspect
|
74
|
+
"<#{value}, #{server.host}:#{server.port}>"
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
data/lib/memcache_db.rb
ADDED
@@ -0,0 +1,896 @@
|
|
1
|
+
$TESTING = defined?($TESTING) && $TESTING
|
2
|
+
|
3
|
+
require 'socket'
|
4
|
+
require 'thread'
|
5
|
+
require 'timeout'
|
6
|
+
require 'zlib'
|
7
|
+
require 'digest/sha1'
|
8
|
+
|
9
|
+
require 'continuum_db'
|
10
|
+
|
11
|
+
##
|
12
|
+
# A Ruby client library for memcachedb.
|
13
|
+
#
|
14
|
+
|
15
|
+
class MemCacheDb
|
16
|
+
|
17
|
+
##
|
18
|
+
# The version of MemCacheDb you are using.
|
19
|
+
|
20
|
+
VERSION = '1.0.1'
|
21
|
+
|
22
|
+
##
|
23
|
+
# Default options for the cache object.
|
24
|
+
|
25
|
+
DEFAULT_OPTIONS = {
|
26
|
+
:namespace => nil,
|
27
|
+
:readonly => false,
|
28
|
+
:multithread => false,
|
29
|
+
:failover => true,
|
30
|
+
:timeout => 0.5,
|
31
|
+
:logger => nil,
|
32
|
+
}
|
33
|
+
|
34
|
+
##
|
35
|
+
# Default memcachedb port.
|
36
|
+
|
37
|
+
DEFAULT_PORT = 21201
|
38
|
+
|
39
|
+
##
|
40
|
+
# Default memcachedb server weight.
|
41
|
+
|
42
|
+
DEFAULT_WEIGHT = 1
|
43
|
+
|
44
|
+
##
|
45
|
+
# The namespace for this instance
|
46
|
+
|
47
|
+
attr_reader :namespace
|
48
|
+
|
49
|
+
##
|
50
|
+
# The multithread setting for this instance
|
51
|
+
|
52
|
+
attr_reader :multithread
|
53
|
+
|
54
|
+
##
|
55
|
+
# The servers this client talks to. Play at your own peril.
|
56
|
+
|
57
|
+
attr_reader :servers
|
58
|
+
|
59
|
+
##
|
60
|
+
# Socket timeout limit with this client, defaults to 0.25 sec.
|
61
|
+
# Set to nil to disable timeouts.
|
62
|
+
|
63
|
+
attr_reader :timeout
|
64
|
+
|
65
|
+
##
|
66
|
+
# Should the client try to failover to another server if the
|
67
|
+
# first server is down? Defaults to true.
|
68
|
+
|
69
|
+
attr_reader :failover
|
70
|
+
|
71
|
+
##
|
72
|
+
# Log debug/info/warn/error to the given Logger, defaults to nil.
|
73
|
+
|
74
|
+
attr_reader :logger
|
75
|
+
|
76
|
+
##
|
77
|
+
# Accepts a list of +servers+ and a list of +opts+. +servers+ may be
|
78
|
+
# omitted. See +servers=+ for acceptable server list arguments.
|
79
|
+
#
|
80
|
+
# Valid options for +opts+ are:
|
81
|
+
#
|
82
|
+
# [:namespace] Prepends this value to all keys added or retrieved.
|
83
|
+
# [:readonly] Raises an exception on cache writes when true.
|
84
|
+
# [:multithread] Wraps cache access in a Mutex for thread safety.
|
85
|
+
# [:failover] Should the client try to failover to another server if the
|
86
|
+
# first server is down? Defaults to true.
|
87
|
+
# [:timeout] Time to use as the socket read timeout. Defaults to 0.25 sec,
|
88
|
+
# set to nil to disable timeouts (this is a major performance penalty in Ruby 1.8).
|
89
|
+
# [:logger] Logger to use for info/debug output, defaults to nil
|
90
|
+
# Other options are ignored.
|
91
|
+
|
92
|
+
def initialize(*args)
|
93
|
+
servers = []
|
94
|
+
opts = {}
|
95
|
+
|
96
|
+
case args.length
|
97
|
+
when 0 then # NOP
|
98
|
+
when 1 then
|
99
|
+
arg = args.shift
|
100
|
+
case arg
|
101
|
+
when Hash then opts = arg
|
102
|
+
when Array then servers = arg
|
103
|
+
when String then servers = [arg]
|
104
|
+
else raise ArgumentError, 'first argument must be Array, Hash or String'
|
105
|
+
end
|
106
|
+
when 2 then
|
107
|
+
servers, opts = args
|
108
|
+
else
|
109
|
+
raise ArgumentError, "wrong number of arguments (#{args.length} for 2)"
|
110
|
+
end
|
111
|
+
|
112
|
+
opts = DEFAULT_OPTIONS.merge opts
|
113
|
+
@namespace = opts[:namespace]
|
114
|
+
@readonly = opts[:readonly]
|
115
|
+
@multithread = opts[:multithread]
|
116
|
+
@timeout = opts[:timeout]
|
117
|
+
@failover = opts[:failover]
|
118
|
+
@logger = opts[:logger]
|
119
|
+
@mutex = Mutex.new if @multithread
|
120
|
+
|
121
|
+
logger.info { "memcachedb-client #{VERSION} #{Array(servers).inspect}" } if logger
|
122
|
+
|
123
|
+
self.servers = servers
|
124
|
+
end
|
125
|
+
|
126
|
+
##
|
127
|
+
# Returns a string representation of the cache object.
|
128
|
+
|
129
|
+
def inspect
|
130
|
+
"<MemCacheDb: %d servers, ns: %p, ro: %p>" %
|
131
|
+
[@servers.length, @namespace, @readonly]
|
132
|
+
end
|
133
|
+
|
134
|
+
##
|
135
|
+
# Returns whether there is at least one active server for the object.
|
136
|
+
|
137
|
+
def active?
|
138
|
+
not @servers.empty?
|
139
|
+
end
|
140
|
+
|
141
|
+
##
|
142
|
+
# Returns whether or not the cache object was created read only.
|
143
|
+
|
144
|
+
def readonly?
|
145
|
+
@readonly
|
146
|
+
end
|
147
|
+
|
148
|
+
##
|
149
|
+
# Set the servers that the requests will be distributed between. Entries
|
150
|
+
# can be either strings of the form "hostname:port" or
|
151
|
+
# "hostname:port:weight" or MemCacheDb::Server objects.
|
152
|
+
#
|
153
|
+
def servers=(servers)
|
154
|
+
# Create the server objects.
|
155
|
+
@servers = Array(servers).collect do |server|
|
156
|
+
case server
|
157
|
+
when String
|
158
|
+
host, port, weight = server.split ':', 3
|
159
|
+
port ||= DEFAULT_PORT
|
160
|
+
weight ||= DEFAULT_WEIGHT
|
161
|
+
Server.new self, host, port, weight
|
162
|
+
else
|
163
|
+
if server.multithread != @multithread then
|
164
|
+
raise ArgumentError, "can't mix threaded and non-threaded servers"
|
165
|
+
end
|
166
|
+
server
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
logger.debug { "Servers now: #{@servers.inspect}" } if logger
|
171
|
+
|
172
|
+
# There's no point in doing this if there's only one server
|
173
|
+
@continuum = create_continuum_for(@servers) if @servers.size > 1
|
174
|
+
|
175
|
+
@servers
|
176
|
+
end
|
177
|
+
|
178
|
+
##
|
179
|
+
# Decrements the value for +key+ by +amount+ and returns the new value.
|
180
|
+
# +key+ must already exist. If +key+ is not an integer, it is assumed to be
|
181
|
+
# 0. +key+ can not be decremented below 0.
|
182
|
+
|
183
|
+
def decr(key, amount = 1)
|
184
|
+
raise MemCacheDbError, "Update of readonly cache" if @readonly
|
185
|
+
with_server(key) do |server, cache_key|
|
186
|
+
cache_decr server, cache_key, amount
|
187
|
+
end
|
188
|
+
rescue TypeError => err
|
189
|
+
handle_error nil, err
|
190
|
+
end
|
191
|
+
|
192
|
+
##
|
193
|
+
# Retrieves +key+ from MemCacheDb. If +raw+ is false, the value will be
|
194
|
+
# unmarshalled.
|
195
|
+
|
196
|
+
def get(key, raw = false)
|
197
|
+
with_server(key) do |server, cache_key|
|
198
|
+
value = cache_get server, cache_key
|
199
|
+
logger.debug { "GET #{key} from #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
|
200
|
+
return nil if value.nil?
|
201
|
+
value = Marshal.load value unless raw
|
202
|
+
return value
|
203
|
+
end
|
204
|
+
rescue TypeError => err
|
205
|
+
handle_error nil, err
|
206
|
+
end
|
207
|
+
|
208
|
+
##
|
209
|
+
# Retrieves multiple values from memcachedb in parallel, if possible.
|
210
|
+
#
|
211
|
+
# The memcachedb protocol supports the ability to retrieve multiple
|
212
|
+
# keys in a single request. Pass in an array of keys to this method
|
213
|
+
# and it will:
|
214
|
+
#
|
215
|
+
# 1. map the key to the appropriate memcachedb server
|
216
|
+
# 2. send a single request to each server that has one or more key values
|
217
|
+
#
|
218
|
+
# Returns a hash of values.
|
219
|
+
#
|
220
|
+
# cache["a"] = 1
|
221
|
+
# cache["b"] = 2
|
222
|
+
# cache.get_multi "a", "b" # => { "a" => 1, "b" => 2 }
|
223
|
+
|
224
|
+
def get_multi(*keys)
|
225
|
+
raise MemCacheDbError, 'No active servers' unless active?
|
226
|
+
|
227
|
+
keys.flatten!
|
228
|
+
key_count = keys.length
|
229
|
+
cache_keys = {}
|
230
|
+
server_keys = Hash.new { |h,k| h[k] = [] }
|
231
|
+
|
232
|
+
# map keys to servers
|
233
|
+
keys.each do |key|
|
234
|
+
server, cache_key = request_setup key
|
235
|
+
cache_keys[cache_key] = key
|
236
|
+
server_keys[server] << cache_key
|
237
|
+
end
|
238
|
+
|
239
|
+
results = {}
|
240
|
+
|
241
|
+
server_keys.each do |server, keys_for_server|
|
242
|
+
keys_for_server_str = keys_for_server.join ' '
|
243
|
+
begin
|
244
|
+
values = cache_get_multi server, keys_for_server_str
|
245
|
+
values.each do |key, value|
|
246
|
+
results[cache_keys[key]] = Marshal.load value
|
247
|
+
end
|
248
|
+
rescue IndexError => e
|
249
|
+
# Ignore this server and try the others
|
250
|
+
logger.warn { "Unable to retrieve #{keys_for_server.size} elements from #{server.inspect}: #{e.message}"} if logger
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
return results
|
255
|
+
rescue TypeError => err
|
256
|
+
handle_error nil, err
|
257
|
+
end
|
258
|
+
|
259
|
+
##
|
260
|
+
# Increments the value for +key+ by +amount+ and returns the new value.
|
261
|
+
# +key+ must already exist. If +key+ is not an integer, it is assumed to be
|
262
|
+
# 0.
|
263
|
+
|
264
|
+
def incr(key, amount = 1)
|
265
|
+
raise MemCacheDbError, "Update of readonly cache" if @readonly
|
266
|
+
with_server(key) do |server, cache_key|
|
267
|
+
cache_incr server, cache_key, amount
|
268
|
+
end
|
269
|
+
rescue TypeError => err
|
270
|
+
handle_error nil, err
|
271
|
+
end
|
272
|
+
|
273
|
+
##
|
274
|
+
# Add +key+ to the cache with value +value+ that expires in +expiry+
|
275
|
+
# seconds. If +raw+ is true, +value+ will not be Marshalled.
|
276
|
+
#
|
277
|
+
# Warning: Readers should not call this method in the event of a cache miss;
|
278
|
+
# see MemCacheDb#add.
|
279
|
+
|
280
|
+
ONE_MB = 1024 * 1024
|
281
|
+
|
282
|
+
def set(key, value, expiry = 0, raw = false)
|
283
|
+
raise MemCacheDbError, "Update of readonly cache" if @readonly
|
284
|
+
with_server(key) do |server, cache_key|
|
285
|
+
|
286
|
+
value = Marshal.dump value unless raw
|
287
|
+
logger.debug { "SET #{key} to #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
|
288
|
+
|
289
|
+
data = value.to_s
|
290
|
+
raise MemCacheDbError, "Value too large, MemCacheDbd can only store 1MB of data per key" if data.size > ONE_MB
|
291
|
+
|
292
|
+
command = "set #{cache_key} 0 #{expiry} #{data.size}\r\n#{data}\r\n"
|
293
|
+
|
294
|
+
with_socket_management(server) do |socket|
|
295
|
+
socket.write command
|
296
|
+
result = socket.gets
|
297
|
+
raise_on_error_response! result
|
298
|
+
|
299
|
+
if result.nil?
|
300
|
+
server.close
|
301
|
+
raise MemCacheDbError, "lost connection to #{server.host}:#{server.port}"
|
302
|
+
end
|
303
|
+
|
304
|
+
result
|
305
|
+
end
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
##
|
310
|
+
# Add +key+ to the cache with value +value+ that expires in +expiry+
|
311
|
+
# seconds, but only if +key+ does not already exist in the cache.
|
312
|
+
# If +raw+ is true, +value+ will not be Marshalled.
|
313
|
+
#
|
314
|
+
# Readers should call this method in the event of a cache miss, not
|
315
|
+
# MemCacheDb#set or MemCacheDb#[]=.
|
316
|
+
|
317
|
+
def add(key, value, expiry = 0, raw = false)
|
318
|
+
raise MemCacheDbError, "Update of readonly cache" if @readonly
|
319
|
+
with_server(key) do |server, cache_key|
|
320
|
+
value = Marshal.dump value unless raw
|
321
|
+
logger.debug { "ADD #{key} to #{server}: #{value ? value.to_s.size : 'nil'}" } if logger
|
322
|
+
command = "add #{cache_key} 0 #{expiry} #{value.to_s.size}\r\n#{value}\r\n"
|
323
|
+
|
324
|
+
with_socket_management(server) do |socket|
|
325
|
+
socket.write command
|
326
|
+
result = socket.gets
|
327
|
+
raise_on_error_response! result
|
328
|
+
result
|
329
|
+
end
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
##
|
334
|
+
# Removes +key+ from the cache in +expiry+ seconds.
|
335
|
+
|
336
|
+
def delete(key, expiry = 0)
|
337
|
+
raise MemCacheDbError, "Update of readonly cache" if @readonly
|
338
|
+
with_server(key) do |server, cache_key|
|
339
|
+
with_socket_management(server) do |socket|
|
340
|
+
socket.write "delete #{cache_key} #{expiry}\r\n"
|
341
|
+
result = socket.gets
|
342
|
+
raise_on_error_response! result
|
343
|
+
result
|
344
|
+
end
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
348
|
+
##
|
349
|
+
# Flush the cache from all MemCacheDb servers.
|
350
|
+
|
351
|
+
def flush_all
|
352
|
+
raise MemCacheDbError, 'No active servers' unless active?
|
353
|
+
raise MemCacheDbError, "Update of readonly cache" if @readonly
|
354
|
+
|
355
|
+
begin
|
356
|
+
@mutex.lock if @multithread
|
357
|
+
@servers.each do |server|
|
358
|
+
with_socket_management(server) do |socket|
|
359
|
+
socket.write "flush_all\r\n"
|
360
|
+
result = socket.gets
|
361
|
+
raise_on_error_response! result
|
362
|
+
result
|
363
|
+
end
|
364
|
+
end
|
365
|
+
rescue IndexError => err
|
366
|
+
handle_error nil, err
|
367
|
+
ensure
|
368
|
+
@mutex.unlock if @multithread
|
369
|
+
end
|
370
|
+
end
|
371
|
+
|
372
|
+
##
|
373
|
+
# Reset the connection to all memcachedb servers. This should be called if
|
374
|
+
# there is a problem with a cache lookup that might have left the connection
|
375
|
+
# in a corrupted state.
|
376
|
+
|
377
|
+
def reset
|
378
|
+
@servers.each { |server| server.close }
|
379
|
+
end
|
380
|
+
|
381
|
+
##
|
382
|
+
# Returns statistics for each memcachedb server. An explanation of the
|
383
|
+
# statistics can be found in the memcachedb docs:
|
384
|
+
#
|
385
|
+
# http://code.sixapart.com/svn/memcached/trunk/server/doc/protocol.txt
|
386
|
+
#
|
387
|
+
# Example:
|
388
|
+
#
|
389
|
+
# >> pp CACHE.stats
|
390
|
+
# {"localhost:11211"=>
|
391
|
+
# {"bytes"=>4718,
|
392
|
+
# "pid"=>20188,
|
393
|
+
# "connection_structures"=>4,
|
394
|
+
# "time"=>1162278121,
|
395
|
+
# "pointer_size"=>32,
|
396
|
+
# "limit_maxbytes"=>67108864,
|
397
|
+
# "cmd_get"=>14532,
|
398
|
+
# "version"=>"1.2.0",
|
399
|
+
# "bytes_written"=>432583,
|
400
|
+
# "cmd_set"=>32,
|
401
|
+
# "get_misses"=>0,
|
402
|
+
# "total_connections"=>19,
|
403
|
+
# "curr_connections"=>3,
|
404
|
+
# "curr_items"=>4,
|
405
|
+
# "uptime"=>1557,
|
406
|
+
# "get_hits"=>14532,
|
407
|
+
# "total_items"=>32,
|
408
|
+
# "rusage_system"=>0.313952,
|
409
|
+
# "rusage_user"=>0.119981,
|
410
|
+
# "bytes_read"=>190619}}
|
411
|
+
# => nil
|
412
|
+
|
413
|
+
def stats
|
414
|
+
raise MemCacheDbError, "No active servers" unless active?
|
415
|
+
server_stats = {}
|
416
|
+
|
417
|
+
@servers.each do |server|
|
418
|
+
next unless server.alive?
|
419
|
+
|
420
|
+
with_socket_management(server) do |socket|
|
421
|
+
value = nil
|
422
|
+
socket.write "stats\r\n"
|
423
|
+
stats = {}
|
424
|
+
while line = socket.gets do
|
425
|
+
raise_on_error_response! line
|
426
|
+
break if line == "END\r\n"
|
427
|
+
if line =~ /\ASTAT ([\S]+) ([\w\.\:]+)/ then
|
428
|
+
name, value = $1, $2
|
429
|
+
stats[name] = case name
|
430
|
+
when 'version'
|
431
|
+
value
|
432
|
+
when 'rusage_user', 'rusage_system' then
|
433
|
+
seconds, microseconds = value.split(/:/, 2)
|
434
|
+
microseconds ||= 0
|
435
|
+
Float(seconds) + (Float(microseconds) / 1_000_000)
|
436
|
+
else
|
437
|
+
if value =~ /\A\d+\Z/ then
|
438
|
+
value.to_i
|
439
|
+
else
|
440
|
+
value
|
441
|
+
end
|
442
|
+
end
|
443
|
+
end
|
444
|
+
end
|
445
|
+
server_stats["#{server.host}:#{server.port}"] = stats
|
446
|
+
end
|
447
|
+
end
|
448
|
+
|
449
|
+
raise MemCacheDbError, "No active servers" if server_stats.empty?
|
450
|
+
server_stats
|
451
|
+
end
|
452
|
+
|
453
|
+
##
|
454
|
+
# Shortcut to get a value from the cache.
|
455
|
+
|
456
|
+
alias [] get
|
457
|
+
|
458
|
+
##
|
459
|
+
# Shortcut to save a value in the cache. This method does not set an
|
460
|
+
# expiration on the entry. Use set to specify an explicit expiry.
|
461
|
+
|
462
|
+
def []=(key, value)
|
463
|
+
set key, value
|
464
|
+
end
|
465
|
+
|
466
|
+
protected unless $TESTING
|
467
|
+
|
468
|
+
##
|
469
|
+
# Create a key for the cache, incorporating the namespace qualifier if
|
470
|
+
# requested.
|
471
|
+
|
472
|
+
def make_cache_key(key)
|
473
|
+
if namespace.nil? then
|
474
|
+
key
|
475
|
+
else
|
476
|
+
"#{@namespace}:#{key}"
|
477
|
+
end
|
478
|
+
end
|
479
|
+
|
480
|
+
##
|
481
|
+
# Returns an interoperable hash value for +key+. (I think, docs are
|
482
|
+
# sketchy for down servers).
|
483
|
+
|
484
|
+
def hash_for(key)
|
485
|
+
Zlib.crc32(key)
|
486
|
+
end
|
487
|
+
|
488
|
+
##
|
489
|
+
# Pick a server to handle the request based on a hash of the key.
|
490
|
+
|
491
|
+
def get_server_for_key(key, options = {})
|
492
|
+
raise ArgumentError, "illegal character in key #{key.inspect}" if
|
493
|
+
key =~ /\s/
|
494
|
+
raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
|
495
|
+
raise MemCacheDbError, "No servers available" if @servers.empty?
|
496
|
+
return @servers.first if @servers.length == 1
|
497
|
+
|
498
|
+
hkey = hash_for(key)
|
499
|
+
|
500
|
+
20.times do |try|
|
501
|
+
entryidx = Continuum.binary_search(@continuum, hkey)
|
502
|
+
server = @continuum[entryidx].server
|
503
|
+
return server if server.alive?
|
504
|
+
break unless failover
|
505
|
+
hkey = hash_for "#{try}#{key}"
|
506
|
+
end
|
507
|
+
|
508
|
+
raise MemCacheDbError, "No servers available"
|
509
|
+
end
|
510
|
+
|
511
|
+
##
|
512
|
+
# Performs a raw decr for +cache_key+ from +server+. Returns nil if not
|
513
|
+
# found.
|
514
|
+
|
515
|
+
def cache_decr(server, cache_key, amount)
|
516
|
+
with_socket_management(server) do |socket|
|
517
|
+
socket.write "decr #{cache_key} #{amount}\r\n"
|
518
|
+
text = socket.gets
|
519
|
+
raise_on_error_response! text
|
520
|
+
return nil if text == "NOT_FOUND\r\n"
|
521
|
+
return text.to_i
|
522
|
+
end
|
523
|
+
end
|
524
|
+
|
525
|
+
##
|
526
|
+
# Fetches the raw data for +cache_key+ from +server+. Returns nil on cache
|
527
|
+
# miss.
|
528
|
+
|
529
|
+
def cache_get(server, cache_key)
|
530
|
+
with_socket_management(server) do |socket|
|
531
|
+
socket.write "get #{cache_key}\r\n"
|
532
|
+
keyline = socket.gets # "VALUE <key> <flags> <bytes>\r\n"
|
533
|
+
|
534
|
+
if keyline.nil? then
|
535
|
+
server.close
|
536
|
+
raise MemCacheDbError, "lost connection to #{server.host}:#{server.port}"
|
537
|
+
end
|
538
|
+
|
539
|
+
raise_on_error_response! keyline
|
540
|
+
return nil if keyline == "END\r\n"
|
541
|
+
|
542
|
+
unless keyline =~ /(\d+)\r/ then
|
543
|
+
server.close
|
544
|
+
raise MemCacheDbError, "unexpected response #{keyline.inspect}"
|
545
|
+
end
|
546
|
+
value = socket.read $1.to_i
|
547
|
+
socket.read 2 # "\r\n"
|
548
|
+
socket.gets # "END\r\n"
|
549
|
+
return value
|
550
|
+
end
|
551
|
+
end
|
552
|
+
|
553
|
+
##
|
554
|
+
# Fetches +cache_keys+ from +server+ using a multi-get.
|
555
|
+
|
556
|
+
def cache_get_multi(server, cache_keys)
|
557
|
+
with_socket_management(server) do |socket|
|
558
|
+
values = {}
|
559
|
+
socket.write "get #{cache_keys}\r\n"
|
560
|
+
|
561
|
+
while keyline = socket.gets do
|
562
|
+
return values if keyline == "END\r\n"
|
563
|
+
raise_on_error_response! keyline
|
564
|
+
|
565
|
+
unless keyline =~ /\AVALUE (.+) (.+) (.+)/ then
|
566
|
+
server.close
|
567
|
+
raise MemCacheDbError, "unexpected response #{keyline.inspect}"
|
568
|
+
end
|
569
|
+
|
570
|
+
key, data_length = $1, $3
|
571
|
+
values[$1] = socket.read data_length.to_i
|
572
|
+
socket.read(2) # "\r\n"
|
573
|
+
end
|
574
|
+
|
575
|
+
server.close
|
576
|
+
raise MemCacheDbError, "lost connection to #{server.host}:#{server.port}" # TODO: retry here too
|
577
|
+
end
|
578
|
+
end
|
579
|
+
|
580
|
+
##
|
581
|
+
# Performs a raw incr for +cache_key+ from +server+. Returns nil if not
|
582
|
+
# found.
|
583
|
+
|
584
|
+
def cache_incr(server, cache_key, amount)
|
585
|
+
with_socket_management(server) do |socket|
|
586
|
+
socket.write "incr #{cache_key} #{amount}\r\n"
|
587
|
+
text = socket.gets
|
588
|
+
raise_on_error_response! text
|
589
|
+
return nil if text == "NOT_FOUND\r\n"
|
590
|
+
return text.to_i
|
591
|
+
end
|
592
|
+
end
|
593
|
+
|
594
|
+
##
|
595
|
+
# Gets or creates a socket connected to the given server, and yields it
|
596
|
+
# to the block, wrapped in a mutex synchronization if @multithread is true.
|
597
|
+
#
|
598
|
+
# If a socket error (SocketError, SystemCallError, IOError) or protocol error
|
599
|
+
# (MemCacheDbError) is raised by the block, closes the socket, attempts to
|
600
|
+
# connect again, and retries the block (once). If an error is again raised,
|
601
|
+
# reraises it as MemCacheDbError.
|
602
|
+
#
|
603
|
+
# If unable to connect to the server (or if in the reconnect wait period),
|
604
|
+
# raises MemCacheDbError. Note that the socket connect code marks a server
|
605
|
+
# dead for a timeout period, so retrying does not apply to connection attempt
|
606
|
+
# failures (but does still apply to unexpectedly lost connections etc.).
|
607
|
+
|
608
|
+
def with_socket_management(server, &block)
|
609
|
+
@mutex.lock if @multithread
|
610
|
+
retried = false
|
611
|
+
|
612
|
+
begin
|
613
|
+
socket = server.socket
|
614
|
+
|
615
|
+
# Raise an IndexError to show this server is out of whack. If were inside
|
616
|
+
# a with_server block, we'll catch it and attempt to restart the operation.
|
617
|
+
|
618
|
+
raise IndexError, "No connection to server (#{server.status})" if socket.nil?
|
619
|
+
|
620
|
+
block.call(socket)
|
621
|
+
|
622
|
+
rescue SocketError => err
|
623
|
+
logger.warn { "Socket failure: #{err.message}" } if logger
|
624
|
+
server.mark_dead(err)
|
625
|
+
handle_error(server, err)
|
626
|
+
|
627
|
+
rescue MemCacheDbError, SystemCallError, IOError => err
|
628
|
+
logger.warn { "Generic failure: #{err.class.name}: #{err.message}" } if logger
|
629
|
+
handle_error(server, err) if retried || socket.nil?
|
630
|
+
retried = true
|
631
|
+
retry
|
632
|
+
end
|
633
|
+
ensure
|
634
|
+
@mutex.unlock if @multithread
|
635
|
+
end
|
636
|
+
|
637
|
+
def with_server(key)
|
638
|
+
retried = false
|
639
|
+
begin
|
640
|
+
server, cache_key = request_setup(key)
|
641
|
+
yield server, cache_key
|
642
|
+
rescue IndexError => e
|
643
|
+
logger.warn { "Server failed: #{e.class.name}: #{e.message}" } if logger
|
644
|
+
if !retried && @servers.size > 1
|
645
|
+
logger.info { "Connection to server #{server.inspect} DIED! Retrying operation..." } if logger
|
646
|
+
retried = true
|
647
|
+
retry
|
648
|
+
end
|
649
|
+
handle_error(nil, e)
|
650
|
+
end
|
651
|
+
end
|
652
|
+
|
653
|
+
##
|
654
|
+
# Handles +error+ from +server+.
|
655
|
+
|
656
|
+
def handle_error(server, error)
|
657
|
+
raise error if error.is_a?(MemCacheDbError)
|
658
|
+
server.close if server
|
659
|
+
new_error = MemCacheDbError.new error.message
|
660
|
+
new_error.set_backtrace error.backtrace
|
661
|
+
raise new_error
|
662
|
+
end
|
663
|
+
|
664
|
+
##
|
665
|
+
# Performs setup for making a request with +key+ from memcachedb. Returns
|
666
|
+
# the server to fetch the key from and the complete key to use.
|
667
|
+
|
668
|
+
def request_setup(key)
|
669
|
+
raise MemCacheDbError, 'No active servers' unless active?
|
670
|
+
cache_key = make_cache_key key
|
671
|
+
server = get_server_for_key cache_key
|
672
|
+
return server, cache_key
|
673
|
+
end
|
674
|
+
|
675
|
+
def raise_on_error_response!(response)
|
676
|
+
if response =~ /\A(?:CLIENT_|SERVER_)?ERROR(.*)/
|
677
|
+
raise MemCacheDbError, $1.strip
|
678
|
+
end
|
679
|
+
end
|
680
|
+
|
681
|
+
def create_continuum_for(servers)
|
682
|
+
total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
|
683
|
+
continuum = []
|
684
|
+
|
685
|
+
servers.each do |server|
|
686
|
+
entry_count_for(server, servers.size, total_weight).times do |idx|
|
687
|
+
hash = Digest::SHA1.hexdigest("#{server.host}:#{server.port}:#{idx}")
|
688
|
+
value = Integer("0x#{hash[0..7]}")
|
689
|
+
continuum << Continuum::Entry.new(value, server)
|
690
|
+
end
|
691
|
+
end
|
692
|
+
|
693
|
+
continuum.sort { |a, b| a.value <=> b.value }
|
694
|
+
end
|
695
|
+
|
696
|
+
def entry_count_for(server, total_servers, total_weight)
|
697
|
+
((total_servers * Continuum::POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
|
698
|
+
end
|
699
|
+
|
700
|
+
##
|
701
|
+
# This class represents a memcachedb server instance.
|
702
|
+
|
703
|
+
class Server
|
704
|
+
|
705
|
+
##
|
706
|
+
# The amount of time to wait to establish a connection with a memcachedb
|
707
|
+
# server. If a connection cannot be established within this time limit,
|
708
|
+
# the server will be marked as down.
|
709
|
+
|
710
|
+
CONNECT_TIMEOUT = 0.25
|
711
|
+
|
712
|
+
##
|
713
|
+
# The amount of time to wait before attempting to re-establish a
|
714
|
+
# connection with a server that is marked dead.
|
715
|
+
|
716
|
+
RETRY_DELAY = 30.0
|
717
|
+
|
718
|
+
##
|
719
|
+
# The host the memcachedb server is running on.
|
720
|
+
|
721
|
+
attr_reader :host
|
722
|
+
|
723
|
+
##
|
724
|
+
# The port the memcachedb server is listening on.
|
725
|
+
|
726
|
+
attr_reader :port
|
727
|
+
|
728
|
+
##
|
729
|
+
# The weight given to the server.
|
730
|
+
|
731
|
+
attr_reader :weight
|
732
|
+
|
733
|
+
##
|
734
|
+
# The time of next retry if the connection is dead.
|
735
|
+
|
736
|
+
attr_reader :retry
|
737
|
+
|
738
|
+
##
|
739
|
+
# A text status string describing the state of the server.
|
740
|
+
|
741
|
+
attr_reader :status
|
742
|
+
|
743
|
+
attr_reader :multithread
|
744
|
+
attr_reader :logger
|
745
|
+
|
746
|
+
##
|
747
|
+
# Create a new MemCacheDb::Server object for the memcachedb instance
|
748
|
+
# listening on the given host and port, weighted by the given weight.
|
749
|
+
|
750
|
+
def initialize(memcache, host, port = DEFAULT_PORT, weight = DEFAULT_WEIGHT)
|
751
|
+
raise ArgumentError, "No host specified" if host.nil? or host.empty?
|
752
|
+
raise ArgumentError, "No port specified" if port.nil? or port.to_i.zero?
|
753
|
+
|
754
|
+
@host = host
|
755
|
+
@port = port.to_i
|
756
|
+
@weight = weight.to_i
|
757
|
+
|
758
|
+
@multithread = memcache.multithread
|
759
|
+
@mutex = Mutex.new
|
760
|
+
|
761
|
+
@sock = nil
|
762
|
+
@retry = nil
|
763
|
+
@status = 'NOT CONNECTED'
|
764
|
+
@timeout = memcache.timeout
|
765
|
+
@logger = memcache.logger
|
766
|
+
end
|
767
|
+
|
768
|
+
##
|
769
|
+
# Return a string representation of the server object.
|
770
|
+
|
771
|
+
def inspect
|
772
|
+
"<MemCacheDb::Server: %s:%d [%d] (%s)>" % [@host, @port, @weight, @status]
|
773
|
+
end
|
774
|
+
|
775
|
+
##
|
776
|
+
# Check whether the server connection is alive. This will cause the
|
777
|
+
# socket to attempt to connect if it isn't already connected and or if
|
778
|
+
# the server was previously marked as down and the retry time has
|
779
|
+
# been exceeded.
|
780
|
+
|
781
|
+
def alive?
|
782
|
+
!!socket
|
783
|
+
end
|
784
|
+
|
785
|
+
##
|
786
|
+
# Try to connect to the memcachedb server targeted by this object.
|
787
|
+
# Returns the connected socket object on success or nil on failure.
|
788
|
+
|
789
|
+
def socket
|
790
|
+
@mutex.lock if @multithread
|
791
|
+
return @sock if @sock and not @sock.closed?
|
792
|
+
|
793
|
+
@sock = nil
|
794
|
+
|
795
|
+
# If the host was dead, don't retry for a while.
|
796
|
+
return if @retry and @retry > Time.now
|
797
|
+
|
798
|
+
# Attempt to connect if not already connected.
|
799
|
+
begin
|
800
|
+
@sock = @timeout ? TCPTimeoutSocket.new(@host, @port, @timeout) : TCPSocket.new(@host, @port)
|
801
|
+
|
802
|
+
if Socket.constants.include? 'TCP_NODELAY' then
|
803
|
+
@sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
|
804
|
+
end
|
805
|
+
@retry = nil
|
806
|
+
@status = 'CONNECTED'
|
807
|
+
rescue SocketError, SystemCallError, IOError, Timeout::Error => err
|
808
|
+
logger.warn { "Unable to open socket: #{err.class.name}, #{err.message}" } if logger
|
809
|
+
mark_dead err
|
810
|
+
end
|
811
|
+
|
812
|
+
return @sock
|
813
|
+
ensure
|
814
|
+
@mutex.unlock if @multithread
|
815
|
+
end
|
816
|
+
|
817
|
+
##
|
818
|
+
# Close the connection to the memcachedb server targeted by this
|
819
|
+
# object. The server is not considered dead.
|
820
|
+
|
821
|
+
def close
|
822
|
+
@mutex.lock if @multithread
|
823
|
+
@sock.close if @sock && !@sock.closed?
|
824
|
+
@sock = nil
|
825
|
+
@retry = nil
|
826
|
+
@status = "NOT CONNECTED"
|
827
|
+
ensure
|
828
|
+
@mutex.unlock if @multithread
|
829
|
+
end
|
830
|
+
|
831
|
+
##
|
832
|
+
# Mark the server as dead and close its socket.
|
833
|
+
|
834
|
+
def mark_dead(error)
|
835
|
+
@sock.close if @sock && !@sock.closed?
|
836
|
+
@sock = nil
|
837
|
+
@retry = Time.now + RETRY_DELAY
|
838
|
+
|
839
|
+
reason = "#{error.class.name}: #{error.message}"
|
840
|
+
@status = sprintf "%s:%s DEAD (%s), will retry at %s", @host, @port, reason, @retry
|
841
|
+
@logger.info { @status } if @logger
|
842
|
+
end
|
843
|
+
|
844
|
+
end
|
845
|
+
|
846
|
+
##
|
847
|
+
# Base MemCacheDb exception class.
|
848
|
+
|
849
|
+
class MemCacheDbError < RuntimeError; end
|
850
|
+
|
851
|
+
end
|
852
|
+
|
853
|
+
# TCPSocket facade class which implements timeouts.
|
854
|
+
class TCPTimeoutSocket
|
855
|
+
|
856
|
+
def initialize(host, port, timeout)
|
857
|
+
Timeout::timeout(MemCacheDb::Server::CONNECT_TIMEOUT, SocketError) do
|
858
|
+
@sock = TCPSocket.new(host, port)
|
859
|
+
@len = timeout
|
860
|
+
end
|
861
|
+
end
|
862
|
+
|
863
|
+
def write(*args)
|
864
|
+
Timeout::timeout(@len, SocketError) do
|
865
|
+
@sock.write(*args)
|
866
|
+
end
|
867
|
+
end
|
868
|
+
|
869
|
+
def gets(*args)
|
870
|
+
Timeout::timeout(@len, SocketError) do
|
871
|
+
@sock.gets(*args)
|
872
|
+
end
|
873
|
+
end
|
874
|
+
|
875
|
+
def read(*args)
|
876
|
+
Timeout::timeout(@len, SocketError) do
|
877
|
+
@sock.read(*args)
|
878
|
+
end
|
879
|
+
end
|
880
|
+
|
881
|
+
def _socket
|
882
|
+
@sock
|
883
|
+
end
|
884
|
+
|
885
|
+
def method_missing(meth, *args)
|
886
|
+
@sock.__send__(meth, *args)
|
887
|
+
end
|
888
|
+
|
889
|
+
def closed?
|
890
|
+
@sock.closed?
|
891
|
+
end
|
892
|
+
|
893
|
+
def close
|
894
|
+
@sock.close
|
895
|
+
end
|
896
|
+
end
|