activesupport 2.2.3 → 2.3.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of activesupport might be problematic. Click here for more details.
- data/CHANGELOG +128 -89
- data/lib/active_support.rb +31 -33
- data/lib/active_support/backtrace_cleaner.rb +72 -0
- data/lib/active_support/buffered_logger.rb +9 -7
- data/lib/active_support/cache.rb +13 -8
- data/lib/active_support/cache/drb_store.rb +2 -3
- data/lib/active_support/cache/mem_cache_store.rb +6 -1
- data/lib/active_support/cache/strategy/local_cache.rb +104 -0
- data/lib/active_support/callbacks.rb +20 -21
- data/lib/active_support/core_ext.rb +1 -1
- data/lib/active_support/core_ext/array.rb +2 -0
- data/lib/active_support/core_ext/array/conversions.rb +26 -13
- data/lib/active_support/core_ext/array/wrapper.rb +24 -0
- data/lib/active_support/core_ext/benchmark.rb +13 -6
- data/lib/active_support/core_ext/cgi/escape_skipping_slashes.rb +14 -5
- data/lib/active_support/core_ext/class/attribute_accessors.rb +24 -24
- data/lib/active_support/core_ext/class/delegating_attributes.rb +20 -19
- data/lib/active_support/core_ext/class/inheritable_attributes.rb +34 -34
- data/lib/active_support/core_ext/date/conversions.rb +3 -3
- data/lib/active_support/core_ext/date_time/conversions.rb +1 -1
- data/lib/active_support/core_ext/enumerable.rb +9 -0
- data/lib/active_support/core_ext/exception.rb +12 -8
- data/lib/active_support/core_ext/file/atomic.rb +2 -2
- data/lib/active_support/core_ext/hash/conversions.rb +32 -54
- data/lib/active_support/core_ext/hash/indifferent_access.rb +6 -0
- data/lib/active_support/core_ext/hash/keys.rb +1 -1
- data/lib/active_support/core_ext/hash/slice.rb +8 -1
- data/lib/active_support/core_ext/logger.rb +8 -6
- data/lib/active_support/core_ext/module/aliasing.rb +3 -3
- data/lib/active_support/core_ext/module/attr_accessor_with_default.rb +4 -4
- data/lib/active_support/core_ext/module/attribute_accessors.rb +24 -24
- data/lib/active_support/core_ext/module/delegation.rb +29 -3
- data/lib/active_support/core_ext/module/synchronization.rb +5 -5
- data/lib/active_support/core_ext/object/conversions.rb +2 -1
- data/lib/active_support/core_ext/object/misc.rb +16 -0
- data/lib/active_support/core_ext/range/conversions.rb +1 -1
- data/lib/active_support/core_ext/rexml.rb +29 -24
- data/lib/active_support/core_ext/string/inflections.rb +3 -3
- data/lib/active_support/core_ext/time/calculations.rb +1 -2
- data/lib/active_support/core_ext/time/conversions.rb +1 -1
- data/lib/active_support/core_ext/try.rb +36 -0
- data/lib/active_support/dependencies.rb +18 -14
- data/lib/active_support/deprecation.rb +10 -57
- data/lib/active_support/duration.rb +3 -1
- data/lib/active_support/inflections.rb +1 -0
- data/lib/active_support/inflector.rb +16 -7
- data/lib/active_support/json/decoding.rb +21 -3
- data/lib/active_support/json/encoders/date.rb +1 -1
- data/lib/active_support/json/encoders/date_time.rb +1 -1
- data/lib/active_support/json/encoders/hash.rb +10 -11
- data/lib/active_support/json/encoders/time.rb +1 -1
- data/lib/active_support/json/encoding.rb +23 -29
- data/lib/active_support/locale/en.yml +3 -2
- data/lib/active_support/memoizable.rb +61 -43
- data/lib/active_support/message_encryptor.rb +70 -0
- data/lib/active_support/message_verifier.rb +46 -0
- data/lib/active_support/multibyte.rb +6 -30
- data/lib/active_support/multibyte/chars.rb +30 -9
- data/lib/active_support/multibyte/unicode_database.rb +4 -4
- data/lib/active_support/option_merger.rb +7 -1
- data/lib/active_support/ordered_hash.rb +75 -27
- data/lib/active_support/secure_random.rb +8 -6
- data/lib/active_support/test_case.rb +32 -17
- data/lib/active_support/testing/{core_ext/test/unit/assertions.rb → assertions.rb} +13 -20
- data/lib/active_support/testing/declarative.rb +21 -0
- data/lib/active_support/testing/deprecation.rb +55 -0
- data/lib/active_support/testing/performance.rb +1 -1
- data/lib/active_support/testing/setup_and_teardown.rb +57 -86
- data/lib/active_support/time_with_zone.rb +8 -6
- data/lib/active_support/values/time_zone.rb +1 -0
- data/lib/active_support/vendor.rb +6 -11
- data/lib/active_support/vendor/i18n-0.1.3/MIT-LICENSE +20 -0
- data/lib/active_support/vendor/i18n-0.1.3/README.textile +20 -0
- data/lib/active_support/vendor/i18n-0.1.3/Rakefile +5 -0
- data/lib/active_support/vendor/i18n-0.1.3/i18n.gemspec +27 -0
- data/lib/active_support/vendor/{i18n-0.0.1 → i18n-0.1.3/lib}/i18n.rb +42 -37
- data/lib/active_support/vendor/{i18n-0.0.1 → i18n-0.1.3/lib}/i18n/backend/simple.rb +37 -39
- data/lib/active_support/vendor/{i18n-0.0.1 → i18n-0.1.3/lib}/i18n/exceptions.rb +3 -3
- data/lib/active_support/vendor/i18n-0.1.3/test/all.rb +5 -0
- data/lib/active_support/vendor/i18n-0.1.3/test/i18n_exceptions_test.rb +100 -0
- data/lib/active_support/vendor/i18n-0.1.3/test/i18n_test.rb +125 -0
- data/lib/active_support/vendor/i18n-0.1.3/test/locale/en.rb +1 -0
- data/lib/active_support/vendor/i18n-0.1.3/test/locale/en.yml +3 -0
- data/lib/active_support/vendor/i18n-0.1.3/test/simple_backend_test.rb +568 -0
- data/lib/active_support/vendor/{memcache-client-1.5.1 → memcache-client-1.6.5}/memcache.rb +381 -295
- data/lib/active_support/version.rb +2 -2
- data/lib/active_support/xml_mini.rb +31 -0
- data/lib/active_support/xml_mini/libxml.rb +133 -0
- data/lib/active_support/xml_mini/nokogiri.rb +77 -0
- data/lib/active_support/xml_mini/rexml.rb +108 -0
- metadata +85 -14
- data/lib/active_support/multibyte/utils.rb +0 -61
- data/lib/active_support/testing/core_ext/test.rb +0 -6
- data/lib/active_support/vendor/xml-simple-1.0.11/xmlsimple.rb +0 -1021
@@ -1,75 +1,21 @@
|
|
1
|
-
|
2
|
-
# The Robot Co-op. All rights reserved.
|
3
|
-
#
|
4
|
-
# Redistribution and use in source and binary forms, with or without
|
5
|
-
# modification, are permitted provided that the following conditions
|
6
|
-
# are met:
|
7
|
-
#
|
8
|
-
# 1. Redistributions of source code must retain the above copyright
|
9
|
-
# notice, this list of conditions and the following disclaimer.
|
10
|
-
# 2. Redistributions in binary form must reproduce the above copyright
|
11
|
-
# notice, this list of conditions and the following disclaimer in the
|
12
|
-
# documentation and/or other materials provided with the distribution.
|
13
|
-
# 3. Neither the names of the authors nor the names of their contributors
|
14
|
-
# may be used to endorse or promote products derived from this software
|
15
|
-
# without specific prior written permission.
|
16
|
-
#
|
17
|
-
# THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
|
18
|
-
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
19
|
-
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
20
|
-
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
|
21
|
-
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
22
|
-
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
|
23
|
-
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
|
24
|
-
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
25
|
-
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
26
|
-
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
27
|
-
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
28
|
-
|
1
|
+
$TESTING = defined?($TESTING) && $TESTING
|
29
2
|
|
30
3
|
require 'socket'
|
31
4
|
require 'thread'
|
32
5
|
require 'timeout'
|
33
|
-
require '
|
34
|
-
|
35
|
-
class String
|
36
|
-
|
37
|
-
##
|
38
|
-
# Uses the ITU-T polynomial in the CRC32 algorithm.
|
39
|
-
|
40
|
-
def crc32_ITU_T
|
41
|
-
n = length
|
42
|
-
r = 0xFFFFFFFF
|
43
|
-
|
44
|
-
n.times do |i|
|
45
|
-
r ^= self[i]
|
46
|
-
8.times do
|
47
|
-
if (r & 1) != 0 then
|
48
|
-
r = (r>>1) ^ 0xEDB88320
|
49
|
-
else
|
50
|
-
r >>= 1
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
r ^ 0xFFFFFFFF
|
56
|
-
end
|
57
|
-
|
58
|
-
end
|
6
|
+
require 'zlib'
|
7
|
+
require 'digest/sha1'
|
59
8
|
|
60
9
|
##
|
61
10
|
# A Ruby client library for memcached.
|
62
11
|
#
|
63
|
-
# This is intended to provide access to basic memcached functionality. It
|
64
|
-
# does not attempt to be complete implementation of the entire API, but it is
|
65
|
-
# approaching a complete implementation.
|
66
12
|
|
67
13
|
class MemCache
|
68
14
|
|
69
15
|
##
|
70
16
|
# The version of MemCache you are using.
|
71
17
|
|
72
|
-
VERSION = '1.
|
18
|
+
VERSION = '1.6.4.99'
|
73
19
|
|
74
20
|
##
|
75
21
|
# Default options for the cache object.
|
@@ -77,7 +23,10 @@ class MemCache
|
|
77
23
|
DEFAULT_OPTIONS = {
|
78
24
|
:namespace => nil,
|
79
25
|
:readonly => false,
|
80
|
-
:multithread =>
|
26
|
+
:multithread => true,
|
27
|
+
:failover => true,
|
28
|
+
:timeout => 0.5,
|
29
|
+
:logger => nil,
|
81
30
|
}
|
82
31
|
|
83
32
|
##
|
@@ -90,13 +39,6 @@ class MemCache
|
|
90
39
|
|
91
40
|
DEFAULT_WEIGHT = 1
|
92
41
|
|
93
|
-
##
|
94
|
-
# The amount of time to wait for a response from a memcached server. If a
|
95
|
-
# response is not completed within this time, the connection to the server
|
96
|
-
# will be closed and an error will be raised.
|
97
|
-
|
98
|
-
attr_accessor :request_timeout
|
99
|
-
|
100
42
|
##
|
101
43
|
# The namespace for this instance
|
102
44
|
|
@@ -112,6 +54,23 @@ class MemCache
|
|
112
54
|
|
113
55
|
attr_reader :servers
|
114
56
|
|
57
|
+
##
|
58
|
+
# Socket timeout limit with this client, defaults to 0.5 sec.
|
59
|
+
# Set to nil to disable timeouts.
|
60
|
+
|
61
|
+
attr_reader :timeout
|
62
|
+
|
63
|
+
##
|
64
|
+
# Should the client try to failover to another server if the
|
65
|
+
# first server is down? Defaults to true.
|
66
|
+
|
67
|
+
attr_reader :failover
|
68
|
+
|
69
|
+
##
|
70
|
+
# Log debug/info/warn/error to the given Logger, defaults to nil.
|
71
|
+
|
72
|
+
attr_reader :logger
|
73
|
+
|
115
74
|
##
|
116
75
|
# Accepts a list of +servers+ and a list of +opts+. +servers+ may be
|
117
76
|
# omitted. See +servers=+ for acceptable server list arguments.
|
@@ -121,7 +80,11 @@ class MemCache
|
|
121
80
|
# [:namespace] Prepends this value to all keys added or retrieved.
|
122
81
|
# [:readonly] Raises an exception on cache writes when true.
|
123
82
|
# [:multithread] Wraps cache access in a Mutex for thread safety.
|
124
|
-
#
|
83
|
+
# [:failover] Should the client try to failover to another server if the
|
84
|
+
# first server is down? Defaults to true.
|
85
|
+
# [:timeout] Time to use as the socket read timeout. Defaults to 0.5 sec,
|
86
|
+
# set to nil to disable timeouts (this is a major performance penalty in Ruby 1.8).
|
87
|
+
# [:logger] Logger to use for info/debug output, defaults to nil
|
125
88
|
# Other options are ignored.
|
126
89
|
|
127
90
|
def initialize(*args)
|
@@ -148,8 +111,15 @@ class MemCache
|
|
148
111
|
@namespace = opts[:namespace]
|
149
112
|
@readonly = opts[:readonly]
|
150
113
|
@multithread = opts[:multithread]
|
114
|
+
@timeout = opts[:timeout]
|
115
|
+
@failover = opts[:failover]
|
116
|
+
@logger = opts[:logger]
|
151
117
|
@mutex = Mutex.new if @multithread
|
152
|
-
|
118
|
+
|
119
|
+
logger.info { "memcache-client #{VERSION} #{Array(servers).inspect}" } if logger
|
120
|
+
|
121
|
+
Thread.current[:memcache_client] = self.object_id if !@multithread
|
122
|
+
|
153
123
|
self.servers = servers
|
154
124
|
end
|
155
125
|
|
@@ -157,8 +127,8 @@ class MemCache
|
|
157
127
|
# Returns a string representation of the cache object.
|
158
128
|
|
159
129
|
def inspect
|
160
|
-
"<MemCache: %d servers,
|
161
|
-
[@servers.length, @
|
130
|
+
"<MemCache: %d servers, ns: %p, ro: %p>" %
|
131
|
+
[@servers.length, @namespace, @readonly]
|
162
132
|
end
|
163
133
|
|
164
134
|
##
|
@@ -179,31 +149,27 @@ class MemCache
|
|
179
149
|
# Set the servers that the requests will be distributed between. Entries
|
180
150
|
# can be either strings of the form "hostname:port" or
|
181
151
|
# "hostname:port:weight" or MemCache::Server objects.
|
182
|
-
|
152
|
+
#
|
183
153
|
def servers=(servers)
|
184
154
|
# Create the server objects.
|
185
|
-
@servers = servers.collect do |server|
|
155
|
+
@servers = Array(servers).collect do |server|
|
186
156
|
case server
|
187
157
|
when String
|
188
158
|
host, port, weight = server.split ':', 3
|
189
159
|
port ||= DEFAULT_PORT
|
190
160
|
weight ||= DEFAULT_WEIGHT
|
191
161
|
Server.new self, host, port, weight
|
192
|
-
when Server
|
193
|
-
if server.memcache.multithread != @multithread then
|
194
|
-
raise ArgumentError, "can't mix threaded and non-threaded servers"
|
195
|
-
end
|
196
|
-
server
|
197
162
|
else
|
198
|
-
|
163
|
+
server
|
199
164
|
end
|
200
165
|
end
|
201
166
|
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
167
|
+
logger.debug { "Servers now: #{@servers.inspect}" } if logger
|
168
|
+
|
169
|
+
# There's no point in doing this if there's only one server
|
170
|
+
@continuum = create_continuum_for(@servers) if @servers.size > 1
|
171
|
+
|
172
|
+
@servers
|
207
173
|
end
|
208
174
|
|
209
175
|
##
|
@@ -212,15 +178,12 @@ class MemCache
|
|
212
178
|
# 0. +key+ can not be decremented below 0.
|
213
179
|
|
214
180
|
def decr(key, amount = 1)
|
215
|
-
|
216
|
-
|
217
|
-
if @multithread then
|
218
|
-
threadsafe_cache_decr server, cache_key, amount
|
219
|
-
else
|
181
|
+
raise MemCacheError, "Update of readonly cache" if @readonly
|
182
|
+
with_server(key) do |server, cache_key|
|
220
183
|
cache_decr server, cache_key, amount
|
221
184
|
end
|
222
|
-
rescue TypeError
|
223
|
-
handle_error
|
185
|
+
rescue TypeError => err
|
186
|
+
handle_error nil, err
|
224
187
|
end
|
225
188
|
|
226
189
|
##
|
@@ -228,21 +191,15 @@ class MemCache
|
|
228
191
|
# unmarshalled.
|
229
192
|
|
230
193
|
def get(key, raw = false)
|
231
|
-
server, cache_key
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
value = Marshal.load value unless raw
|
242
|
-
|
243
|
-
return value
|
244
|
-
rescue TypeError, SocketError, SystemCallError, IOError => err
|
245
|
-
handle_error server, err
|
194
|
+
with_server(key) do |server, cache_key|
|
195
|
+
value = cache_get server, cache_key
|
196
|
+
logger.debug { "GET #{key} from #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
|
197
|
+
return nil if value.nil?
|
198
|
+
value = Marshal.load value unless raw
|
199
|
+
return value
|
200
|
+
end
|
201
|
+
rescue TypeError => err
|
202
|
+
handle_error nil, err
|
246
203
|
end
|
247
204
|
|
248
205
|
##
|
@@ -260,6 +217,8 @@ class MemCache
|
|
260
217
|
# cache["a"] = 1
|
261
218
|
# cache["b"] = 2
|
262
219
|
# cache.get_multi "a", "b" # => { "a" => 1, "b" => 2 }
|
220
|
+
#
|
221
|
+
# Note that get_multi assumes the values are marshalled.
|
263
222
|
|
264
223
|
def get_multi(*keys)
|
265
224
|
raise MemCacheError, 'No active servers' unless active?
|
@@ -279,37 +238,35 @@ class MemCache
|
|
279
238
|
results = {}
|
280
239
|
|
281
240
|
server_keys.each do |server, keys_for_server|
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
241
|
+
keys_for_server_str = keys_for_server.join ' '
|
242
|
+
begin
|
243
|
+
values = cache_get_multi server, keys_for_server_str
|
244
|
+
values.each do |key, value|
|
245
|
+
results[cache_keys[key]] = Marshal.load value
|
246
|
+
end
|
247
|
+
rescue IndexError => e
|
248
|
+
# Ignore this server and try the others
|
249
|
+
logger.warn { "Unable to retrieve #{keys_for_server.size} elements from #{server.inspect}: #{e.message}"} if logger
|
290
250
|
end
|
291
251
|
end
|
292
252
|
|
293
253
|
return results
|
294
|
-
rescue TypeError
|
295
|
-
handle_error
|
254
|
+
rescue TypeError => err
|
255
|
+
handle_error nil, err
|
296
256
|
end
|
297
257
|
|
298
258
|
##
|
299
|
-
# Increments the value for +key+ by +amount+ and
|
259
|
+
# Increments the value for +key+ by +amount+ and returns the new value.
|
300
260
|
# +key+ must already exist. If +key+ is not an integer, it is assumed to be
|
301
261
|
# 0.
|
302
262
|
|
303
263
|
def incr(key, amount = 1)
|
304
|
-
|
305
|
-
|
306
|
-
if @multithread then
|
307
|
-
threadsafe_cache_incr server, cache_key, amount
|
308
|
-
else
|
264
|
+
raise MemCacheError, "Update of readonly cache" if @readonly
|
265
|
+
with_server(key) do |server, cache_key|
|
309
266
|
cache_incr server, cache_key, amount
|
310
267
|
end
|
311
|
-
rescue TypeError
|
312
|
-
handle_error
|
268
|
+
rescue TypeError => err
|
269
|
+
handle_error nil, err
|
313
270
|
end
|
314
271
|
|
315
272
|
##
|
@@ -319,25 +276,32 @@ class MemCache
|
|
319
276
|
# Warning: Readers should not call this method in the event of a cache miss;
|
320
277
|
# see MemCache#add.
|
321
278
|
|
279
|
+
ONE_MB = 1024 * 1024
|
280
|
+
|
322
281
|
def set(key, value, expiry = 0, raw = false)
|
323
282
|
raise MemCacheError, "Update of readonly cache" if @readonly
|
324
|
-
server, cache_key
|
325
|
-
socket = server.socket
|
283
|
+
with_server(key) do |server, cache_key|
|
326
284
|
|
327
|
-
|
328
|
-
|
285
|
+
value = Marshal.dump value unless raw
|
286
|
+
logger.debug { "SET #{key} to #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
|
329
287
|
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
288
|
+
data = value.to_s
|
289
|
+
raise MemCacheError, "Value too large, memcached can only store 1MB of data per key" if data.size > ONE_MB
|
290
|
+
|
291
|
+
command = "set #{cache_key} 0 #{expiry} #{data.size}\r\n#{data}\r\n"
|
292
|
+
|
293
|
+
with_socket_management(server) do |socket|
|
294
|
+
socket.write command
|
295
|
+
result = socket.gets
|
296
|
+
raise_on_error_response! result
|
297
|
+
|
298
|
+
if result.nil?
|
299
|
+
server.close
|
300
|
+
raise MemCacheError, "lost connection to #{server.host}:#{server.port}"
|
301
|
+
end
|
302
|
+
|
303
|
+
result
|
304
|
+
end
|
341
305
|
end
|
342
306
|
end
|
343
307
|
|
@@ -351,23 +315,17 @@ class MemCache
|
|
351
315
|
|
352
316
|
def add(key, value, expiry = 0, raw = false)
|
353
317
|
raise MemCacheError, "Update of readonly cache" if @readonly
|
354
|
-
server, cache_key
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
result
|
366
|
-
rescue SocketError, SystemCallError, IOError => err
|
367
|
-
server.close
|
368
|
-
raise MemCacheError, err.message
|
369
|
-
ensure
|
370
|
-
@mutex.unlock if @multithread
|
318
|
+
with_server(key) do |server, cache_key|
|
319
|
+
value = Marshal.dump value unless raw
|
320
|
+
logger.debug { "ADD #{key} to #{server}: #{value ? value.to_s.size : 'nil'}" } if logger
|
321
|
+
command = "add #{cache_key} 0 #{expiry} #{value.to_s.size}\r\n#{value}\r\n"
|
322
|
+
|
323
|
+
with_socket_management(server) do |socket|
|
324
|
+
socket.write command
|
325
|
+
result = socket.gets
|
326
|
+
raise_on_error_response! result
|
327
|
+
result
|
328
|
+
end
|
371
329
|
end
|
372
330
|
end
|
373
331
|
|
@@ -375,26 +333,15 @@ class MemCache
|
|
375
333
|
# Removes +key+ from the cache in +expiry+ seconds.
|
376
334
|
|
377
335
|
def delete(key, expiry = 0)
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
begin
|
388
|
-
sock.write "delete #{cache_key} #{expiry}\r\n"
|
389
|
-
result = sock.gets
|
390
|
-
raise_on_error_response! result
|
391
|
-
result
|
392
|
-
rescue SocketError, SystemCallError, IOError => err
|
393
|
-
server.close
|
394
|
-
raise MemCacheError, err.message
|
336
|
+
raise MemCacheError, "Update of readonly cache" if @readonly
|
337
|
+
with_server(key) do |server, cache_key|
|
338
|
+
with_socket_management(server) do |socket|
|
339
|
+
socket.write "delete #{cache_key} #{expiry}\r\n"
|
340
|
+
result = socket.gets
|
341
|
+
raise_on_error_response! result
|
342
|
+
result
|
343
|
+
end
|
395
344
|
end
|
396
|
-
ensure
|
397
|
-
@mutex.unlock if @multithread
|
398
345
|
end
|
399
346
|
|
400
347
|
##
|
@@ -403,23 +350,18 @@ class MemCache
|
|
403
350
|
def flush_all
|
404
351
|
raise MemCacheError, 'No active servers' unless active?
|
405
352
|
raise MemCacheError, "Update of readonly cache" if @readonly
|
353
|
+
|
406
354
|
begin
|
407
|
-
@mutex.lock if @multithread
|
408
355
|
@servers.each do |server|
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
sock.write "flush_all\r\n"
|
413
|
-
result = sock.gets
|
356
|
+
with_socket_management(server) do |socket|
|
357
|
+
socket.write "flush_all\r\n"
|
358
|
+
result = socket.gets
|
414
359
|
raise_on_error_response! result
|
415
360
|
result
|
416
|
-
rescue SocketError, SystemCallError, IOError => err
|
417
|
-
server.close
|
418
|
-
raise MemCacheError, err.message
|
419
361
|
end
|
420
362
|
end
|
421
|
-
|
422
|
-
|
363
|
+
rescue IndexError => err
|
364
|
+
handle_error nil, err
|
423
365
|
end
|
424
366
|
end
|
425
367
|
|
@@ -469,17 +411,16 @@ class MemCache
|
|
469
411
|
server_stats = {}
|
470
412
|
|
471
413
|
@servers.each do |server|
|
472
|
-
|
473
|
-
raise MemCacheError, "No connection to server" if sock.nil?
|
414
|
+
next unless server.alive?
|
474
415
|
|
475
|
-
|
476
|
-
|
477
|
-
|
416
|
+
with_socket_management(server) do |socket|
|
417
|
+
value = nil
|
418
|
+
socket.write "stats\r\n"
|
478
419
|
stats = {}
|
479
|
-
while line =
|
420
|
+
while line = socket.gets do
|
480
421
|
raise_on_error_response! line
|
481
422
|
break if line == "END\r\n"
|
482
|
-
if line =~ /\ASTAT ([\
|
423
|
+
if line =~ /\ASTAT ([\S]+) ([\w\.\:]+)/ then
|
483
424
|
name, value = $1, $2
|
484
425
|
stats[name] = case name
|
485
426
|
when 'version'
|
@@ -498,12 +439,10 @@ class MemCache
|
|
498
439
|
end
|
499
440
|
end
|
500
441
|
server_stats["#{server.host}:#{server.port}"] = stats
|
501
|
-
rescue SocketError, SystemCallError, IOError => err
|
502
|
-
server.close
|
503
|
-
raise MemCacheError, err.message
|
504
442
|
end
|
505
443
|
end
|
506
444
|
|
445
|
+
raise MemCacheError, "No active servers" if server_stats.empty?
|
507
446
|
server_stats
|
508
447
|
end
|
509
448
|
|
@@ -520,7 +459,7 @@ class MemCache
|
|
520
459
|
set key, value
|
521
460
|
end
|
522
461
|
|
523
|
-
protected
|
462
|
+
protected unless $TESTING
|
524
463
|
|
525
464
|
##
|
526
465
|
# Create a key for the cache, incorporating the namespace qualifier if
|
@@ -534,46 +473,49 @@ class MemCache
|
|
534
473
|
end
|
535
474
|
end
|
536
475
|
|
476
|
+
##
|
477
|
+
# Returns an interoperable hash value for +key+. (I think, docs are
|
478
|
+
# sketchy for down servers).
|
479
|
+
|
480
|
+
def hash_for(key)
|
481
|
+
Zlib.crc32(key)
|
482
|
+
end
|
483
|
+
|
537
484
|
##
|
538
485
|
# Pick a server to handle the request based on a hash of the key.
|
539
486
|
|
540
|
-
def get_server_for_key(key)
|
487
|
+
def get_server_for_key(key, options = {})
|
541
488
|
raise ArgumentError, "illegal character in key #{key.inspect}" if
|
542
489
|
key =~ /\s/
|
543
490
|
raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
|
544
491
|
raise MemCacheError, "No servers available" if @servers.empty?
|
545
492
|
return @servers.first if @servers.length == 1
|
546
493
|
|
547
|
-
hkey = hash_for
|
494
|
+
hkey = hash_for(key)
|
548
495
|
|
549
496
|
20.times do |try|
|
550
|
-
|
497
|
+
entryidx = Continuum.binary_search(@continuum, hkey)
|
498
|
+
server = @continuum[entryidx].server
|
551
499
|
return server if server.alive?
|
552
|
-
|
500
|
+
break unless failover
|
501
|
+
hkey = hash_for "#{try}#{key}"
|
553
502
|
end
|
554
503
|
|
555
504
|
raise MemCacheError, "No servers available"
|
556
505
|
end
|
557
506
|
|
558
|
-
##
|
559
|
-
# Returns an interoperable hash value for +key+. (I think, docs are
|
560
|
-
# sketchy for down servers).
|
561
|
-
|
562
|
-
def hash_for(key)
|
563
|
-
(key.crc32_ITU_T >> 16) & 0x7fff
|
564
|
-
end
|
565
|
-
|
566
507
|
##
|
567
508
|
# Performs a raw decr for +cache_key+ from +server+. Returns nil if not
|
568
509
|
# found.
|
569
510
|
|
570
511
|
def cache_decr(server, cache_key, amount)
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
575
|
-
|
576
|
-
|
512
|
+
with_socket_management(server) do |socket|
|
513
|
+
socket.write "decr #{cache_key} #{amount}\r\n"
|
514
|
+
text = socket.gets
|
515
|
+
raise_on_error_response! text
|
516
|
+
return nil if text == "NOT_FOUND\r\n"
|
517
|
+
return text.to_i
|
518
|
+
end
|
577
519
|
end
|
578
520
|
|
579
521
|
##
|
@@ -581,52 +523,54 @@ class MemCache
|
|
581
523
|
# miss.
|
582
524
|
|
583
525
|
def cache_get(server, cache_key)
|
584
|
-
|
585
|
-
|
586
|
-
|
526
|
+
with_socket_management(server) do |socket|
|
527
|
+
socket.write "get #{cache_key}\r\n"
|
528
|
+
keyline = socket.gets # "VALUE <key> <flags> <bytes>\r\n"
|
587
529
|
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
530
|
+
if keyline.nil? then
|
531
|
+
server.close
|
532
|
+
raise MemCacheError, "lost connection to #{server.host}:#{server.port}"
|
533
|
+
end
|
592
534
|
|
593
|
-
|
594
|
-
|
535
|
+
raise_on_error_response! keyline
|
536
|
+
return nil if keyline == "END\r\n"
|
595
537
|
|
596
|
-
|
597
|
-
|
598
|
-
|
538
|
+
unless keyline =~ /(\d+)\r/ then
|
539
|
+
server.close
|
540
|
+
raise MemCacheError, "unexpected response #{keyline.inspect}"
|
541
|
+
end
|
542
|
+
value = socket.read $1.to_i
|
543
|
+
socket.read 2 # "\r\n"
|
544
|
+
socket.gets # "END\r\n"
|
545
|
+
return value
|
599
546
|
end
|
600
|
-
value = socket.read $1.to_i
|
601
|
-
socket.read 2 # "\r\n"
|
602
|
-
socket.gets # "END\r\n"
|
603
|
-
return value
|
604
547
|
end
|
605
548
|
|
606
549
|
##
|
607
550
|
# Fetches +cache_keys+ from +server+ using a multi-get.
|
608
551
|
|
609
552
|
def cache_get_multi(server, cache_keys)
|
610
|
-
|
611
|
-
|
612
|
-
|
553
|
+
with_socket_management(server) do |socket|
|
554
|
+
values = {}
|
555
|
+
socket.write "get #{cache_keys}\r\n"
|
613
556
|
|
614
|
-
|
615
|
-
|
616
|
-
|
557
|
+
while keyline = socket.gets do
|
558
|
+
return values if keyline == "END\r\n"
|
559
|
+
raise_on_error_response! keyline
|
617
560
|
|
618
|
-
|
619
|
-
|
620
|
-
|
561
|
+
unless keyline =~ /\AVALUE (.+) (.+) (.+)/ then
|
562
|
+
server.close
|
563
|
+
raise MemCacheError, "unexpected response #{keyline.inspect}"
|
564
|
+
end
|
565
|
+
|
566
|
+
key, data_length = $1, $3
|
567
|
+
values[$1] = socket.read data_length.to_i
|
568
|
+
socket.read(2) # "\r\n"
|
621
569
|
end
|
622
570
|
|
623
|
-
|
624
|
-
|
625
|
-
socket.read(2) # "\r\n"
|
571
|
+
server.close
|
572
|
+
raise MemCacheError, "lost connection to #{server.host}:#{server.port}" # TODO: retry here too
|
626
573
|
end
|
627
|
-
|
628
|
-
server.close
|
629
|
-
raise MemCacheError, "lost connection to #{server.host}:#{server.port}"
|
630
574
|
end
|
631
575
|
|
632
576
|
##
|
@@ -634,18 +578,81 @@ class MemCache
|
|
634
578
|
# found.
|
635
579
|
|
636
580
|
def cache_incr(server, cache_key, amount)
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
581
|
+
with_socket_management(server) do |socket|
|
582
|
+
socket.write "incr #{cache_key} #{amount}\r\n"
|
583
|
+
text = socket.gets
|
584
|
+
raise_on_error_response! text
|
585
|
+
return nil if text == "NOT_FOUND\r\n"
|
586
|
+
return text.to_i
|
587
|
+
end
|
588
|
+
end
|
589
|
+
|
590
|
+
##
|
591
|
+
# Gets or creates a socket connected to the given server, and yields it
|
592
|
+
# to the block, wrapped in a mutex synchronization if @multithread is true.
|
593
|
+
#
|
594
|
+
# If a socket error (SocketError, SystemCallError, IOError) or protocol error
|
595
|
+
# (MemCacheError) is raised by the block, closes the socket, attempts to
|
596
|
+
# connect again, and retries the block (once). If an error is again raised,
|
597
|
+
# reraises it as MemCacheError.
|
598
|
+
#
|
599
|
+
# If unable to connect to the server (or if in the reconnect wait period),
|
600
|
+
# raises MemCacheError. Note that the socket connect code marks a server
|
601
|
+
# dead for a timeout period, so retrying does not apply to connection attempt
|
602
|
+
# failures (but does still apply to unexpectedly lost connections etc.).
|
603
|
+
|
604
|
+
def with_socket_management(server, &block)
|
605
|
+
check_multithread_status!
|
606
|
+
|
607
|
+
@mutex.lock if @multithread
|
608
|
+
retried = false
|
609
|
+
|
610
|
+
begin
|
611
|
+
socket = server.socket
|
612
|
+
|
613
|
+
# Raise an IndexError to show this server is out of whack. If were inside
|
614
|
+
# a with_server block, we'll catch it and attempt to restart the operation.
|
615
|
+
|
616
|
+
raise IndexError, "No connection to server (#{server.status})" if socket.nil?
|
617
|
+
|
618
|
+
block.call(socket)
|
619
|
+
|
620
|
+
rescue SocketError => err
|
621
|
+
logger.warn { "Socket failure: #{err.message}" } if logger
|
622
|
+
server.mark_dead(err)
|
623
|
+
handle_error(server, err)
|
624
|
+
|
625
|
+
rescue MemCacheError, SystemCallError, IOError => err
|
626
|
+
logger.warn { "Generic failure: #{err.class.name}: #{err.message}" } if logger
|
627
|
+
handle_error(server, err) if retried || socket.nil?
|
628
|
+
retried = true
|
629
|
+
retry
|
630
|
+
end
|
631
|
+
ensure
|
632
|
+
@mutex.unlock if @multithread
|
633
|
+
end
|
634
|
+
|
635
|
+
def with_server(key)
|
636
|
+
retried = false
|
637
|
+
begin
|
638
|
+
server, cache_key = request_setup(key)
|
639
|
+
yield server, cache_key
|
640
|
+
rescue IndexError => e
|
641
|
+
logger.warn { "Server failed: #{e.class.name}: #{e.message}" } if logger
|
642
|
+
if !retried && @servers.size > 1
|
643
|
+
logger.info { "Connection to server #{server.inspect} DIED! Retrying operation..." } if logger
|
644
|
+
retried = true
|
645
|
+
retry
|
646
|
+
end
|
647
|
+
handle_error(nil, e)
|
648
|
+
end
|
643
649
|
end
|
644
650
|
|
645
651
|
##
|
646
652
|
# Handles +error+ from +server+.
|
647
653
|
|
648
654
|
def handle_error(server, error)
|
655
|
+
raise error if error.is_a?(MemCacheError)
|
649
656
|
server.close if server
|
650
657
|
new_error = MemCacheError.new error.message
|
651
658
|
new_error.set_backtrace error.backtrace
|
@@ -660,45 +667,46 @@ class MemCache
|
|
660
667
|
raise MemCacheError, 'No active servers' unless active?
|
661
668
|
cache_key = make_cache_key key
|
662
669
|
server = get_server_for_key cache_key
|
663
|
-
raise MemCacheError, 'No connection to server' if server.socket.nil?
|
664
670
|
return server, cache_key
|
665
671
|
end
|
666
672
|
|
667
|
-
def
|
668
|
-
|
669
|
-
|
670
|
-
|
671
|
-
@mutex.unlock
|
673
|
+
def raise_on_error_response!(response)
|
674
|
+
if response =~ /\A(?:CLIENT_|SERVER_)?ERROR(.*)/
|
675
|
+
raise MemCacheError, $1.strip
|
676
|
+
end
|
672
677
|
end
|
673
678
|
|
674
|
-
def
|
675
|
-
|
676
|
-
|
677
|
-
ensure
|
678
|
-
@mutex.unlock
|
679
|
-
end
|
679
|
+
def create_continuum_for(servers)
|
680
|
+
total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
|
681
|
+
continuum = []
|
680
682
|
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
683
|
+
servers.each do |server|
|
684
|
+
entry_count_for(server, servers.size, total_weight).times do |idx|
|
685
|
+
hash = Digest::SHA1.hexdigest("#{server.host}:#{server.port}:#{idx}")
|
686
|
+
value = Integer("0x#{hash[0..7]}")
|
687
|
+
continuum << Continuum::Entry.new(value, server)
|
688
|
+
end
|
689
|
+
end
|
690
|
+
|
691
|
+
continuum.sort { |a, b| a.value <=> b.value }
|
686
692
|
end
|
687
693
|
|
688
|
-
def
|
689
|
-
|
690
|
-
cache_incr server, cache_key, amount
|
691
|
-
ensure
|
692
|
-
@mutex.unlock
|
694
|
+
def entry_count_for(server, total_servers, total_weight)
|
695
|
+
((total_servers * Continuum::POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
|
693
696
|
end
|
694
697
|
|
695
|
-
def
|
696
|
-
if
|
697
|
-
|
698
|
+
def check_multithread_status!
|
699
|
+
return if @multithread
|
700
|
+
|
701
|
+
if Thread.current[:memcache_client] != self.object_id
|
702
|
+
raise MemCacheError, <<-EOM
|
703
|
+
You are accessing this memcache-client instance from multiple threads but have not enabled multithread support.
|
704
|
+
Normally: MemCache.new(['localhost:11211'], :multithread => true)
|
705
|
+
In Rails: config.cache_store = [:mem_cache_store, 'localhost:11211', { :multithread => true }]
|
706
|
+
EOM
|
698
707
|
end
|
699
708
|
end
|
700
709
|
|
701
|
-
|
702
710
|
##
|
703
711
|
# This class represents a memcached server instance.
|
704
712
|
|
@@ -742,6 +750,8 @@ class MemCache
|
|
742
750
|
|
743
751
|
attr_reader :status
|
744
752
|
|
753
|
+
attr_reader :logger
|
754
|
+
|
745
755
|
##
|
746
756
|
# Create a new MemCache::Server object for the memcached instance
|
747
757
|
# listening on the given host and port, weighted by the given weight.
|
@@ -750,17 +760,15 @@ class MemCache
|
|
750
760
|
raise ArgumentError, "No host specified" if host.nil? or host.empty?
|
751
761
|
raise ArgumentError, "No port specified" if port.nil? or port.to_i.zero?
|
752
762
|
|
753
|
-
@memcache = memcache
|
754
763
|
@host = host
|
755
764
|
@port = port.to_i
|
756
765
|
@weight = weight.to_i
|
757
766
|
|
758
|
-
@multithread = @memcache.multithread
|
759
|
-
@mutex = Mutex.new
|
760
|
-
|
761
767
|
@sock = nil
|
762
768
|
@retry = nil
|
763
769
|
@status = 'NOT CONNECTED'
|
770
|
+
@timeout = memcache.timeout
|
771
|
+
@logger = memcache.logger
|
764
772
|
end
|
765
773
|
|
766
774
|
##
|
@@ -785,7 +793,6 @@ class MemCache
|
|
785
793
|
# Returns the connected socket object on success or nil on failure.
|
786
794
|
|
787
795
|
def socket
|
788
|
-
@mutex.lock if @multithread
|
789
796
|
return @sock if @sock and not @sock.closed?
|
790
797
|
|
791
798
|
@sock = nil
|
@@ -795,21 +802,19 @@ class MemCache
|
|
795
802
|
|
796
803
|
# Attempt to connect if not already connected.
|
797
804
|
begin
|
798
|
-
@sock = timeout
|
799
|
-
|
800
|
-
end
|
805
|
+
@sock = @timeout ? TCPTimeoutSocket.new(@host, @port, @timeout) : TCPSocket.new(@host, @port)
|
806
|
+
|
801
807
|
if Socket.constants.include? 'TCP_NODELAY' then
|
802
808
|
@sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
|
803
809
|
end
|
804
810
|
@retry = nil
|
805
811
|
@status = 'CONNECTED'
|
806
812
|
rescue SocketError, SystemCallError, IOError, Timeout::Error => err
|
807
|
-
|
813
|
+
logger.warn { "Unable to open socket: #{err.class.name}, #{err.message}" } if logger
|
814
|
+
mark_dead err
|
808
815
|
end
|
809
816
|
|
810
817
|
return @sock
|
811
|
-
ensure
|
812
|
-
@mutex.unlock if @multithread
|
813
818
|
end
|
814
819
|
|
815
820
|
##
|
@@ -817,27 +822,25 @@ class MemCache
|
|
817
822
|
# object. The server is not considered dead.
|
818
823
|
|
819
824
|
def close
|
820
|
-
@mutex.lock if @multithread
|
821
825
|
@sock.close if @sock && !@sock.closed?
|
822
826
|
@sock = nil
|
823
827
|
@retry = nil
|
824
828
|
@status = "NOT CONNECTED"
|
825
|
-
ensure
|
826
|
-
@mutex.unlock if @multithread
|
827
829
|
end
|
828
830
|
|
829
|
-
private
|
830
|
-
|
831
831
|
##
|
832
832
|
# Mark the server as dead and close its socket.
|
833
833
|
|
834
|
-
def mark_dead(
|
834
|
+
def mark_dead(error)
|
835
835
|
@sock.close if @sock && !@sock.closed?
|
836
836
|
@sock = nil
|
837
837
|
@retry = Time.now + RETRY_DELAY
|
838
838
|
|
839
|
-
|
839
|
+
reason = "#{error.class.name}: #{error.message}"
|
840
|
+
@status = sprintf "%s:%s DEAD (%s), will retry at %s", @host, @port, reason, @retry
|
841
|
+
@logger.info { @status } if @logger
|
840
842
|
end
|
843
|
+
|
841
844
|
end
|
842
845
|
|
843
846
|
##
|
@@ -847,3 +850,86 @@ class MemCache
|
|
847
850
|
|
848
851
|
end
|
849
852
|
|
853
|
+
# TCPSocket facade class which implements timeouts.
|
854
|
+
class TCPTimeoutSocket
|
855
|
+
|
856
|
+
def initialize(host, port, timeout)
|
857
|
+
Timeout::timeout(MemCache::Server::CONNECT_TIMEOUT, SocketError) do
|
858
|
+
@sock = TCPSocket.new(host, port)
|
859
|
+
@len = timeout
|
860
|
+
end
|
861
|
+
end
|
862
|
+
|
863
|
+
def write(*args)
|
864
|
+
Timeout::timeout(@len, SocketError) do
|
865
|
+
@sock.write(*args)
|
866
|
+
end
|
867
|
+
end
|
868
|
+
|
869
|
+
def gets(*args)
|
870
|
+
Timeout::timeout(@len, SocketError) do
|
871
|
+
@sock.gets(*args)
|
872
|
+
end
|
873
|
+
end
|
874
|
+
|
875
|
+
def read(*args)
|
876
|
+
Timeout::timeout(@len, SocketError) do
|
877
|
+
@sock.read(*args)
|
878
|
+
end
|
879
|
+
end
|
880
|
+
|
881
|
+
def _socket
|
882
|
+
@sock
|
883
|
+
end
|
884
|
+
|
885
|
+
def method_missing(meth, *args)
|
886
|
+
@sock.__send__(meth, *args)
|
887
|
+
end
|
888
|
+
|
889
|
+
def closed?
|
890
|
+
@sock.closed?
|
891
|
+
end
|
892
|
+
|
893
|
+
def close
|
894
|
+
@sock.close
|
895
|
+
end
|
896
|
+
end
|
897
|
+
|
898
|
+
module Continuum
|
899
|
+
POINTS_PER_SERVER = 160 # this is the default in libmemcached
|
900
|
+
|
901
|
+
# Find the closest index in Continuum with value <= the given value
|
902
|
+
def self.binary_search(ary, value, &block)
|
903
|
+
upper = ary.size - 1
|
904
|
+
lower = 0
|
905
|
+
idx = 0
|
906
|
+
|
907
|
+
while(lower <= upper) do
|
908
|
+
idx = (lower + upper) / 2
|
909
|
+
comp = ary[idx].value <=> value
|
910
|
+
|
911
|
+
if comp == 0
|
912
|
+
return idx
|
913
|
+
elsif comp > 0
|
914
|
+
upper = idx - 1
|
915
|
+
else
|
916
|
+
lower = idx + 1
|
917
|
+
end
|
918
|
+
end
|
919
|
+
return upper
|
920
|
+
end
|
921
|
+
|
922
|
+
class Entry
|
923
|
+
attr_reader :value
|
924
|
+
attr_reader :server
|
925
|
+
|
926
|
+
def initialize(val, srv)
|
927
|
+
@value = val
|
928
|
+
@server = srv
|
929
|
+
end
|
930
|
+
|
931
|
+
def inspect
|
932
|
+
"<#{value}, #{server.host}:#{server.port}>"
|
933
|
+
end
|
934
|
+
end
|
935
|
+
end
|