KellyMahan-memcachedb-client 1.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,3 @@
1
+ = 1.0
2
+
3
+ derived from memcache-client 1.6.3 http://github.com/mperham for use with memcachedb
@@ -0,0 +1,28 @@
1
+ Copyright 2005-2009 Bob Cottrell, Eric Hodel, Mike Perham, Kelly Mahan.
2
+ All rights reserved.
3
+
4
+ Redistribution and use in source and binary forms, with or without
5
+ modification, are permitted provided that the following conditions
6
+ are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright
9
+ notice, this list of conditions and the following disclaimer.
10
+ 2. Redistributions in binary form must reproduce the above copyright
11
+ notice, this list of conditions and the following disclaimer in the
12
+ documentation and/or other materials provided with the distribution.
13
+ 3. Neither the names of the authors nor the names of their contributors
14
+ may be used to endorse or promote products derived from this software
15
+ without specific prior written permission.
16
+
17
+ THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
18
+ OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
21
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
22
+ OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
23
+ OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
24
+ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25
+ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
26
+ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
+
@@ -0,0 +1,28 @@
1
+ = memcachedb-client
2
+
3
+ A pure ruby library for accessing memcachedb. Forked from http://github.com/mperham/memcache-client
4
+
5
+ Source:
6
+
7
+ http://github.com/KellyMahan/memcachedb-client
8
+
9
+ == Installing memcachedb-client
10
+
11
+ Just install the gem:
12
+
13
+ $ sudo gem install memcachedb-client
14
+
15
+ == Using memcachedb-client
16
+
17
+ With one server:
18
+
19
+ CACHE = MemCacheDb.new 'localhost:21201', :namespace => 'my_namespace'
20
+
21
+ Or with multiple servers:
22
+
23
+ CACHE = MemCacheDb.new %w[one.example.com:21201 two.example.com:21201],
24
+ :namespace => 'my_namespace'
25
+
26
+ See MemCacheDb.new for details. Please note memcachedb-client is not thread-safe
27
+ by default. You should create a separate instance for each thread in your
28
+ process.
@@ -0,0 +1,26 @@
1
+ # vim: syntax=Ruby
2
+ require 'rubygems'
3
+ require 'rake/rdoctask'
4
+ require 'rake/testtask'
5
+
6
+ task :gem do
7
+ sh "gem build memcachedb-client.gemspec"
8
+ end
9
+
10
+ task :install => [:gem] do
11
+ sh "sudo gem install memcachedb-client-*.gem"
12
+ end
13
+
14
+ Rake::RDocTask.new do |rd|
15
+ rd.main = "README.rdoc"
16
+ rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
17
+ rd.rdoc_dir = 'doc'
18
+ end
19
+
20
+ Rake::TestTask.new
21
+
22
+ task :default => :test
23
+
24
+ task :rcov do
25
+ `rcov -Ilib test/*.rb`
26
+ end
@@ -0,0 +1,77 @@
1
+ module ContinuumDb
2
+ POINTS_PER_SERVER = 160 # this is the default in libmemcached
3
+
4
+ class << self
5
+
6
+ begin
7
+ require 'inline'
8
+ inline do |builder|
9
+ builder.c <<-EOM
10
+ int binary_search(VALUE ary, unsigned int r) {
11
+ int upper = RARRAY_LEN(ary) - 1;
12
+ int lower = 0;
13
+ int idx = 0;
14
+ ID value = rb_intern("value");
15
+
16
+ while (lower <= upper) {
17
+ idx = (lower + upper) / 2;
18
+
19
+ VALUE continuumValue = rb_funcall(RARRAY_PTR(ary)[idx], value, 0);
20
+ unsigned int l = NUM2UINT(continuumValue);
21
+ if (l == r) {
22
+ return idx;
23
+ }
24
+ else if (l > r) {
25
+ upper = idx - 1;
26
+ }
27
+ else {
28
+ lower = idx + 1;
29
+ }
30
+ }
31
+ return upper;
32
+ }
33
+ EOM
34
+ end
35
+ rescue Exception => e
36
+ puts "Unable to generate native code, falling back to Ruby: #{e.message}"
37
+
38
+ # slow but pure ruby version
39
+ # Find the closest index in Continuum with value <= the given value
40
+ def binary_search(ary, value, &block)
41
+ upper = ary.size - 1
42
+ lower = 0
43
+ idx = 0
44
+
45
+ while(lower <= upper) do
46
+ idx = (lower + upper) / 2
47
+ comp = ary[idx].value <=> value
48
+
49
+ if comp == 0
50
+ return idx
51
+ elsif comp > 0
52
+ upper = idx - 1
53
+ else
54
+ lower = idx + 1
55
+ end
56
+ end
57
+ return upper
58
+ end
59
+
60
+ end
61
+ end
62
+
63
+
64
+ class Entry
65
+ attr_reader :value
66
+ attr_reader :server
67
+
68
+ def initialize(val, srv)
69
+ @value = val
70
+ @server = srv
71
+ end
72
+
73
+ def inspect
74
+ "<#{value}, #{server.host}:#{server.port}>"
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,896 @@
1
+ $TESTING = defined?($TESTING) && $TESTING
2
+
3
+ require 'socket'
4
+ require 'thread'
5
+ require 'timeout'
6
+ require 'zlib'
7
+ require 'digest/sha1'
8
+
9
+ require 'continuum_db'
10
+
11
+ ##
12
+ # A Ruby client library for memcachedb.
13
+ #
14
+
15
+ class MemCacheDb
16
+
17
+ ##
18
+ # The version of MemCacheDb you are using.
19
+
20
+ VERSION = '1.0.1'
21
+
22
+ ##
23
+ # Default options for the cache object.
24
+
25
+ DEFAULT_OPTIONS = {
26
+ :namespace => nil,
27
+ :readonly => false,
28
+ :multithread => false,
29
+ :failover => true,
30
+ :timeout => 0.5,
31
+ :logger => nil,
32
+ }
33
+
34
+ ##
35
+ # Default memcachedb port.
36
+
37
+ DEFAULT_PORT = 21201
38
+
39
+ ##
40
+ # Default memcachedb server weight.
41
+
42
+ DEFAULT_WEIGHT = 1
43
+
44
+ ##
45
+ # The namespace for this instance
46
+
47
+ attr_reader :namespace
48
+
49
+ ##
50
+ # The multithread setting for this instance
51
+
52
+ attr_reader :multithread
53
+
54
+ ##
55
+ # The servers this client talks to. Play at your own peril.
56
+
57
+ attr_reader :servers
58
+
59
+ ##
60
+ # Socket timeout limit with this client, defaults to 0.25 sec.
61
+ # Set to nil to disable timeouts.
62
+
63
+ attr_reader :timeout
64
+
65
+ ##
66
+ # Should the client try to failover to another server if the
67
+ # first server is down? Defaults to true.
68
+
69
+ attr_reader :failover
70
+
71
+ ##
72
+ # Log debug/info/warn/error to the given Logger, defaults to nil.
73
+
74
+ attr_reader :logger
75
+
76
+ ##
77
+ # Accepts a list of +servers+ and a list of +opts+. +servers+ may be
78
+ # omitted. See +servers=+ for acceptable server list arguments.
79
+ #
80
+ # Valid options for +opts+ are:
81
+ #
82
+ # [:namespace] Prepends this value to all keys added or retrieved.
83
+ # [:readonly] Raises an exception on cache writes when true.
84
+ # [:multithread] Wraps cache access in a Mutex for thread safety.
85
+ # [:failover] Should the client try to failover to another server if the
86
+ # first server is down? Defaults to true.
87
+ # [:timeout] Time to use as the socket read timeout. Defaults to 0.25 sec,
88
+ # set to nil to disable timeouts (this is a major performance penalty in Ruby 1.8).
89
+ # [:logger] Logger to use for info/debug output, defaults to nil
90
+ # Other options are ignored.
91
+
92
+ def initialize(*args)
93
+ servers = []
94
+ opts = {}
95
+
96
+ case args.length
97
+ when 0 then # NOP
98
+ when 1 then
99
+ arg = args.shift
100
+ case arg
101
+ when Hash then opts = arg
102
+ when Array then servers = arg
103
+ when String then servers = [arg]
104
+ else raise ArgumentError, 'first argument must be Array, Hash or String'
105
+ end
106
+ when 2 then
107
+ servers, opts = args
108
+ else
109
+ raise ArgumentError, "wrong number of arguments (#{args.length} for 2)"
110
+ end
111
+
112
+ opts = DEFAULT_OPTIONS.merge opts
113
+ @namespace = opts[:namespace]
114
+ @readonly = opts[:readonly]
115
+ @multithread = opts[:multithread]
116
+ @timeout = opts[:timeout]
117
+ @failover = opts[:failover]
118
+ @logger = opts[:logger]
119
+ @mutex = Mutex.new if @multithread
120
+
121
+ logger.info { "memcachedb-client #{VERSION} #{Array(servers).inspect}" } if logger
122
+
123
+ self.servers = servers
124
+ end
125
+
126
+ ##
127
+ # Returns a string representation of the cache object.
128
+
129
+ def inspect
130
+ "<MemCacheDb: %d servers, ns: %p, ro: %p>" %
131
+ [@servers.length, @namespace, @readonly]
132
+ end
133
+
134
+ ##
135
+ # Returns whether there is at least one active server for the object.
136
+
137
+ def active?
138
+ not @servers.empty?
139
+ end
140
+
141
+ ##
142
+ # Returns whether or not the cache object was created read only.
143
+
144
+ def readonly?
145
+ @readonly
146
+ end
147
+
148
+ ##
149
+ # Set the servers that the requests will be distributed between. Entries
150
+ # can be either strings of the form "hostname:port" or
151
+ # "hostname:port:weight" or MemCacheDb::Server objects.
152
+ #
153
+ def servers=(servers)
154
+ # Create the server objects.
155
+ @servers = Array(servers).collect do |server|
156
+ case server
157
+ when String
158
+ host, port, weight = server.split ':', 3
159
+ port ||= DEFAULT_PORT
160
+ weight ||= DEFAULT_WEIGHT
161
+ Server.new self, host, port, weight
162
+ else
163
+ if server.multithread != @multithread then
164
+ raise ArgumentError, "can't mix threaded and non-threaded servers"
165
+ end
166
+ server
167
+ end
168
+ end
169
+
170
+ logger.debug { "Servers now: #{@servers.inspect}" } if logger
171
+
172
+ # There's no point in doing this if there's only one server
173
+ @continuum = create_continuum_for(@servers) if @servers.size > 1
174
+
175
+ @servers
176
+ end
177
+
178
+ ##
179
+ # Decrements the value for +key+ by +amount+ and returns the new value.
180
+ # +key+ must already exist. If +key+ is not an integer, it is assumed to be
181
+ # 0. +key+ can not be decremented below 0.
182
+
183
+ def decr(key, amount = 1)
184
+ raise MemCacheDbError, "Update of readonly cache" if @readonly
185
+ with_server(key) do |server, cache_key|
186
+ cache_decr server, cache_key, amount
187
+ end
188
+ rescue TypeError => err
189
+ handle_error nil, err
190
+ end
191
+
192
+ ##
193
+ # Retrieves +key+ from MemCacheDb. If +raw+ is false, the value will be
194
+ # unmarshalled.
195
+
196
+ def get(key, raw = false)
197
+ with_server(key) do |server, cache_key|
198
+ value = cache_get server, cache_key
199
+ logger.debug { "GET #{key} from #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
200
+ return nil if value.nil?
201
+ value = Marshal.load value unless raw
202
+ return value
203
+ end
204
+ rescue TypeError => err
205
+ handle_error nil, err
206
+ end
207
+
208
+ ##
209
+ # Retrieves multiple values from memcachedb in parallel, if possible.
210
+ #
211
+ # The memcachedb protocol supports the ability to retrieve multiple
212
+ # keys in a single request. Pass in an array of keys to this method
213
+ # and it will:
214
+ #
215
+ # 1. map the key to the appropriate memcachedb server
216
+ # 2. send a single request to each server that has one or more key values
217
+ #
218
+ # Returns a hash of values.
219
+ #
220
+ # cache["a"] = 1
221
+ # cache["b"] = 2
222
+ # cache.get_multi "a", "b" # => { "a" => 1, "b" => 2 }
223
+
224
+ def get_multi(*keys)
225
+ raise MemCacheDbError, 'No active servers' unless active?
226
+
227
+ keys.flatten!
228
+ key_count = keys.length
229
+ cache_keys = {}
230
+ server_keys = Hash.new { |h,k| h[k] = [] }
231
+
232
+ # map keys to servers
233
+ keys.each do |key|
234
+ server, cache_key = request_setup key
235
+ cache_keys[cache_key] = key
236
+ server_keys[server] << cache_key
237
+ end
238
+
239
+ results = {}
240
+
241
+ server_keys.each do |server, keys_for_server|
242
+ keys_for_server_str = keys_for_server.join ' '
243
+ begin
244
+ values = cache_get_multi server, keys_for_server_str
245
+ values.each do |key, value|
246
+ results[cache_keys[key]] = Marshal.load value
247
+ end
248
+ rescue IndexError => e
249
+ # Ignore this server and try the others
250
+ logger.warn { "Unable to retrieve #{keys_for_server.size} elements from #{server.inspect}: #{e.message}"} if logger
251
+ end
252
+ end
253
+
254
+ return results
255
+ rescue TypeError => err
256
+ handle_error nil, err
257
+ end
258
+
259
+ ##
260
+ # Increments the value for +key+ by +amount+ and returns the new value.
261
+ # +key+ must already exist. If +key+ is not an integer, it is assumed to be
262
+ # 0.
263
+
264
+ def incr(key, amount = 1)
265
+ raise MemCacheDbError, "Update of readonly cache" if @readonly
266
+ with_server(key) do |server, cache_key|
267
+ cache_incr server, cache_key, amount
268
+ end
269
+ rescue TypeError => err
270
+ handle_error nil, err
271
+ end
272
+
273
+ ##
274
+ # Add +key+ to the cache with value +value+ that expires in +expiry+
275
+ # seconds. If +raw+ is true, +value+ will not be Marshalled.
276
+ #
277
+ # Warning: Readers should not call this method in the event of a cache miss;
278
+ # see MemCacheDb#add.
279
+
280
+ ONE_MB = 1024 * 1024
281
+
282
+ def set(key, value, expiry = 0, raw = false)
283
+ raise MemCacheDbError, "Update of readonly cache" if @readonly
284
+ with_server(key) do |server, cache_key|
285
+
286
+ value = Marshal.dump value unless raw
287
+ logger.debug { "SET #{key} to #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
288
+
289
+ data = value.to_s
290
+ raise MemCacheDbError, "Value too large, MemCacheDbd can only store 1MB of data per key" if data.size > ONE_MB
291
+
292
+ command = "set #{cache_key} 0 #{expiry} #{data.size}\r\n#{data}\r\n"
293
+
294
+ with_socket_management(server) do |socket|
295
+ socket.write command
296
+ result = socket.gets
297
+ raise_on_error_response! result
298
+
299
+ if result.nil?
300
+ server.close
301
+ raise MemCacheDbError, "lost connection to #{server.host}:#{server.port}"
302
+ end
303
+
304
+ result
305
+ end
306
+ end
307
+ end
308
+
309
+ ##
310
+ # Add +key+ to the cache with value +value+ that expires in +expiry+
311
+ # seconds, but only if +key+ does not already exist in the cache.
312
+ # If +raw+ is true, +value+ will not be Marshalled.
313
+ #
314
+ # Readers should call this method in the event of a cache miss, not
315
+ # MemCacheDb#set or MemCacheDb#[]=.
316
+
317
+ def add(key, value, expiry = 0, raw = false)
318
+ raise MemCacheDbError, "Update of readonly cache" if @readonly
319
+ with_server(key) do |server, cache_key|
320
+ value = Marshal.dump value unless raw
321
+ logger.debug { "ADD #{key} to #{server}: #{value ? value.to_s.size : 'nil'}" } if logger
322
+ command = "add #{cache_key} 0 #{expiry} #{value.to_s.size}\r\n#{value}\r\n"
323
+
324
+ with_socket_management(server) do |socket|
325
+ socket.write command
326
+ result = socket.gets
327
+ raise_on_error_response! result
328
+ result
329
+ end
330
+ end
331
+ end
332
+
333
+ ##
334
+ # Removes +key+ from the cache in +expiry+ seconds.
335
+
336
+ def delete(key, expiry = 0)
337
+ raise MemCacheDbError, "Update of readonly cache" if @readonly
338
+ with_server(key) do |server, cache_key|
339
+ with_socket_management(server) do |socket|
340
+ socket.write "delete #{cache_key} #{expiry}\r\n"
341
+ result = socket.gets
342
+ raise_on_error_response! result
343
+ result
344
+ end
345
+ end
346
+ end
347
+
348
+ ##
349
+ # Flush the cache from all MemCacheDb servers.
350
+
351
+ def flush_all
352
+ raise MemCacheDbError, 'No active servers' unless active?
353
+ raise MemCacheDbError, "Update of readonly cache" if @readonly
354
+
355
+ begin
356
+ @mutex.lock if @multithread
357
+ @servers.each do |server|
358
+ with_socket_management(server) do |socket|
359
+ socket.write "flush_all\r\n"
360
+ result = socket.gets
361
+ raise_on_error_response! result
362
+ result
363
+ end
364
+ end
365
+ rescue IndexError => err
366
+ handle_error nil, err
367
+ ensure
368
+ @mutex.unlock if @multithread
369
+ end
370
+ end
371
+
372
+ ##
373
+ # Reset the connection to all memcachedb servers. This should be called if
374
+ # there is a problem with a cache lookup that might have left the connection
375
+ # in a corrupted state.
376
+
377
+ def reset
378
+ @servers.each { |server| server.close }
379
+ end
380
+
381
+ ##
382
+ # Returns statistics for each memcachedb server. An explanation of the
383
+ # statistics can be found in the memcachedb docs:
384
+ #
385
+ # http://code.sixapart.com/svn/memcached/trunk/server/doc/protocol.txt
386
+ #
387
+ # Example:
388
+ #
389
+ # >> pp CACHE.stats
390
+ # {"localhost:11211"=>
391
+ # {"bytes"=>4718,
392
+ # "pid"=>20188,
393
+ # "connection_structures"=>4,
394
+ # "time"=>1162278121,
395
+ # "pointer_size"=>32,
396
+ # "limit_maxbytes"=>67108864,
397
+ # "cmd_get"=>14532,
398
+ # "version"=>"1.2.0",
399
+ # "bytes_written"=>432583,
400
+ # "cmd_set"=>32,
401
+ # "get_misses"=>0,
402
+ # "total_connections"=>19,
403
+ # "curr_connections"=>3,
404
+ # "curr_items"=>4,
405
+ # "uptime"=>1557,
406
+ # "get_hits"=>14532,
407
+ # "total_items"=>32,
408
+ # "rusage_system"=>0.313952,
409
+ # "rusage_user"=>0.119981,
410
+ # "bytes_read"=>190619}}
411
+ # => nil
412
+
413
+ def stats
414
+ raise MemCacheDbError, "No active servers" unless active?
415
+ server_stats = {}
416
+
417
+ @servers.each do |server|
418
+ next unless server.alive?
419
+
420
+ with_socket_management(server) do |socket|
421
+ value = nil
422
+ socket.write "stats\r\n"
423
+ stats = {}
424
+ while line = socket.gets do
425
+ raise_on_error_response! line
426
+ break if line == "END\r\n"
427
+ if line =~ /\ASTAT ([\S]+) ([\w\.\:]+)/ then
428
+ name, value = $1, $2
429
+ stats[name] = case name
430
+ when 'version'
431
+ value
432
+ when 'rusage_user', 'rusage_system' then
433
+ seconds, microseconds = value.split(/:/, 2)
434
+ microseconds ||= 0
435
+ Float(seconds) + (Float(microseconds) / 1_000_000)
436
+ else
437
+ if value =~ /\A\d+\Z/ then
438
+ value.to_i
439
+ else
440
+ value
441
+ end
442
+ end
443
+ end
444
+ end
445
+ server_stats["#{server.host}:#{server.port}"] = stats
446
+ end
447
+ end
448
+
449
+ raise MemCacheDbError, "No active servers" if server_stats.empty?
450
+ server_stats
451
+ end
452
+
453
+ ##
454
+ # Shortcut to get a value from the cache.
455
+
456
+ alias [] get
457
+
458
+ ##
459
+ # Shortcut to save a value in the cache. This method does not set an
460
+ # expiration on the entry. Use set to specify an explicit expiry.
461
+
462
+ def []=(key, value)
463
+ set key, value
464
+ end
465
+
466
+ protected unless $TESTING
467
+
468
+ ##
469
+ # Create a key for the cache, incorporating the namespace qualifier if
470
+ # requested.
471
+
472
+ def make_cache_key(key)
473
+ if namespace.nil? then
474
+ key
475
+ else
476
+ "#{@namespace}:#{key}"
477
+ end
478
+ end
479
+
480
+ ##
481
+ # Returns an interoperable hash value for +key+. (I think, docs are
482
+ # sketchy for down servers).
483
+
484
+ def hash_for(key)
485
+ Zlib.crc32(key)
486
+ end
487
+
488
+ ##
489
+ # Pick a server to handle the request based on a hash of the key.
490
+
491
+ def get_server_for_key(key, options = {})
492
+ raise ArgumentError, "illegal character in key #{key.inspect}" if
493
+ key =~ /\s/
494
+ raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
495
+ raise MemCacheDbError, "No servers available" if @servers.empty?
496
+ return @servers.first if @servers.length == 1
497
+
498
+ hkey = hash_for(key)
499
+
500
+ 20.times do |try|
501
+ entryidx = Continuum.binary_search(@continuum, hkey)
502
+ server = @continuum[entryidx].server
503
+ return server if server.alive?
504
+ break unless failover
505
+ hkey = hash_for "#{try}#{key}"
506
+ end
507
+
508
+ raise MemCacheDbError, "No servers available"
509
+ end
510
+
511
+ ##
512
+ # Performs a raw decr for +cache_key+ from +server+. Returns nil if not
513
+ # found.
514
+
515
+ def cache_decr(server, cache_key, amount)
516
+ with_socket_management(server) do |socket|
517
+ socket.write "decr #{cache_key} #{amount}\r\n"
518
+ text = socket.gets
519
+ raise_on_error_response! text
520
+ return nil if text == "NOT_FOUND\r\n"
521
+ return text.to_i
522
+ end
523
+ end
524
+
525
+ ##
526
+ # Fetches the raw data for +cache_key+ from +server+. Returns nil on cache
527
+ # miss.
528
+
529
+ def cache_get(server, cache_key)
530
+ with_socket_management(server) do |socket|
531
+ socket.write "get #{cache_key}\r\n"
532
+ keyline = socket.gets # "VALUE <key> <flags> <bytes>\r\n"
533
+
534
+ if keyline.nil? then
535
+ server.close
536
+ raise MemCacheDbError, "lost connection to #{server.host}:#{server.port}"
537
+ end
538
+
539
+ raise_on_error_response! keyline
540
+ return nil if keyline == "END\r\n"
541
+
542
+ unless keyline =~ /(\d+)\r/ then
543
+ server.close
544
+ raise MemCacheDbError, "unexpected response #{keyline.inspect}"
545
+ end
546
+ value = socket.read $1.to_i
547
+ socket.read 2 # "\r\n"
548
+ socket.gets # "END\r\n"
549
+ return value
550
+ end
551
+ end
552
+
553
+ ##
554
+ # Fetches +cache_keys+ from +server+ using a multi-get.
555
+
556
+ def cache_get_multi(server, cache_keys)
557
+ with_socket_management(server) do |socket|
558
+ values = {}
559
+ socket.write "get #{cache_keys}\r\n"
560
+
561
+ while keyline = socket.gets do
562
+ return values if keyline == "END\r\n"
563
+ raise_on_error_response! keyline
564
+
565
+ unless keyline =~ /\AVALUE (.+) (.+) (.+)/ then
566
+ server.close
567
+ raise MemCacheDbError, "unexpected response #{keyline.inspect}"
568
+ end
569
+
570
+ key, data_length = $1, $3
571
+ values[$1] = socket.read data_length.to_i
572
+ socket.read(2) # "\r\n"
573
+ end
574
+
575
+ server.close
576
+ raise MemCacheDbError, "lost connection to #{server.host}:#{server.port}" # TODO: retry here too
577
+ end
578
+ end
579
+
580
+ ##
581
+ # Performs a raw incr for +cache_key+ from +server+. Returns nil if not
582
+ # found.
583
+
584
+ def cache_incr(server, cache_key, amount)
585
+ with_socket_management(server) do |socket|
586
+ socket.write "incr #{cache_key} #{amount}\r\n"
587
+ text = socket.gets
588
+ raise_on_error_response! text
589
+ return nil if text == "NOT_FOUND\r\n"
590
+ return text.to_i
591
+ end
592
+ end
593
+
594
+ ##
595
+ # Gets or creates a socket connected to the given server, and yields it
596
+ # to the block, wrapped in a mutex synchronization if @multithread is true.
597
+ #
598
+ # If a socket error (SocketError, SystemCallError, IOError) or protocol error
599
+ # (MemCacheDbError) is raised by the block, closes the socket, attempts to
600
+ # connect again, and retries the block (once). If an error is again raised,
601
+ # reraises it as MemCacheDbError.
602
+ #
603
+ # If unable to connect to the server (or if in the reconnect wait period),
604
+ # raises MemCacheDbError. Note that the socket connect code marks a server
605
+ # dead for a timeout period, so retrying does not apply to connection attempt
606
+ # failures (but does still apply to unexpectedly lost connections etc.).
607
+
608
+ def with_socket_management(server, &block)
609
+ @mutex.lock if @multithread
610
+ retried = false
611
+
612
+ begin
613
+ socket = server.socket
614
+
615
+ # Raise an IndexError to show this server is out of whack. If were inside
616
+ # a with_server block, we'll catch it and attempt to restart the operation.
617
+
618
+ raise IndexError, "No connection to server (#{server.status})" if socket.nil?
619
+
620
+ block.call(socket)
621
+
622
+ rescue SocketError => err
623
+ logger.warn { "Socket failure: #{err.message}" } if logger
624
+ server.mark_dead(err)
625
+ handle_error(server, err)
626
+
627
+ rescue MemCacheDbError, SystemCallError, IOError => err
628
+ logger.warn { "Generic failure: #{err.class.name}: #{err.message}" } if logger
629
+ handle_error(server, err) if retried || socket.nil?
630
+ retried = true
631
+ retry
632
+ end
633
+ ensure
634
+ @mutex.unlock if @multithread
635
+ end
636
+
637
+ def with_server(key)
638
+ retried = false
639
+ begin
640
+ server, cache_key = request_setup(key)
641
+ yield server, cache_key
642
+ rescue IndexError => e
643
+ logger.warn { "Server failed: #{e.class.name}: #{e.message}" } if logger
644
+ if !retried && @servers.size > 1
645
+ logger.info { "Connection to server #{server.inspect} DIED! Retrying operation..." } if logger
646
+ retried = true
647
+ retry
648
+ end
649
+ handle_error(nil, e)
650
+ end
651
+ end
652
+
653
+ ##
654
+ # Handles +error+ from +server+.
655
+
656
+ def handle_error(server, error)
657
+ raise error if error.is_a?(MemCacheDbError)
658
+ server.close if server
659
+ new_error = MemCacheDbError.new error.message
660
+ new_error.set_backtrace error.backtrace
661
+ raise new_error
662
+ end
663
+
664
+ ##
665
+ # Performs setup for making a request with +key+ from memcachedb. Returns
666
+ # the server to fetch the key from and the complete key to use.
667
+
668
+ def request_setup(key)
669
+ raise MemCacheDbError, 'No active servers' unless active?
670
+ cache_key = make_cache_key key
671
+ server = get_server_for_key cache_key
672
+ return server, cache_key
673
+ end
674
+
675
+ def raise_on_error_response!(response)
676
+ if response =~ /\A(?:CLIENT_|SERVER_)?ERROR(.*)/
677
+ raise MemCacheDbError, $1.strip
678
+ end
679
+ end
680
+
681
+ def create_continuum_for(servers)
682
+ total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
683
+ continuum = []
684
+
685
+ servers.each do |server|
686
+ entry_count_for(server, servers.size, total_weight).times do |idx|
687
+ hash = Digest::SHA1.hexdigest("#{server.host}:#{server.port}:#{idx}")
688
+ value = Integer("0x#{hash[0..7]}")
689
+ continuum << Continuum::Entry.new(value, server)
690
+ end
691
+ end
692
+
693
+ continuum.sort { |a, b| a.value <=> b.value }
694
+ end
695
+
696
+ def entry_count_for(server, total_servers, total_weight)
697
+ ((total_servers * Continuum::POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
698
+ end
699
+
700
+ ##
701
+ # This class represents a memcachedb server instance.
702
+
703
+ class Server
704
+
705
+ ##
706
+ # The amount of time to wait to establish a connection with a memcachedb
707
+ # server. If a connection cannot be established within this time limit,
708
+ # the server will be marked as down.
709
+
710
+ CONNECT_TIMEOUT = 0.25
711
+
712
+ ##
713
+ # The amount of time to wait before attempting to re-establish a
714
+ # connection with a server that is marked dead.
715
+
716
+ RETRY_DELAY = 30.0
717
+
718
+ ##
719
+ # The host the memcachedb server is running on.
720
+
721
+ attr_reader :host
722
+
723
+ ##
724
+ # The port the memcachedb server is listening on.
725
+
726
+ attr_reader :port
727
+
728
+ ##
729
+ # The weight given to the server.
730
+
731
+ attr_reader :weight
732
+
733
+ ##
734
+ # The time of next retry if the connection is dead.
735
+
736
+ attr_reader :retry
737
+
738
+ ##
739
+ # A text status string describing the state of the server.
740
+
741
+ attr_reader :status
742
+
743
+ attr_reader :multithread
744
+ attr_reader :logger
745
+
746
+ ##
747
+ # Create a new MemCacheDb::Server object for the memcachedb instance
748
+ # listening on the given host and port, weighted by the given weight.
749
+
750
+ def initialize(memcache, host, port = DEFAULT_PORT, weight = DEFAULT_WEIGHT)
751
+ raise ArgumentError, "No host specified" if host.nil? or host.empty?
752
+ raise ArgumentError, "No port specified" if port.nil? or port.to_i.zero?
753
+
754
+ @host = host
755
+ @port = port.to_i
756
+ @weight = weight.to_i
757
+
758
+ @multithread = memcache.multithread
759
+ @mutex = Mutex.new
760
+
761
+ @sock = nil
762
+ @retry = nil
763
+ @status = 'NOT CONNECTED'
764
+ @timeout = memcache.timeout
765
+ @logger = memcache.logger
766
+ end
767
+
768
+ ##
769
+ # Return a string representation of the server object.
770
+
771
+ def inspect
772
+ "<MemCacheDb::Server: %s:%d [%d] (%s)>" % [@host, @port, @weight, @status]
773
+ end
774
+
775
+ ##
776
+ # Check whether the server connection is alive. This will cause the
777
+ # socket to attempt to connect if it isn't already connected and or if
778
+ # the server was previously marked as down and the retry time has
779
+ # been exceeded.
780
+
781
+ def alive?
782
+ !!socket
783
+ end
784
+
785
+ ##
786
+ # Try to connect to the memcachedb server targeted by this object.
787
+ # Returns the connected socket object on success or nil on failure.
788
+
789
+ def socket
790
+ @mutex.lock if @multithread
791
+ return @sock if @sock and not @sock.closed?
792
+
793
+ @sock = nil
794
+
795
+ # If the host was dead, don't retry for a while.
796
+ return if @retry and @retry > Time.now
797
+
798
+ # Attempt to connect if not already connected.
799
+ begin
800
+ @sock = @timeout ? TCPTimeoutSocket.new(@host, @port, @timeout) : TCPSocket.new(@host, @port)
801
+
802
+ if Socket.constants.include? 'TCP_NODELAY' then
803
+ @sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
804
+ end
805
+ @retry = nil
806
+ @status = 'CONNECTED'
807
+ rescue SocketError, SystemCallError, IOError, Timeout::Error => err
808
+ logger.warn { "Unable to open socket: #{err.class.name}, #{err.message}" } if logger
809
+ mark_dead err
810
+ end
811
+
812
+ return @sock
813
+ ensure
814
+ @mutex.unlock if @multithread
815
+ end
816
+
817
+ ##
818
+ # Close the connection to the memcachedb server targeted by this
819
+ # object. The server is not considered dead.
820
+
821
+ def close
822
+ @mutex.lock if @multithread
823
+ @sock.close if @sock && !@sock.closed?
824
+ @sock = nil
825
+ @retry = nil
826
+ @status = "NOT CONNECTED"
827
+ ensure
828
+ @mutex.unlock if @multithread
829
+ end
830
+
831
+ ##
832
+ # Mark the server as dead and close its socket.
833
+
834
+ def mark_dead(error)
835
+ @sock.close if @sock && !@sock.closed?
836
+ @sock = nil
837
+ @retry = Time.now + RETRY_DELAY
838
+
839
+ reason = "#{error.class.name}: #{error.message}"
840
+ @status = sprintf "%s:%s DEAD (%s), will retry at %s", @host, @port, reason, @retry
841
+ @logger.info { @status } if @logger
842
+ end
843
+
844
+ end
845
+
846
+ ##
847
+ # Base MemCacheDb exception class.
848
+
849
+ class MemCacheDbError < RuntimeError; end
850
+
851
+ end
852
+
853
+ # TCPSocket facade class which implements timeouts.
854
+ class TCPTimeoutSocket
855
+
856
+ def initialize(host, port, timeout)
857
+ Timeout::timeout(MemCacheDb::Server::CONNECT_TIMEOUT, SocketError) do
858
+ @sock = TCPSocket.new(host, port)
859
+ @len = timeout
860
+ end
861
+ end
862
+
863
+ def write(*args)
864
+ Timeout::timeout(@len, SocketError) do
865
+ @sock.write(*args)
866
+ end
867
+ end
868
+
869
+ def gets(*args)
870
+ Timeout::timeout(@len, SocketError) do
871
+ @sock.gets(*args)
872
+ end
873
+ end
874
+
875
+ def read(*args)
876
+ Timeout::timeout(@len, SocketError) do
877
+ @sock.read(*args)
878
+ end
879
+ end
880
+
881
+ def _socket
882
+ @sock
883
+ end
884
+
885
+ def method_missing(meth, *args)
886
+ @sock.__send__(meth, *args)
887
+ end
888
+
889
+ def closed?
890
+ @sock.closed?
891
+ end
892
+
893
+ def close
894
+ @sock.close
895
+ end
896
+ end