mob-dalli 1.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,532 @@
1
+ require 'socket'
2
+ require 'timeout'
3
+ require 'zlib'
4
+
5
+ module Dalli
6
+ class Server
7
+ attr_accessor :hostname
8
+ attr_accessor :port
9
+ attr_accessor :weight
10
+ attr_accessor :options
11
+
12
+ DEFAULTS = {
13
+ # seconds between trying to contact a remote server
14
+ :down_retry_delay => 1,
15
+ # connect/read/write timeout for socket operations
16
+ :socket_timeout => 0.5,
17
+ # times a socket operation may fail before considering the server dead
18
+ :socket_max_failures => 2,
19
+ # amount of time to sleep between retries when a failure occurs
20
+ :socket_failure_delay => 0.01,
21
+ # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
22
+ :value_max_bytes => 1024 * 1024,
23
+ :username => nil,
24
+ :password => nil,
25
+ :async => false,
26
+ }
27
+
28
+ def initialize(attribs, options = {})
29
+ (@hostname, @port, @weight) = attribs.split(':')
30
+ @port ||= 11211
31
+ @port = Integer(@port)
32
+ @weight ||= 1
33
+ @weight = Integer(@weight)
34
+ @fail_count = 0
35
+ @down_at = nil
36
+ @last_down_at = nil
37
+ @options = DEFAULTS.merge(options)
38
+ @sock = nil
39
+ @msg = nil
40
+ end
41
+
42
+ # Chokepoint method for instrumentation
43
+ def request(op, *args)
44
+ raise Dalli::NetworkError, "#{hostname}:#{port} is down: #{@error} #{@msg}" unless alive?
45
+ begin
46
+ send(op, *args)
47
+ rescue Dalli::NetworkError
48
+ raise
49
+ rescue Dalli::MarshalError => ex
50
+ Dalli.logger.error "Marshalling error for key '#{args.first}': #{ex.message}"
51
+ Dalli.logger.error "You are trying to cache a Ruby object which cannot be serialized to memcached."
52
+ Dalli.logger.error ex.backtrace.join("\n\t")
53
+ false
54
+ rescue Dalli::DalliError
55
+ raise
56
+ rescue => ex
57
+ Dalli.logger.error "Unexpected exception in Dalli: #{ex.class.name}: #{ex.message}"
58
+ Dalli.logger.error "This is a bug in Dalli, please enter an issue in Github if it does not already exist."
59
+ Dalli.logger.error ex.backtrace.join("\n\t")
60
+ down!
61
+ end
62
+ end
63
+
64
+ def alive?
65
+ return true if @sock
66
+
67
+ if @last_down_at && @last_down_at + options[:down_retry_delay] >= Time.now
68
+ time = @last_down_at + options[:down_retry_delay] - Time.now
69
+ Dalli.logger.debug { "down_retry_delay not reached for #{hostname}:#{port} (%.3f seconds left)" % time }
70
+ return false
71
+ end
72
+
73
+ connect
74
+ !!@sock
75
+ rescue Dalli::NetworkError
76
+ false
77
+ end
78
+
79
+ def close
80
+ return unless @sock
81
+ @sock.close rescue nil
82
+ @sock = nil
83
+ end
84
+
85
+ def lock!
86
+ end
87
+
88
+ def unlock!
89
+ end
90
+
91
+ # NOTE: Additional public methods should be overridden in Dalli::Threadsafe
92
+
93
+ private
94
+
95
+ def is_unix_socket?(string)
96
+ !!(/^\/(.+)$/ =~ string)
97
+ end
98
+
99
+ def failure!
100
+ Dalli.logger.info { "#{hostname}:#{port} failed (count: #{@fail_count})" }
101
+
102
+ @fail_count += 1
103
+ if @fail_count >= options[:socket_max_failures]
104
+ down!
105
+ else
106
+ close
107
+ sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
108
+ raise Dalli::NetworkError, "Socket operation failed, retrying..."
109
+ end
110
+ end
111
+
112
+ def down!
113
+ close
114
+
115
+ @last_down_at = Time.now
116
+
117
+ if @down_at
118
+ time = Time.now - @down_at
119
+ Dalli.logger.debug { "#{hostname}:#{port} is still down (for %.3f seconds now)" % time }
120
+ else
121
+ @down_at = @last_down_at
122
+ Dalli.logger.warn { "#{hostname}:#{port} is down" }
123
+ end
124
+
125
+ @error = $! && $!.class.name
126
+ @msg = @msg || ($! && $!.message && !$!.message.empty? && $!.message)
127
+ raise Dalli::NetworkError, "#{hostname}:#{port} is down: #{@error} #{@msg}"
128
+ end
129
+
130
+ def up!
131
+ if @down_at
132
+ time = Time.now - @down_at
133
+ Dalli.logger.warn { "#{hostname}:#{port} is back (downtime was %.3f seconds)" % time }
134
+ end
135
+
136
+ @fail_count = 0
137
+ @down_at = nil
138
+ @last_down_at = nil
139
+ @msg = nil
140
+ @error = nil
141
+ end
142
+
143
+ def multi?
144
+ Thread.current[:dalli_multi]
145
+ end
146
+
147
+ def get(key)
148
+ req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
149
+ write(req)
150
+ generic_response(true)
151
+ end
152
+
153
+ def getkq(key)
154
+ req = [REQUEST, OPCODES[:getkq], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:getkq])
155
+ write(req)
156
+ end
157
+
158
+ def set(key, value, ttl, cas, options)
159
+ (value, flags) = serialize(key, value, options)
160
+
161
+ req = [REQUEST, OPCODES[multi? ? :setq : :set], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:set])
162
+ write(req)
163
+ generic_response unless multi?
164
+ end
165
+
166
+ def add(key, value, ttl, options)
167
+ (value, flags) = serialize(key, value, options)
168
+
169
+ req = [REQUEST, OPCODES[multi? ? :addq : :add], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:add])
170
+ write(req)
171
+ generic_response unless multi?
172
+ end
173
+
174
+ def replace(key, value, ttl, options)
175
+ (value, flags) = serialize(key, value, options)
176
+ req = [REQUEST, OPCODES[multi? ? :replaceq : :replace], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:replace])
177
+ write(req)
178
+ generic_response unless multi?
179
+ end
180
+
181
+ def delete(key)
182
+ req = [REQUEST, OPCODES[multi? ? :deleteq : :delete], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:delete])
183
+ write(req)
184
+ generic_response unless multi?
185
+ end
186
+
187
+ def flush(ttl)
188
+ req = [REQUEST, OPCODES[:flush], 0, 4, 0, 0, 4, 0, 0, 0].pack(FORMAT[:flush])
189
+ write(req)
190
+ generic_response
191
+ end
192
+
193
+ def decr(key, count, ttl, default)
194
+ expiry = default ? ttl : 0xFFFFFFFF
195
+ default ||= 0
196
+ (h, l) = split(count)
197
+ (dh, dl) = split(default)
198
+ req = [REQUEST, OPCODES[:decr], key.bytesize, 20, 0, 0, key.bytesize + 20, 0, 0, h, l, dh, dl, expiry, key].pack(FORMAT[:decr])
199
+ write(req)
200
+ body = generic_response
201
+ body ? longlong(*body.unpack('NN')) : body
202
+ end
203
+
204
+ def incr(key, count, ttl, default)
205
+ expiry = default ? ttl : 0xFFFFFFFF
206
+ default ||= 0
207
+ (h, l) = split(count)
208
+ (dh, dl) = split(default)
209
+ req = [REQUEST, OPCODES[:incr], key.bytesize, 20, 0, 0, key.bytesize + 20, 0, 0, h, l, dh, dl, expiry, key].pack(FORMAT[:incr])
210
+ write(req)
211
+ body = generic_response
212
+ body ? longlong(*body.unpack('NN')) : body
213
+ end
214
+
215
+ # Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
216
+ # We need to read all the responses at once.
217
+ def noop
218
+ req = [REQUEST, OPCODES[:noop], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
219
+ write(req)
220
+ multi_response
221
+ end
222
+
223
+ def append(key, value)
224
+ req = [REQUEST, OPCODES[:append], key.bytesize, 0, 0, 0, value.bytesize + key.bytesize, 0, 0, key, value].pack(FORMAT[:append])
225
+ write(req)
226
+ generic_response
227
+ end
228
+
229
+ def prepend(key, value)
230
+ req = [REQUEST, OPCODES[:prepend], key.bytesize, 0, 0, 0, value.bytesize + key.bytesize, 0, 0, key, value].pack(FORMAT[:prepend])
231
+ write(req)
232
+ generic_response
233
+ end
234
+
235
+ def stats(info='')
236
+ req = [REQUEST, OPCODES[:stat], info.bytesize, 0, 0, 0, info.bytesize, 0, 0, info].pack(FORMAT[:stat])
237
+ write(req)
238
+ keyvalue_response
239
+ end
240
+
241
+ def cas(key)
242
+ req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
243
+ write(req)
244
+ cas_response
245
+ end
246
+
247
+ def version
248
+ req = [REQUEST, OPCODES[:version], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
249
+ write(req)
250
+ generic_response
251
+ end
252
+
253
+ COMPRESSION_MIN_SIZE = 1024
254
+
255
+ # http://www.hjp.at/zettel/m/memcached_flags.rxml
256
+ # Looks like most clients use bit 0 to indicate native language serialization
257
+ # and bit 1 to indicate gzip compression.
258
+ FLAG_MARSHALLED = 0x1
259
+ FLAG_COMPRESSED = 0x2
260
+
261
+ def serialize(key, value, options=nil)
262
+ marshalled = false
263
+ value = unless options && options[:raw]
264
+ marshalled = true
265
+ begin
266
+ Marshal.dump(value)
267
+ rescue => ex
268
+ # Marshalling can throw several different types of generic Ruby exceptions.
269
+ # Convert to a specific exception so we can special case it higher up the stack.
270
+ exc = Dalli::MarshalError.new(ex.message)
271
+ exc.set_backtrace ex.backtrace
272
+ raise exc
273
+ end
274
+ else
275
+ value.to_s
276
+ end
277
+ compressed = false
278
+ if @options[:compression] && value.bytesize >= COMPRESSION_MIN_SIZE
279
+ value = Zlib::Deflate.deflate(value)
280
+ compressed = true
281
+ end
282
+ raise Dalli::DalliError, "Value too large, memcached can only store #{@options[:value_max_bytes]} bytes per key [key: #{key}, size: #{value.bytesize}]" if value.bytesize > @options[:value_max_bytes]
283
+ flags = 0
284
+ flags |= FLAG_COMPRESSED if compressed
285
+ flags |= FLAG_MARSHALLED if marshalled
286
+ flags |= options[:flags] if options && options[:flags]
287
+ [value, flags]
288
+ end
289
+
290
+ def deserialize(value, flags)
291
+ value = Zlib::Inflate.inflate(value) if (flags & FLAG_COMPRESSED) != 0
292
+ value = Marshal.load(value) if (flags & FLAG_MARSHALLED) != 0
293
+ value
294
+ rescue TypeError, ArgumentError
295
+ raise DalliError, "Unable to unmarshal value: #{$!.message}"
296
+ rescue Zlib::Error
297
+ raise DalliError, "Unable to uncompress value: #{$!.message}"
298
+ end
299
+
300
+ def cas_response
301
+ header = read(24)
302
+ raise Dalli::NetworkError, 'No response' if !header
303
+ (extras, type, status, count, _, cas) = header.unpack(CAS_HEADER)
304
+ data = read(count) if count > 0
305
+ if status == 1
306
+ nil
307
+ elsif status != 0
308
+ raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
309
+ elsif data
310
+ flags = data[0...extras].unpack('N')[0]
311
+ value = data[extras..-1]
312
+ data = deserialize(value, flags)
313
+ end
314
+ [data, cas]
315
+ end
316
+
317
+ CAS_HEADER = '@4CCnNNQ'
318
+ NORMAL_HEADER = '@4CCnN'
319
+ KV_HEADER = '@2n@6nN'
320
+
321
+ def generic_response(unpack=false)
322
+ header = read(24)
323
+ raise Dalli::NetworkError, 'No response' if !header
324
+ (extras, type, status, count) = header.unpack(NORMAL_HEADER)
325
+ data = read(count) if count > 0
326
+ if status == 1
327
+ nil
328
+ elsif status == 2 || status == 5
329
+ false # Not stored, normal status for add operation
330
+ elsif status != 0
331
+ raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
332
+ elsif data
333
+ flags = data[0...extras].unpack('N')[0]
334
+ value = data[extras..-1]
335
+ unpack ? deserialize(value, flags) : value
336
+ else
337
+ true
338
+ end
339
+ end
340
+
341
+ def keyvalue_response
342
+ hash = {}
343
+ loop do
344
+ header = read(24)
345
+ raise Dalli::NetworkError, 'No response' if !header
346
+ (key_length, status, body_length) = header.unpack(KV_HEADER)
347
+ return hash if key_length == 0
348
+ key = read(key_length)
349
+ value = read(body_length - key_length) if body_length - key_length > 0
350
+ hash[key] = value
351
+ end
352
+ end
353
+
354
+ def multi_response
355
+ hash = {}
356
+ loop do
357
+ header = read(24)
358
+ raise Dalli::NetworkError, 'No response' if !header
359
+ (key_length, status, body_length) = header.unpack(KV_HEADER)
360
+ return hash if key_length == 0
361
+ flags = read(4).unpack('N')[0]
362
+ key = read(key_length)
363
+ value = read(body_length - key_length - 4) if body_length - key_length - 4 > 0
364
+ hash[key] = deserialize(value, flags)
365
+ end
366
+ end
367
+
368
+ def write(bytes)
369
+ begin
370
+ @sock.write(bytes)
371
+ rescue SystemCallError, Timeout::Error
372
+ failure!
373
+ retry
374
+ end
375
+ end
376
+
377
+ def read(count)
378
+ begin
379
+ @sock.readfull(count)
380
+ rescue SystemCallError, Timeout::Error, EOFError
381
+ failure!
382
+ retry
383
+ end
384
+ end
385
+
386
+ def connect
387
+ Dalli.logger.debug { "Dalli::Server#connect #{hostname}:#{port}" }
388
+
389
+ begin
390
+ if @hostname =~ /^\//
391
+ @sock = USocket.new(hostname)
392
+ elsif options[:async]
393
+ raise Dalli::DalliError, "EM support not enabled, as em-synchrony is not installed." if not defined?(AsyncSocket)
394
+ @sock = AsyncSocket.open(hostname, port, :timeout => options[:socket_timeout])
395
+ else
396
+ @sock = KSocket.open(hostname, port, :timeout => options[:socket_timeout])
397
+ end
398
+ @version = version # trigger actual connect
399
+ sasl_authentication if need_auth?
400
+ up!
401
+ rescue Dalli::DalliError # SASL auth failure
402
+ raise
403
+ rescue SystemCallError, Timeout::Error, EOFError, SocketError
404
+ # SocketError = DNS resolution failure
405
+ failure!
406
+ retry
407
+ end
408
+ end
409
+
410
+ def split(n)
411
+ [n >> 32, 0xFFFFFFFF & n]
412
+ end
413
+
414
+ def longlong(a, b)
415
+ (a << 32) | b
416
+ end
417
+
418
+ REQUEST = 0x80
419
+ RESPONSE = 0x81
420
+
421
+ RESPONSE_CODES = {
422
+ 0 => 'No error',
423
+ 1 => 'Key not found',
424
+ 2 => 'Key exists',
425
+ 3 => 'Value too large',
426
+ 4 => 'Invalid arguments',
427
+ 5 => 'Item not stored',
428
+ 6 => 'Incr/decr on a non-numeric value',
429
+ 0x20 => 'Authentication required',
430
+ 0x81 => 'Unknown command',
431
+ 0x82 => 'Out of memory',
432
+ }
433
+
434
+ OPCODES = {
435
+ :get => 0x00,
436
+ :set => 0x01,
437
+ :add => 0x02,
438
+ :replace => 0x03,
439
+ :delete => 0x04,
440
+ :incr => 0x05,
441
+ :decr => 0x06,
442
+ :flush => 0x08,
443
+ :noop => 0x0A,
444
+ :version => 0x0B,
445
+ :getkq => 0x0D,
446
+ :append => 0x0E,
447
+ :prepend => 0x0F,
448
+ :stat => 0x10,
449
+ :setq => 0x11,
450
+ :addq => 0x12,
451
+ :replaceq => 0x13,
452
+ :deleteq => 0x14,
453
+ :incrq => 0x15,
454
+ :decrq => 0x16,
455
+ :auth_negotiation => 0x20,
456
+ :auth_request => 0x21,
457
+ :auth_continue => 0x22,
458
+ }
459
+
460
+ HEADER = "CCnCCnNNQ"
461
+ OP_FORMAT = {
462
+ :get => 'a*',
463
+ :set => 'NNa*a*',
464
+ :add => 'NNa*a*',
465
+ :replace => 'NNa*a*',
466
+ :delete => 'a*',
467
+ :incr => 'NNNNNa*',
468
+ :decr => 'NNNNNa*',
469
+ :flush => 'N',
470
+ :noop => '',
471
+ :getkq => 'a*',
472
+ :version => '',
473
+ :stat => 'a*',
474
+ :append => 'a*a*',
475
+ :prepend => 'a*a*',
476
+ :auth_request => 'a*a*',
477
+ :auth_continue => 'a*a*',
478
+ }
479
+ FORMAT = OP_FORMAT.inject({}) { |memo, (k, v)| memo[k] = HEADER + v; memo }
480
+
481
+
482
+ #######
483
+ # SASL authentication support for NorthScale
484
+ #######
485
+
486
+ def need_auth?
487
+ @options[:username] || ENV['MEMCACHE_USERNAME']
488
+ end
489
+
490
+ def username
491
+ @options[:username] || ENV['MEMCACHE_USERNAME']
492
+ end
493
+
494
+ def password
495
+ @options[:password] || ENV['MEMCACHE_PASSWORD']
496
+ end
497
+
498
+ def sasl_authentication
499
+ Dalli.logger.info { "Dalli/SASL authenticating as #{username}" }
500
+
501
+ # negotiate
502
+ req = [REQUEST, OPCODES[:auth_negotiation], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
503
+ write(req)
504
+ header = read(24)
505
+ raise Dalli::NetworkError, 'No response' if !header
506
+ (extras, type, status, count) = header.unpack(NORMAL_HEADER)
507
+ raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
508
+ content = read(count)
509
+ return (Dalli.logger.debug("Authentication not required/supported by server")) if status == 0x81
510
+ mechanisms = content.split(' ')
511
+ raise NotImplementedError, "Dalli only supports the PLAIN authentication mechanism" if !mechanisms.include?('PLAIN')
512
+
513
+ # request
514
+ mechanism = 'PLAIN'
515
+ msg = "\x0#{username}\x0#{password}"
516
+ req = [REQUEST, OPCODES[:auth_request], mechanism.bytesize, 0, 0, 0, mechanism.bytesize + msg.bytesize, 0, 0, mechanism, msg].pack(FORMAT[:auth_request])
517
+ write(req)
518
+
519
+ header = read(24)
520
+ raise Dalli::NetworkError, 'No response' if !header
521
+ (extras, type, status, count) = header.unpack(NORMAL_HEADER)
522
+ raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
523
+ content = read(count)
524
+ return Dalli.logger.info("Dalli/SASL: #{content}") if status == 0
525
+
526
+ raise Dalli::DalliError, "Error authenticating: #{status}" unless status == 0x21
527
+ raise NotImplementedError, "No two-step authentication mechanisms supported"
528
+ # (step, msg) = sasl.receive('challenge', content)
529
+ # raise Dalli::NetworkError, "Authentication failed" if sasl.failed? || step != 'response'
530
+ end
531
+ end
532
+ end