memcache 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/LICENSE +20 -0
- data/README.rdoc +179 -0
- data/Rakefile +56 -0
- data/VERSION.yml +4 -0
- data/ext/extconf.rb +3 -0
- data/lib/memcache/local_server.rb +107 -0
- data/lib/memcache/migration.rb +23 -0
- data/lib/memcache/null_server.rb +30 -0
- data/lib/memcache/pg_server.rb +163 -0
- data/lib/memcache/segmented_server.rb +97 -0
- data/lib/memcache/server.rb +265 -0
- data/lib/memcache.rb +382 -0
- data/memcache.gemspec +69 -0
- data/test/memcache_local_server_test.rb +11 -0
- data/test/memcache_null_server_test.rb +65 -0
- data/test/memcache_pg_server_test.rb +28 -0
- data/test/memcache_segmented_server_test.rb +21 -0
- data/test/memcache_server_test.rb +35 -0
- data/test/memcache_server_test_helper.rb +159 -0
- data/test/memcache_test.rb +233 -0
- data/test/test_helper.rb +26 -0
- metadata +83 -0
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
pkg
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Justin Balthrop
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,179 @@
|
|
1
|
+
= memcache
|
2
|
+
|
3
|
+
This is the Geni memcached client. It started out as a fork of fiveruns/memcache-client,
|
4
|
+
which was a fork of seattle.rb's memcache-client, but over time, our client has diverged,
|
5
|
+
and I've rewritten most of the code. Of course, a lot of credit is due to those whose code
|
6
|
+
served as a starting point for this code.
|
7
|
+
|
8
|
+
== Usage
|
9
|
+
|
10
|
+
cache = Memcache.new(:server => "localhost:11211")
|
11
|
+
cache.set('stuff', [:symbol, 'String', 1, {:bar => 5}])
|
12
|
+
cache.get('stuff')
|
13
|
+
=> [:symbol, "String", 1, {:bar => 5}]
|
14
|
+
|
15
|
+
cache['things'] = {:foo => '1', :bar => [1,2,3]}
|
16
|
+
cache['things']
|
17
|
+
=> {:foo => "1", :bar => [1,2,3]}
|
18
|
+
|
19
|
+
== How is this different from memcache-client?
|
20
|
+
|
21
|
+
Like memcache-client, _memcache_ (shown in italics when I am referring to this
|
22
|
+
library) is a memcached client, but it differs significantly from memcache-client in
|
23
|
+
several important ways.
|
24
|
+
|
25
|
+
=== Interface
|
26
|
+
|
27
|
+
I tried to keep the basic interface as similar as I could to memcache-client. In some
|
28
|
+
cases, _memcache_ can be a near drop-in replacement for memcache-client. However, I did
|
29
|
+
rename the main class from +MemCache+ to +Memcache+ to prevent confusion and to force
|
30
|
+
those switching to _memcache_ to update their code. Here are the notable interface
|
31
|
+
changes:
|
32
|
+
|
33
|
+
- +expiry+ and +raw+ are specified as options in a hash now, instead of as unnamed parameters.
|
34
|
+
|
35
|
+
cache.set('foo', :a, :expiry => 10.minutes)
|
36
|
+
cache.set('bar', :b, :expiry => Time.parse('5:51pm Nov 24, 2018'))
|
37
|
+
cache.set('baz', 'c', :expiry => 30.minutes, :raw => true)
|
38
|
+
|
39
|
+
- +get_multi+ has been replaced by a more versatile +get+ interface. If the first argument is
|
40
|
+
an array, then a hash of key/value pairs is returned. If the first argument is not an
|
41
|
+
array, then the value alone is returned.
|
42
|
+
|
43
|
+
cache.get('foo') # => :a
|
44
|
+
cache.get(['foo', 'bar']) # => {"foo"=>:a, "bar"=>:b}
|
45
|
+
cache.get(['foo']) # => {"foo"=>:a}
|
46
|
+
|
47
|
+
- +get+ also supports updating the expiry for a single key. this can be used to keep
|
48
|
+
frequently accessed data in cache longer than less accessed data, though usually the
|
49
|
+
memcached LRU algorithm will be sufficient.
|
50
|
+
|
51
|
+
cache.get('foo', :expiry => 1.day)
|
52
|
+
|
53
|
+
- Support for flags has been added to all methods. So you can store additional metadata on
|
54
|
+
each value. Depending on which server version you are using, flags can be 16 bit or 32
|
55
|
+
bit unsigned integers (though it seems that memcache 1.4.1 returns signed values if the
|
56
|
+
upper bit is set).
|
57
|
+
|
58
|
+
cache.set('foo', :aquatic, :flags => 0b11101111)
|
59
|
+
value = cache.get('foo')
|
60
|
+
=> :aquatic
|
61
|
+
value.memcache_flags.to_s(2)
|
62
|
+
=> "11101111"
|
63
|
+
|
64
|
+
cache.set('foo', 'aquatic', :raw => true, :flags => 0xff08)
|
65
|
+
cache.get('foo', :raw => true).memcache_flags.to_s(2)
|
66
|
+
=> "1111111100001000"
|
67
|
+
|
68
|
+
- +incr+ and +decr+ automatically initialize the value to 0 if the key doesn't
|
69
|
+
exist. The +count+ method returns the integer count associated with a given key.
|
70
|
+
|
71
|
+
cache.count('hits') # => 0
|
72
|
+
cache.incr('hits', 52) # => 52
|
73
|
+
cache.decr('hits', 9) # => 43
|
74
|
+
cache.count('hits') # => 43
|
75
|
+
|
76
|
+
- In addition to +add+, which was already supported, support has been added for +replace+,
|
77
|
+
+append+ and +prepend+ from the memcached protocol.
|
78
|
+
|
79
|
+
cache.add('foo', 1)
|
80
|
+
cache.add('foo', 0)
|
81
|
+
cache.get('foo')
|
82
|
+
=> 1
|
83
|
+
|
84
|
+
cache.replace('foo', 2)
|
85
|
+
cache.get('foo')
|
86
|
+
=> 2
|
87
|
+
|
88
|
+
cache.write('foo', 'bar') ## shortcut for cache.set('foo', 'bar', :raw => true)
|
89
|
+
cache.append('foo', 'none') ## append and prepend only works on raw values
|
90
|
+
cache.prepend('foo', 'foo') ##
|
91
|
+
cache.read('foo') ## shortcut for cache.get('foo', :raw => true)
|
92
|
+
=> "foobarnone"
|
93
|
+
|
94
|
+
- Support has also been added for +cas+ (compare-and-set).
|
95
|
+
|
96
|
+
value = cache.get('foo', :cas => true)
|
97
|
+
cache.cas('foo', value.upcase, :cas => value.memcache_cas)
|
98
|
+
cache.get('foo')
|
99
|
+
=> "FOOBARNONE"
|
100
|
+
|
101
|
+
value = cache.get('foo', :cas => true)
|
102
|
+
cache.set('foo', 'modified')
|
103
|
+
cache.cas('foo', value.downcase, :cas => value.memcache_cas)
|
104
|
+
cache.get('foo')
|
105
|
+
=> "modified"
|
106
|
+
|
107
|
+
- Several additional convenience methods have been added including +get_or_add+,
|
108
|
+
+get_or_set+, +update+, +get_some+, +lock+, +unlock+, and +with_lock+.
|
109
|
+
|
110
|
+
=== Implementation
|
111
|
+
|
112
|
+
The underlying architechture of _memcache_ is more modular than memcache-client.
|
113
|
+
A given +Memcache+ instance has a group of servers, just like before, but much more of the
|
114
|
+
functionality in encapsulated inside the <tt>Memcache::Server</tt> object. Really, a +Server+
|
115
|
+
object is a thin wrapper around an remote memcached server that takes care of the socket
|
116
|
+
and protocol details along with basic error handling. The +Memcache+ class handles the
|
117
|
+
partitioning algorithm, marshaling of ruby objects and various higher-level methods.
|
118
|
+
|
119
|
+
By encapsulating the protocol inside the +Server+ object, it becomes very easy to plug-in
|
120
|
+
alternate server implementations. Right now, there are two basic, alternate servers:
|
121
|
+
|
122
|
+
[+LocalServer+] This is an in-process server for storing keys and values in local
|
123
|
+
memory. It is good for testing, when you don't want to spin up an instance
|
124
|
+
of memcached, and also as a second level of caching. For example, in a web
|
125
|
+
application, you can use this as a quick cache which lasts for the
|
126
|
+
duration of a request.
|
127
|
+
|
128
|
+
[+PGServer+] This is an implementation of memcached functionality using SQL. It stores all
|
129
|
+
data in a single postgres table and uses +PGConn+ to select and update this
|
130
|
+
table. This works well as a permanent cache or in the case when your objects
|
131
|
+
are very large. It can also be used in a multi-level cache setup with
|
132
|
+
<tt>Memcache::Server</tt> to provide persistence without sacrificing speed.
|
133
|
+
|
134
|
+
=== Very Large Values
|
135
|
+
|
136
|
+
Memcached limits the size of values to 1MB. This is done to reduce memory usage, but it
|
137
|
+
means that large data structures, which are also often costly to compute, cannot be stored
|
138
|
+
easily. We solve this problem by providing an additional server called
|
139
|
+
<tt>Memcache::SegmentedServer</tt>. It inherits from <tt>Memcache::Server</tt>, but
|
140
|
+
includes code to segment and reassemble large values. Mike Stangel at Geni originally
|
141
|
+
wrote this code as an extension to memcache-client and I adapted it for the new
|
142
|
+
architecture.
|
143
|
+
|
144
|
+
You can use segmented values either by passing +SegmentedServer+ objects to +Memcache+, or
|
145
|
+
you can use the +segment_large_values+ option.
|
146
|
+
|
147
|
+
server = Memcache::SegmentedServer.new(:host => 'localhost', :port => 11211)
|
148
|
+
cache = Memcache.new(:server => server)
|
149
|
+
|
150
|
+
cache = Memcache.new(:server => 'localhost:11211', :segment_large_values => true)
|
151
|
+
|
152
|
+
=== Error Handling and Recovery
|
153
|
+
|
154
|
+
We handle errors differently in _memcache_ than memcache-client does. Whenever there is a
|
155
|
+
connection error or other fatal error, memcache-client marks the offending server as dead
|
156
|
+
for 30 seconds, and all calls that require that server fail for the next 30 seconds. This
|
157
|
+
was unacceptable for us in a production environment. We tried changing the retry timeout
|
158
|
+
to 1 second, but still found our exception logs filling up with failed web requests
|
159
|
+
whenever a network connection was broken.
|
160
|
+
|
161
|
+
So, the default behavior in _memcache_ is for reads to be stable even if the underlying
|
162
|
+
server is unavailable. This means, that instead of raising an exception, a read will just
|
163
|
+
return nil if the server is down. Of course, you need to monitor your memcached servers to
|
164
|
+
make sure they aren't down for long, but this allows your site to be resilient to minor
|
165
|
+
network blips. Any error that occurs while unmarshalling a stored object will also return nil.
|
166
|
+
|
167
|
+
Writes, on the other hand, cannot just be ignored when the server is down. For this reason,
|
168
|
+
every write operation is retried once by closing and reopening the connection before
|
169
|
+
finally marking a server as dead and raising an exception. We will not attempt to read
|
170
|
+
from a dead server for 5 seconds, but a write will always attempt to revive a dead server
|
171
|
+
by attempting to connect.
|
172
|
+
|
173
|
+
== Installation
|
174
|
+
|
175
|
+
$ sudo gem install memcache --source http://gemcutter.org
|
176
|
+
|
177
|
+
== License:
|
178
|
+
|
179
|
+
Copyright (c) 2009 Justin Balthrop, Geni.com; Published under The MIT License, see LICENSE
|
data/Rakefile
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'jeweler'
|
6
|
+
Jeweler::Tasks.new do |gem|
|
7
|
+
gem.name = "memcache"
|
8
|
+
gem.summary = %Q{Advanced ruby memcache client}
|
9
|
+
gem.description = %Q{Ruby client for memcached supporting advanced protocol features and pluggable architecture.}
|
10
|
+
gem.email = "code@justinbalthrop.com"
|
11
|
+
gem.homepage = "http://github.com/ninjudd/memcache"
|
12
|
+
gem.authors = ["Justin Balthrop"]
|
13
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
14
|
+
end
|
15
|
+
Jeweler::GemcutterTasks.new
|
16
|
+
rescue LoadError
|
17
|
+
puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
|
18
|
+
end
|
19
|
+
|
20
|
+
require 'rake/testtask'
|
21
|
+
Rake::TestTask.new(:test) do |test|
|
22
|
+
test.libs << 'lib' << 'test'
|
23
|
+
test.pattern = 'test/**/*_test.rb'
|
24
|
+
test.verbose = true
|
25
|
+
end
|
26
|
+
|
27
|
+
begin
|
28
|
+
require 'rcov/rcovtask'
|
29
|
+
Rcov::RcovTask.new do |test|
|
30
|
+
test.libs << 'test'
|
31
|
+
test.pattern = 'test/**/*_test.rb'
|
32
|
+
test.verbose = true
|
33
|
+
end
|
34
|
+
rescue LoadError
|
35
|
+
task :rcov do
|
36
|
+
abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
task :test => :check_dependencies
|
41
|
+
|
42
|
+
task :default => :test
|
43
|
+
|
44
|
+
require 'rake/rdoctask'
|
45
|
+
Rake::RDocTask.new do |rdoc|
|
46
|
+
if File.exist?('VERSION')
|
47
|
+
version = File.read('VERSION')
|
48
|
+
else
|
49
|
+
version = ""
|
50
|
+
end
|
51
|
+
|
52
|
+
rdoc.rdoc_dir = 'rdoc'
|
53
|
+
rdoc.title = "memcache #{version}"
|
54
|
+
rdoc.rdoc_files.include('README*')
|
55
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
56
|
+
end
|
data/VERSION.yml
ADDED
data/ext/extconf.rb
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
class Memcache
|
2
|
+
class LocalServer
|
3
|
+
def initialize
|
4
|
+
@data = {}
|
5
|
+
@expiry = {}
|
6
|
+
end
|
7
|
+
|
8
|
+
def name
|
9
|
+
"local:#{hash}"
|
10
|
+
end
|
11
|
+
|
12
|
+
def stats
|
13
|
+
{ # curr_items may include items that have expired.
|
14
|
+
'curr_items' => @data.size,
|
15
|
+
'expiry_count' => @expiry.size,
|
16
|
+
}
|
17
|
+
end
|
18
|
+
|
19
|
+
def flush_all(delay = 0)
|
20
|
+
raise 'flush_all not supported with delay' if delay != 0
|
21
|
+
@data.clear
|
22
|
+
@expiry.clear
|
23
|
+
end
|
24
|
+
|
25
|
+
def gets(keys)
|
26
|
+
get(keys, true)
|
27
|
+
end
|
28
|
+
|
29
|
+
def get(keys, cas = false)
|
30
|
+
if keys.kind_of?(Array)
|
31
|
+
hash = {}
|
32
|
+
keys.each do |key|
|
33
|
+
key = key.to_s
|
34
|
+
val = get(key)
|
35
|
+
hash[key] = val if val
|
36
|
+
end
|
37
|
+
hash
|
38
|
+
else
|
39
|
+
key = keys.to_s
|
40
|
+
if @expiry[key] and Time.now > @expiry[key]
|
41
|
+
@data[key] = nil
|
42
|
+
@expiry[key] = nil
|
43
|
+
end
|
44
|
+
@data[key]
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def incr(key, amount = 1)
|
49
|
+
key = key.to_s
|
50
|
+
value = get(key)
|
51
|
+
return unless value
|
52
|
+
return unless value =~ /^\d+$/
|
53
|
+
|
54
|
+
value = value.to_i + amount
|
55
|
+
value = 0 if value < 0
|
56
|
+
@data[key] = value.to_s
|
57
|
+
value
|
58
|
+
end
|
59
|
+
|
60
|
+
def decr(key, amount = 1)
|
61
|
+
incr(key, -amount)
|
62
|
+
end
|
63
|
+
|
64
|
+
def delete(key)
|
65
|
+
@data.delete(key.to_s)
|
66
|
+
end
|
67
|
+
|
68
|
+
def set(key, value, expiry = 0, flags = 0)
|
69
|
+
key = key.to_s
|
70
|
+
@data[key] = value.to_s
|
71
|
+
if expiry.kind_of?(Time)
|
72
|
+
@expiry[key] = expiry
|
73
|
+
else
|
74
|
+
expiry = expiry.to_i
|
75
|
+
@expiry[key] = expiry == 0 ? nil : Time.now + expiry
|
76
|
+
end
|
77
|
+
value
|
78
|
+
end
|
79
|
+
|
80
|
+
def cas(key, value, cas, expiry = 0, flags = 0)
|
81
|
+
# No cas implementation yet, just do a set for now.
|
82
|
+
set(key, value, expiry, flags)
|
83
|
+
end
|
84
|
+
|
85
|
+
def add(key, value, expiry = 0, flags = 0)
|
86
|
+
return nil if get(key)
|
87
|
+
set(key, value, expiry)
|
88
|
+
end
|
89
|
+
|
90
|
+
def replace(key, value, expiry = 0, flags = 0)
|
91
|
+
return nil if get(key).nil?
|
92
|
+
set(key, value, expiry)
|
93
|
+
end
|
94
|
+
|
95
|
+
def append(key, value)
|
96
|
+
existing = get(key)
|
97
|
+
return nil if existing.nil?
|
98
|
+
set(key, existing + value)
|
99
|
+
end
|
100
|
+
|
101
|
+
def prepend(key, value)
|
102
|
+
existing = get(key)
|
103
|
+
return nil if existing.nil?
|
104
|
+
set(key, value + existing)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
class Memcache
|
2
|
+
class Migration < ActiveRecord::Migration
|
3
|
+
class << self
|
4
|
+
attr_accessor :table
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.up
|
8
|
+
create_table table, :id => false do |t|
|
9
|
+
t.string :key
|
10
|
+
t.text :value
|
11
|
+
t.timestamp :expires_at
|
12
|
+
t.timestamp :updated_at
|
13
|
+
end
|
14
|
+
|
15
|
+
add_index table, [:key], :unique => true
|
16
|
+
add_index table, [:expires_at]
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.down
|
20
|
+
drop_table table
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
class Memcache
|
2
|
+
class NullServer
|
3
|
+
def name
|
4
|
+
"null"
|
5
|
+
end
|
6
|
+
|
7
|
+
def flush_all(delay = nil)
|
8
|
+
end
|
9
|
+
|
10
|
+
def get(keys)
|
11
|
+
keys.kind_of?(Array) ? {} : nil
|
12
|
+
end
|
13
|
+
|
14
|
+
def incr(key, amount = nil)
|
15
|
+
nil
|
16
|
+
end
|
17
|
+
|
18
|
+
def delete(key, expiry = nil)
|
19
|
+
nil
|
20
|
+
end
|
21
|
+
|
22
|
+
def set(key, value, expiry = nil)
|
23
|
+
nil
|
24
|
+
end
|
25
|
+
|
26
|
+
def add(key, value, expiry = nil)
|
27
|
+
nil
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,163 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'memcache/migration'
|
3
|
+
|
4
|
+
class Memcache
|
5
|
+
class PGServer
|
6
|
+
attr_reader :db, :table
|
7
|
+
|
8
|
+
def initialize(opts)
|
9
|
+
@table = opts[:table]
|
10
|
+
@db = opts[:db] || ActiveRecord::Base.connection.raw_connection
|
11
|
+
end
|
12
|
+
|
13
|
+
def name
|
14
|
+
@name ||= begin
|
15
|
+
db_config = db.instance_variable_get(:@config)
|
16
|
+
"#{db_config[:host]}:#{db_config[:database]}:#{table}"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def flush_all(delay = nil)
|
21
|
+
db.exec("TRUNCATE #{table}")
|
22
|
+
end
|
23
|
+
|
24
|
+
def get(keys)
|
25
|
+
return get([keys])[keys.to_s] unless keys.kind_of?(Array)
|
26
|
+
|
27
|
+
keys = keys.collect {|key| quote(key.to_s)}.join(',')
|
28
|
+
sql = %{
|
29
|
+
SELECT key, value FROM #{table}
|
30
|
+
WHERE key IN (#{keys}) AND #{expiry_clause}
|
31
|
+
}
|
32
|
+
results = {}
|
33
|
+
db.query(sql).each do |key, value|
|
34
|
+
results[key] = value
|
35
|
+
end
|
36
|
+
results
|
37
|
+
end
|
38
|
+
|
39
|
+
def incr(key, amount = 1)
|
40
|
+
transaction do
|
41
|
+
value = get(key)
|
42
|
+
return unless value
|
43
|
+
return unless value =~ /^\d+$/
|
44
|
+
|
45
|
+
value = value.to_i + amount
|
46
|
+
value = 0 if value < 0
|
47
|
+
db.exec %{
|
48
|
+
UPDATE #{table} SET value = #{quote(value)}, updated_at = NOW()
|
49
|
+
WHERE key = #{quote(key)}
|
50
|
+
}
|
51
|
+
value
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def decr(key, amount = 1)
|
56
|
+
incr(key, -amount)
|
57
|
+
end
|
58
|
+
|
59
|
+
def delete(key)
|
60
|
+
result = db.exec %{
|
61
|
+
DELETE FROM #{table}
|
62
|
+
WHERE key = #{quote(key)}
|
63
|
+
}
|
64
|
+
end
|
65
|
+
|
66
|
+
def set(key, value, expiry = 0)
|
67
|
+
transaction do
|
68
|
+
delete(key)
|
69
|
+
insert(key, value, expiry)
|
70
|
+
end
|
71
|
+
value
|
72
|
+
end
|
73
|
+
|
74
|
+
def add(key, value, expiry = 0)
|
75
|
+
delete_expired(key)
|
76
|
+
insert(key, value, expiry)
|
77
|
+
value
|
78
|
+
rescue PGError => e
|
79
|
+
nil
|
80
|
+
end
|
81
|
+
|
82
|
+
def replace(key, value, expiry = 0)
|
83
|
+
delete_expired(key)
|
84
|
+
result = update(key, value, expiry)
|
85
|
+
result.cmdtuples == 1 ? value : nil
|
86
|
+
end
|
87
|
+
|
88
|
+
def append(key, value)
|
89
|
+
delete_expired(key)
|
90
|
+
result = db.exec %{
|
91
|
+
UPDATE #{table}
|
92
|
+
SET value = value || #{quote(value)}, updated_at = NOW()
|
93
|
+
WHERE key = #{quote(key)}
|
94
|
+
}
|
95
|
+
result.cmdtuples == 1
|
96
|
+
end
|
97
|
+
|
98
|
+
def prepend(key, value)
|
99
|
+
delete_expired(key)
|
100
|
+
result = db.exec %{
|
101
|
+
UPDATE #{table}
|
102
|
+
SET value = #{quote(value)} || value, updated_at = NOW()
|
103
|
+
WHERE key = #{quote(key)}
|
104
|
+
}
|
105
|
+
result.cmdtuples == 1
|
106
|
+
end
|
107
|
+
|
108
|
+
private
|
109
|
+
|
110
|
+
def insert(key, value, expiry = 0)
|
111
|
+
db.exec %{
|
112
|
+
INSERT INTO #{table} (key, value, updated_at, expires_at)
|
113
|
+
VALUES (#{quote(key)}, #{quote(value)}, NOW(), #{expiry_sql(expiry)})
|
114
|
+
}
|
115
|
+
end
|
116
|
+
|
117
|
+
def update(key, value, expiry = 0)
|
118
|
+
db.exec %{
|
119
|
+
UPDATE #{table}
|
120
|
+
SET value = #{quote(value)}, updated_at = NOW(), expires_at = #{expiry_sql(expiry)}
|
121
|
+
WHERE key = #{quote(key)}
|
122
|
+
}
|
123
|
+
end
|
124
|
+
|
125
|
+
def transaction
|
126
|
+
return yield if @in_transaction
|
127
|
+
|
128
|
+
begin
|
129
|
+
@in_transaction = true
|
130
|
+
db.exec('BEGIN')
|
131
|
+
value = yield
|
132
|
+
db.exec('COMMIT')
|
133
|
+
value
|
134
|
+
rescue Exception => e
|
135
|
+
db.exec('ROLLBACK')
|
136
|
+
raise e
|
137
|
+
ensure
|
138
|
+
@in_transaction = false
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def quote(string)
|
143
|
+
string.to_s.gsub(/'/,"\'")
|
144
|
+
"'#{string}'"
|
145
|
+
end
|
146
|
+
|
147
|
+
def delete_expired(key)
|
148
|
+
db.exec "DELETE FROM #{table} WHERE key = #{quote(key)} AND NOT (#{expiry_clause})"
|
149
|
+
end
|
150
|
+
|
151
|
+
def expiry_clause
|
152
|
+
"expires_at IS NULL OR expires_at > NOW()"
|
153
|
+
end
|
154
|
+
|
155
|
+
def expiry_sql(expiry)
|
156
|
+
if expiry.kind_of?(Time)
|
157
|
+
quote(expiry.to_s(:db))
|
158
|
+
else
|
159
|
+
expiry == 0 ? 'NULL' : "NOW() + interval '#{expiry} seconds'"
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
require 'digest/sha1'
|
2
|
+
|
3
|
+
class Memcache
|
4
|
+
class SegmentedServer < Server
|
5
|
+
MAX_SIZE = 1000000 # bytes
|
6
|
+
PARTIAL_VALUE = 0x40000000
|
7
|
+
|
8
|
+
def get(keys, opts = {})
|
9
|
+
return get([keys], opts)[keys.to_s] unless keys.kind_of?(Array)
|
10
|
+
return {} if keys.empty?
|
11
|
+
|
12
|
+
results = super
|
13
|
+
keys = {}
|
14
|
+
keys_to_fetch = []
|
15
|
+
results.each do |key, value|
|
16
|
+
next unless segmented?(value)
|
17
|
+
hash, num = value.split(':')
|
18
|
+
keys[key] = []
|
19
|
+
num.to_i.times do |i|
|
20
|
+
hash_key = "#{hash}:#{i}"
|
21
|
+
keys_to_fetch << hash_key
|
22
|
+
keys[key] << hash_key
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
parts = super(keys_to_fetch)
|
27
|
+
keys.each do |key, hashes|
|
28
|
+
value = ''
|
29
|
+
hashes.each do |hash_key|
|
30
|
+
if part = parts[hash_key]
|
31
|
+
value << part
|
32
|
+
else
|
33
|
+
value = nil
|
34
|
+
break
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
if value
|
39
|
+
value.memcache_cas = results[key].memcache_cas
|
40
|
+
value.memcache_flags = results[key].memcache_flags ^ PARTIAL_VALUE
|
41
|
+
results[key] = value
|
42
|
+
end
|
43
|
+
end
|
44
|
+
results
|
45
|
+
end
|
46
|
+
|
47
|
+
def set(key, value, expiry = 0, flags = 0)
|
48
|
+
value, flags = store_segments(key, value, expiry, flags)
|
49
|
+
super(key, value, expiry, flags) && value
|
50
|
+
end
|
51
|
+
|
52
|
+
def cas(key, value, cas, expiry = 0, flags = 0)
|
53
|
+
value, flags = store_segments(key, value, expiry, flags)
|
54
|
+
super(key, value, cas, expiry, flags)
|
55
|
+
end
|
56
|
+
|
57
|
+
def add(key, value, expiry = 0, flags = 0)
|
58
|
+
value, flags = store_segments(key, value, expiry, flags)
|
59
|
+
super(key, value, expiry, flags)
|
60
|
+
end
|
61
|
+
|
62
|
+
def replace(key, value, expiry = 0, flags = 0)
|
63
|
+
value, flags = store_segments(key, value, expiry, flags)
|
64
|
+
super(key, value, expiry, flags)
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def segmented?(value)
|
70
|
+
value.memcache_flags & PARTIAL_VALUE == PARTIAL_VALUE
|
71
|
+
end
|
72
|
+
|
73
|
+
def segment(key, value)
|
74
|
+
hash = Digest::SHA1.hexdigest("#{key}:#{Time.now}:#{rand}")
|
75
|
+
parts = {}
|
76
|
+
i = 0; offset = 0
|
77
|
+
while offset < value.size
|
78
|
+
parts["#{hash}:#{i}"] = value[offset, MAX_SIZE]
|
79
|
+
offset += MAX_SIZE; i += 1
|
80
|
+
end
|
81
|
+
master_key = "#{hash}:#{parts.size}"
|
82
|
+
[master_key, parts]
|
83
|
+
end
|
84
|
+
|
85
|
+
def store_segments(key, value, expiry = 0, flags = 0)
|
86
|
+
if value and value.size > MAX_SIZE
|
87
|
+
master_key, parts = segment(key, value)
|
88
|
+
parts.each do |hash, data|
|
89
|
+
set(hash, data, expiry)
|
90
|
+
end
|
91
|
+
[master_key, flags | PARTIAL_VALUE]
|
92
|
+
else
|
93
|
+
[value, flags]
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|