riak-client 1.0.5 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.gitignore +4 -3
- data/.rspec +1 -0
- data/Gemfile +1 -0
- data/RELEASE_NOTES.md +47 -0
- data/Rakefile +0 -1
- data/erl_src/riak_kv_test_backend.erl +34 -0
- data/lib/riak/client.rb +3 -1
- data/lib/riak/client/beefcake/messages.rb +49 -1
- data/lib/riak/client/beefcake/object_methods.rb +14 -21
- data/lib/riak/client/beefcake_protobuffs_backend.rb +58 -12
- data/lib/riak/client/decaying.rb +31 -23
- data/lib/riak/client/feature_detection.rb +88 -0
- data/lib/riak/client/http_backend.rb +27 -6
- data/lib/riak/client/http_backend/configuration.rb +13 -0
- data/lib/riak/client/http_backend/object_methods.rb +33 -25
- data/lib/riak/client/node.rb +7 -2
- data/lib/riak/client/protobuffs_backend.rb +54 -3
- data/lib/riak/client/search.rb +2 -2
- data/lib/riak/conflict.rb +13 -0
- data/lib/riak/locale/en.yml +2 -0
- data/lib/riak/map_reduce.rb +1 -1
- data/lib/riak/map_reduce/filter_builder.rb +2 -2
- data/lib/riak/map_reduce/results.rb +49 -0
- data/lib/riak/node/console.rb +17 -16
- data/lib/riak/node/generation.rb +9 -0
- data/lib/riak/rcontent.rb +168 -0
- data/lib/riak/robject.rb +37 -157
- data/lib/riak/util/escape.rb +5 -1
- data/lib/riak/version.rb +1 -1
- data/riak-client.gemspec +37 -5
- data/spec/fixtures/multipart-basic-conflict.txt +15 -0
- data/spec/fixtures/munchausen.txt +1033 -0
- data/spec/integration/riak/cluster_spec.rb +1 -1
- data/spec/integration/riak/http_backends_spec.rb +23 -2
- data/spec/integration/riak/node_spec.rb +2 -2
- data/spec/integration/riak/protobuffs_backends_spec.rb +17 -2
- data/spec/integration/riak/test_server_spec.rb +1 -1
- data/spec/integration/riak/threading_spec.rb +3 -3
- data/spec/riak/beefcake_protobuffs_backend_spec.rb +58 -25
- data/spec/riak/escape_spec.rb +3 -0
- data/spec/riak/feature_detection_spec.rb +61 -0
- data/spec/riak/http_backend/object_methods_spec.rb +4 -13
- data/spec/riak/http_backend_spec.rb +6 -5
- data/spec/riak/map_reduce_spec.rb +0 -5
- data/spec/riak/robject_spec.rb +12 -11
- data/spec/spec_helper.rb +3 -1
- data/spec/support/riak_test.rb +77 -0
- data/spec/support/search_corpus_setup.rb +18 -0
- data/spec/support/sometimes.rb +1 -1
- data/spec/support/test_server.rb +1 -1
- data/spec/support/unified_backend_examples.rb +53 -7
- data/spec/support/version_filter.rb +4 -11
- metadata +56 -22
- data/lib/riak/client/pool.rb +0 -180
- data/spec/riak/pool_spec.rb +0 -306
data/lib/riak/client/pool.rb
DELETED
@@ -1,180 +0,0 @@
|
|
1
|
-
require 'thread'
|
2
|
-
|
3
|
-
module Riak
|
4
|
-
class Client
|
5
|
-
# A re-entrant thread-safe resource pool. Generates new resources on
|
6
|
-
# demand.
|
7
|
-
# @private
|
8
|
-
class Pool
|
9
|
-
# Raised when a taken element should be deleted from the pool.
|
10
|
-
class BadResource < RuntimeError
|
11
|
-
end
|
12
|
-
|
13
|
-
# An element of the pool. Comprises an object with an owning
|
14
|
-
# thread.
|
15
|
-
# @private
|
16
|
-
class Element
|
17
|
-
attr_accessor :object
|
18
|
-
attr_accessor :owner
|
19
|
-
def initialize(object)
|
20
|
-
@object = object
|
21
|
-
@owner = owner
|
22
|
-
end
|
23
|
-
|
24
|
-
# Claims this element of the pool for the current Thread.
|
25
|
-
def lock
|
26
|
-
self.owner = Thread.current
|
27
|
-
end
|
28
|
-
|
29
|
-
# Is this element locked/claimed?
|
30
|
-
def locked?
|
31
|
-
not owner.nil?
|
32
|
-
end
|
33
|
-
|
34
|
-
# Releases this element of the pool from the current Thread.
|
35
|
-
def unlock
|
36
|
-
self.owner = nil
|
37
|
-
end
|
38
|
-
|
39
|
-
# Is this element available for use?
|
40
|
-
def unlocked?
|
41
|
-
owner.nil?
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
attr_accessor :pool
|
46
|
-
attr_accessor :open
|
47
|
-
attr_accessor :close
|
48
|
-
|
49
|
-
# Open is a callable which returns a new object for the pool. Close is
|
50
|
-
# called with an object before it is freed.
|
51
|
-
def initialize(open, close)
|
52
|
-
@open = open
|
53
|
-
@close = close
|
54
|
-
@lock = Mutex.new
|
55
|
-
@iterator = Mutex.new
|
56
|
-
@element_released = ConditionVariable.new
|
57
|
-
@pool = Set.new
|
58
|
-
end
|
59
|
-
|
60
|
-
# On each element of the pool, calls close(element) and removes it.
|
61
|
-
# @private
|
62
|
-
def clear
|
63
|
-
each_element do |e|
|
64
|
-
delete_element e
|
65
|
-
end
|
66
|
-
end
|
67
|
-
alias :close :clear
|
68
|
-
|
69
|
-
# Deletes an element of the pool. Calls close on its object.
|
70
|
-
# Not intendend for external use.
|
71
|
-
def delete_element(e)
|
72
|
-
@close.call(e.object)
|
73
|
-
@lock.synchronize do
|
74
|
-
@pool.delete e
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
# Locks each element in turn and closes/deletes elements for which the
|
79
|
-
# object passes the block.
|
80
|
-
def delete_if
|
81
|
-
raise ArgumentError, "block required" unless block_given?
|
82
|
-
|
83
|
-
each_element do |e|
|
84
|
-
if yield e.object
|
85
|
-
delete_element e
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
# Acquire an element of the pool. Yields the object. If all
|
91
|
-
# elements are claimed, it will create another one.
|
92
|
-
# @yield [obj] a block that will perform some action with the
|
93
|
-
# element of the pool
|
94
|
-
# @yieldparam [Object] resource a resource managed by the pool.
|
95
|
-
# Locked for the duration of the block
|
96
|
-
# @param [callable] :filter a callable which receives objects and has
|
97
|
-
# the opportunity to reject each in turn.
|
98
|
-
# @param [Object] :default if no resources are available, use this object
|
99
|
-
# instead of calling #open.
|
100
|
-
# @private
|
101
|
-
def take(opts = {})
|
102
|
-
unless block_given?
|
103
|
-
raise ArgumentError, "block required"
|
104
|
-
end
|
105
|
-
|
106
|
-
r = nil
|
107
|
-
begin
|
108
|
-
e = nil
|
109
|
-
@lock.synchronize do
|
110
|
-
# Find an existing element.
|
111
|
-
if f = opts[:filter]
|
112
|
-
e = pool.find { |e| e.unlocked? and f.call(e.object) }
|
113
|
-
else
|
114
|
-
e = pool.find { |e| e.unlocked? }
|
115
|
-
end
|
116
|
-
|
117
|
-
unless e
|
118
|
-
# No objects were acceptable
|
119
|
-
resource = opts[:default] || @open.call
|
120
|
-
e = Element.new(resource)
|
121
|
-
pool << e
|
122
|
-
end
|
123
|
-
e.lock
|
124
|
-
end
|
125
|
-
|
126
|
-
r = yield e.object
|
127
|
-
rescue BadResource
|
128
|
-
delete_element e
|
129
|
-
raise
|
130
|
-
ensure
|
131
|
-
# Unlock
|
132
|
-
if e
|
133
|
-
e.unlock
|
134
|
-
@element_released.signal
|
135
|
-
end
|
136
|
-
end
|
137
|
-
r
|
138
|
-
end
|
139
|
-
alias >> take
|
140
|
-
|
141
|
-
# Iterate over a snapshot of the pool. Yielded objects are locked for the
|
142
|
-
# duration of the block. This may block the current thread until elements
|
143
|
-
# are released by other threads.
|
144
|
-
# @private
|
145
|
-
def each_element
|
146
|
-
targets = @pool.to_a
|
147
|
-
unlocked = []
|
148
|
-
|
149
|
-
@iterator.synchronize do
|
150
|
-
until targets.empty?
|
151
|
-
@lock.synchronize do
|
152
|
-
unlocked, targets = targets.partition {|e| e.unlocked? }
|
153
|
-
unlocked.each {|e| e.lock }
|
154
|
-
end
|
155
|
-
|
156
|
-
unlocked.each do |e|
|
157
|
-
begin
|
158
|
-
yield e
|
159
|
-
ensure
|
160
|
-
e.unlock
|
161
|
-
end
|
162
|
-
end
|
163
|
-
@element_released.wait(@iterator) unless targets.empty?
|
164
|
-
end
|
165
|
-
end
|
166
|
-
end
|
167
|
-
|
168
|
-
# As each_element, but yields objects, not wrapper elements.
|
169
|
-
def each
|
170
|
-
each_element do |e|
|
171
|
-
yield e.object
|
172
|
-
end
|
173
|
-
end
|
174
|
-
|
175
|
-
def size
|
176
|
-
@lock.synchronize { @pool.size }
|
177
|
-
end
|
178
|
-
end
|
179
|
-
end
|
180
|
-
end
|
data/spec/riak/pool_spec.rb
DELETED
@@ -1,306 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'thread'
|
3
|
-
|
4
|
-
describe Riak::Client::Pool do
|
5
|
-
describe 'basics' do
|
6
|
-
subject {
|
7
|
-
described_class.new(
|
8
|
-
lambda { [0] },
|
9
|
-
lambda { |x| }
|
10
|
-
)
|
11
|
-
}
|
12
|
-
|
13
|
-
it 'yields a new object' do
|
14
|
-
subject.take do |x|
|
15
|
-
x.should == [0]
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
it 'retains a single object for serial access' do
|
20
|
-
n = 100
|
21
|
-
n.times do |i|
|
22
|
-
subject.take do |x|
|
23
|
-
x.should == [i]
|
24
|
-
x[0] += 1
|
25
|
-
end
|
26
|
-
end
|
27
|
-
subject.size.should == 1
|
28
|
-
end
|
29
|
-
|
30
|
-
it 'should be re-entrant' do
|
31
|
-
n = 10
|
32
|
-
n.times do |i|
|
33
|
-
subject.take do |x|
|
34
|
-
x.replace [1]
|
35
|
-
subject.take do |y|
|
36
|
-
y.replace [2]
|
37
|
-
subject.take do |z|
|
38
|
-
z.replace [3]
|
39
|
-
subject.take do |t|
|
40
|
-
t.replace [4]
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
subject.pool.map { |e| e.object.first }.sort.should == [1,2,3,4]
|
47
|
-
end
|
48
|
-
|
49
|
-
|
50
|
-
it 'should unlock when exceptions are raised' do
|
51
|
-
begin
|
52
|
-
subject.take do |x|
|
53
|
-
x << 1
|
54
|
-
subject.take do |y|
|
55
|
-
x << 2
|
56
|
-
y << 3
|
57
|
-
raise
|
58
|
-
end
|
59
|
-
end
|
60
|
-
rescue
|
61
|
-
end
|
62
|
-
subject.pool.all? { |e| not e.owner }.should == true
|
63
|
-
subject.pool.map { |e| e.object }.to_set.should == [
|
64
|
-
[0,1,2],
|
65
|
-
[0,3]
|
66
|
-
].to_set
|
67
|
-
end
|
68
|
-
|
69
|
-
it 'should delete when BadResource is raised' do
|
70
|
-
subject.open = lambda do
|
71
|
-
m = mock('resource')
|
72
|
-
m.should_receive(:close)
|
73
|
-
m
|
74
|
-
end
|
75
|
-
subject.close = lambda do |res|
|
76
|
-
res.close
|
77
|
-
end
|
78
|
-
|
79
|
-
lambda do
|
80
|
-
subject.take do |x|
|
81
|
-
raise Riak::Client::Pool::BadResource
|
82
|
-
end
|
83
|
-
end.should raise_error(Riak::Client::Pool::BadResource)
|
84
|
-
subject.size.should == 0
|
85
|
-
end
|
86
|
-
end
|
87
|
-
|
88
|
-
describe 'threads' do
|
89
|
-
subject {
|
90
|
-
described_class.new(
|
91
|
-
lambda { [] },
|
92
|
-
lambda { |x| }
|
93
|
-
)
|
94
|
-
}
|
95
|
-
|
96
|
-
it 'should allocate n objects for n concurrent operations' do
|
97
|
-
# n threads concurrently allocate and sign objects from the pool
|
98
|
-
n = 10
|
99
|
-
readyq = Queue.new
|
100
|
-
finishq = Queue.new
|
101
|
-
threads = (0...n).map do
|
102
|
-
Thread.new do
|
103
|
-
subject.take do |x|
|
104
|
-
readyq << 1
|
105
|
-
x << Thread.current
|
106
|
-
finishq.pop
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
|
-
n.times { readyq.pop }
|
112
|
-
n.times { finishq << 1 }
|
113
|
-
|
114
|
-
# Wait for completion
|
115
|
-
threads.each do |t|
|
116
|
-
t.join
|
117
|
-
end
|
118
|
-
|
119
|
-
# Should have taken exactly n objects to do this
|
120
|
-
subject.size.should == n
|
121
|
-
# And each one should be signed exactly once
|
122
|
-
subject.pool.map do |e|
|
123
|
-
e.object.size.should == 1
|
124
|
-
e.object.first
|
125
|
-
end.to_set.should == threads.to_set
|
126
|
-
end
|
127
|
-
|
128
|
-
it 'take with filter and default' do
|
129
|
-
n = 10
|
130
|
-
subject = described_class.new(
|
131
|
-
lambda { [] },
|
132
|
-
lambda { |x| }
|
133
|
-
)
|
134
|
-
|
135
|
-
# Allocate several elements of the pool
|
136
|
-
q = Queue.new
|
137
|
-
threads = (0...n).map do |i|
|
138
|
-
Thread.new do
|
139
|
-
subject.take do |a|
|
140
|
-
a << i
|
141
|
-
q << 1
|
142
|
-
sleep 0.02
|
143
|
-
end
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
# Wait for all threads to have acquired an element
|
148
|
-
n.times { q.pop }
|
149
|
-
|
150
|
-
threads.each do |t|
|
151
|
-
t.join
|
152
|
-
end
|
153
|
-
|
154
|
-
# Get and delete existing even elements
|
155
|
-
got = Set.new
|
156
|
-
(n / 2).times do
|
157
|
-
begin
|
158
|
-
subject.take(
|
159
|
-
:filter => lambda { |x| x.first.even? },
|
160
|
-
:default => [:default]
|
161
|
-
) do |x|
|
162
|
-
got << x.first
|
163
|
-
raise Riak::Client::Pool::BadResource
|
164
|
-
end
|
165
|
-
rescue Riak::Client::Pool::BadResource
|
166
|
-
end
|
167
|
-
end
|
168
|
-
got.should == (0...n).select(&:even?).to_set
|
169
|
-
|
170
|
-
# This time, no even elements exist, so we should get the default.
|
171
|
-
subject.take(
|
172
|
-
:filter => lambda { |x| x.first.even? },
|
173
|
-
:default => :default
|
174
|
-
) do |x|
|
175
|
-
x.should == :default
|
176
|
-
end
|
177
|
-
end
|
178
|
-
|
179
|
-
it 'iterates over a snapshot of all connections, even ones in use' do
|
180
|
-
started = Queue.new
|
181
|
-
n = 30
|
182
|
-
threads = (0..n).map do
|
183
|
-
Thread.new do
|
184
|
-
psleep = 0.75 * rand # up to 50ms sleep
|
185
|
-
subject.take do |a|
|
186
|
-
started << 1
|
187
|
-
a << rand
|
188
|
-
sleep psleep
|
189
|
-
end
|
190
|
-
end
|
191
|
-
end
|
192
|
-
|
193
|
-
n.times { started.pop }
|
194
|
-
touched = []
|
195
|
-
|
196
|
-
subject.each do |e|
|
197
|
-
touched << e
|
198
|
-
end
|
199
|
-
|
200
|
-
threads.each do |t|
|
201
|
-
t.join
|
202
|
-
end
|
203
|
-
|
204
|
-
touched.should be_all {|item| subject.pool.find {|e| e.object == item } }
|
205
|
-
end
|
206
|
-
|
207
|
-
it 'should clear' do
|
208
|
-
n = 10
|
209
|
-
subject = described_class.new(
|
210
|
-
lambda { mock('connection').tap {|m| m.should_receive(:teardown) }},
|
211
|
-
lambda { |b| b.teardown }
|
212
|
-
)
|
213
|
-
|
214
|
-
# Allocate several elements of the pool
|
215
|
-
q = Queue.new
|
216
|
-
threads = (0...n).map do |i|
|
217
|
-
Thread.new do
|
218
|
-
subject.take do |a|
|
219
|
-
q << 1
|
220
|
-
sleep 0.1
|
221
|
-
end
|
222
|
-
end
|
223
|
-
end
|
224
|
-
|
225
|
-
# Wait for all threads to have acquired an element
|
226
|
-
n.times { q.pop }
|
227
|
-
|
228
|
-
# Clear the pool while threads still have elements checked out
|
229
|
-
subject.clear
|
230
|
-
subject.pool.should be_empty
|
231
|
-
|
232
|
-
# Wait for threads to complete
|
233
|
-
threads.each do |t|
|
234
|
-
t.join
|
235
|
-
end
|
236
|
-
end
|
237
|
-
|
238
|
-
it 'should delete_if' do
|
239
|
-
n = 10
|
240
|
-
subject = described_class.new(
|
241
|
-
lambda { [] },
|
242
|
-
lambda { |x| }
|
243
|
-
)
|
244
|
-
|
245
|
-
# Allocate several elements of the pool
|
246
|
-
q = Queue.new
|
247
|
-
threads = (0...n).map do |i|
|
248
|
-
Thread.new do
|
249
|
-
subject.take do |a|
|
250
|
-
a << i
|
251
|
-
q << 1
|
252
|
-
sleep 0.02
|
253
|
-
end
|
254
|
-
end
|
255
|
-
end
|
256
|
-
|
257
|
-
# Wait for all threads to have acquired an element
|
258
|
-
n.times { q.pop }
|
259
|
-
|
260
|
-
# Delete odd elements
|
261
|
-
subject.delete_if do |x|
|
262
|
-
x.first.odd?
|
263
|
-
end
|
264
|
-
|
265
|
-
# Verify odds are gone.
|
266
|
-
subject.pool.all? do |x|
|
267
|
-
x.object.first.even?
|
268
|
-
end.should == true
|
269
|
-
|
270
|
-
# Wait for threads
|
271
|
-
threads.each do |t|
|
272
|
-
t.join
|
273
|
-
end
|
274
|
-
end
|
275
|
-
|
276
|
-
it 'stress test', :slow => true do
|
277
|
-
n = 100
|
278
|
-
psleep = 0.8
|
279
|
-
tsleep = 0.01
|
280
|
-
rounds = 100
|
281
|
-
|
282
|
-
threads = (0...n).map do
|
283
|
-
Thread.new do
|
284
|
-
rounds.times do |i|
|
285
|
-
subject.take do |a|
|
286
|
-
a.should == []
|
287
|
-
a << Thread.current
|
288
|
-
a.should == [Thread.current]
|
289
|
-
|
290
|
-
# Sleep and check
|
291
|
-
while rand < psleep
|
292
|
-
sleep tsleep
|
293
|
-
a.should == [Thread.current]
|
294
|
-
end
|
295
|
-
|
296
|
-
a.delete Thread.current
|
297
|
-
end
|
298
|
-
end
|
299
|
-
end
|
300
|
-
end
|
301
|
-
threads.each do |t|
|
302
|
-
t.join
|
303
|
-
end
|
304
|
-
end
|
305
|
-
end
|
306
|
-
end
|