riak-client 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/Gemfile +1 -7
- data/README.markdown +66 -0
- data/RELEASE_NOTES.md +27 -0
- data/lib/riak/bucket.rb +24 -5
- data/lib/riak/client.rb +42 -7
- data/lib/riak/client/beefcake/message_codes.rb +56 -0
- data/lib/riak/client/beefcake/messages.rb +190 -18
- data/lib/riak/client/beefcake_protobuffs_backend.rb +143 -10
- data/lib/riak/client/feature_detection.rb +26 -1
- data/lib/riak/client/http_backend.rb +58 -9
- data/lib/riak/client/http_backend/bucket_streamer.rb +15 -0
- data/lib/riak/client/http_backend/chunked_json_streamer.rb +42 -0
- data/lib/riak/client/http_backend/configuration.rb +17 -1
- data/lib/riak/client/http_backend/key_streamer.rb +4 -32
- data/lib/riak/client/protobuffs_backend.rb +12 -34
- data/lib/riak/counter.rb +101 -0
- data/lib/riak/index_collection.rb +71 -0
- data/lib/riak/list_buckets.rb +28 -0
- data/lib/riak/locale/en.yml +14 -0
- data/lib/riak/multiget.rb +123 -0
- data/lib/riak/node.rb +2 -0
- data/lib/riak/node/configuration.rb +32 -21
- data/lib/riak/node/defaults.rb +2 -0
- data/lib/riak/node/generation.rb +19 -7
- data/lib/riak/node/version.rb +2 -16
- data/lib/riak/robject.rb +1 -0
- data/lib/riak/secondary_index.rb +67 -0
- data/lib/riak/version.rb +1 -1
- data/riak-client.gemspec +3 -2
- data/spec/integration/riak/counters_spec.rb +51 -0
- data/spec/integration/riak/http_backends_spec.rb +24 -14
- data/spec/integration/riak/node_spec.rb +6 -28
- data/spec/riak/beefcake_protobuffs_backend_spec.rb +84 -0
- data/spec/riak/bucket_spec.rb +55 -5
- data/spec/riak/client_spec.rb +34 -0
- data/spec/riak/counter_spec.rb +122 -0
- data/spec/riak/index_collection_spec.rb +50 -0
- data/spec/riak/list_buckets_spec.rb +41 -0
- data/spec/riak/multiget_spec.rb +76 -0
- data/spec/riak/robject_spec.rb +4 -1
- data/spec/riak/secondary_index_spec.rb +225 -0
- data/spec/spec_helper.rb +1 -0
- data/spec/support/sometimes.rb +2 -2
- data/spec/support/unified_backend_examples.rb +4 -0
- metadata +41 -47
@@ -77,11 +77,41 @@ module Riak
|
|
77
77
|
decode_response
|
78
78
|
end
|
79
79
|
|
80
|
+
def get_counter(bucket, key, options={})
|
81
|
+
bucket = bucket.name if bucket.is_a? Bucket
|
82
|
+
|
83
|
+
options = normalize_quorums(options)
|
84
|
+
options[:bucket] = bucket
|
85
|
+
options[:key] = key
|
86
|
+
|
87
|
+
request = RpbCounterGetReq.new options
|
88
|
+
write_protobuff :CounterGetReq, request
|
89
|
+
|
90
|
+
decode_response
|
91
|
+
end
|
92
|
+
|
93
|
+
def post_counter(bucket, key, amount, options={})
|
94
|
+
bucket = bucket.name if bucket.is_a? Bucket
|
95
|
+
|
96
|
+
options = normalize_quorums(options)
|
97
|
+
options[:bucket] = bucket
|
98
|
+
options[:key] = key
|
99
|
+
# TODO: raise if ammount doesn't fit in sint64
|
100
|
+
options[:amount] = amount
|
101
|
+
|
102
|
+
request = RpbCounterUpdateReq.new options
|
103
|
+
write_protobuff :CounterUpdateReq, request
|
104
|
+
|
105
|
+
decode_response
|
106
|
+
end
|
107
|
+
|
80
108
|
def get_bucket_props(bucket)
|
81
109
|
bucket = bucket.name if Bucket === bucket
|
82
110
|
req = RpbGetBucketReq.new(:bucket => maybe_encode(bucket))
|
83
111
|
write_protobuff(:GetBucketReq, req)
|
84
|
-
decode_response
|
112
|
+
resp = normalize_quorums decode_response
|
113
|
+
normalized = normalize_hooks resp
|
114
|
+
normalized.stringify_keys
|
85
115
|
end
|
86
116
|
|
87
117
|
def set_bucket_props(bucket, props)
|
@@ -92,9 +122,16 @@ module Riak
|
|
92
122
|
decode_response
|
93
123
|
end
|
94
124
|
|
95
|
-
def
|
125
|
+
def reset_bucket_props(bucket)
|
126
|
+
bucket = bucket.name if Bucket === bucket
|
127
|
+
req = RpbResetBucketReq.new(:bucket => maybe_encode(bucket))
|
128
|
+
write_protobuff(:ResetBucketReq)
|
129
|
+
decode_response
|
130
|
+
end
|
131
|
+
|
132
|
+
def list_keys(bucket, options={}, &block)
|
96
133
|
bucket = bucket.name if Bucket === bucket
|
97
|
-
req = RpbListKeysReq.new(:bucket => maybe_encode(bucket))
|
134
|
+
req = RpbListKeysReq.new(options.merge(:bucket => maybe_encode(bucket)))
|
98
135
|
write_protobuff(:ListKeysReq, req)
|
99
136
|
keys = []
|
100
137
|
while msg = decode_response
|
@@ -108,6 +145,21 @@ module Riak
|
|
108
145
|
block_given? || keys
|
109
146
|
end
|
110
147
|
|
148
|
+
# override the simple list_buckets
|
149
|
+
def list_buckets(options={}, &blk)
|
150
|
+
if block_given?
|
151
|
+
return streaming_list_buckets options, &blk
|
152
|
+
end
|
153
|
+
|
154
|
+
raise t("streaming_bucket_list_without_block") if options[:stream]
|
155
|
+
|
156
|
+
request = RpbListBucketsReq.new options
|
157
|
+
|
158
|
+
write_protobuff :ListBucketsReq, request
|
159
|
+
|
160
|
+
decode_response
|
161
|
+
end
|
162
|
+
|
111
163
|
def mapred(mr, &block)
|
112
164
|
raise MapReduceError.new(t("empty_map_reduce_query")) if mr.query.empty? && !mapred_phaseless?
|
113
165
|
req = RpbMapRedReq.new(:request => mr.to_json, :content_type => "application/json")
|
@@ -124,7 +176,7 @@ module Riak
|
|
124
176
|
block_given? || results.report
|
125
177
|
end
|
126
178
|
|
127
|
-
def get_index(bucket, index, query)
|
179
|
+
def get_index(bucket, index, query, query_options={}, &block)
|
128
180
|
return super unless pb_indexes?
|
129
181
|
bucket = bucket.name if Bucket === bucket
|
130
182
|
if Range === query
|
@@ -139,9 +191,14 @@ module Riak
|
|
139
191
|
:key => query.to_s
|
140
192
|
}
|
141
193
|
end
|
142
|
-
|
194
|
+
|
195
|
+
options.merge!(:bucket => bucket, :index => index)
|
196
|
+
options.merge!(query_options)
|
197
|
+
options[:stream] = block_given?
|
198
|
+
|
199
|
+
req = RpbIndexReq.new(options)
|
143
200
|
write_protobuff(:IndexReq, req)
|
144
|
-
|
201
|
+
decode_index_response(&block)
|
145
202
|
end
|
146
203
|
|
147
204
|
def search(index, query, options={})
|
@@ -166,12 +223,22 @@ module Riak
|
|
166
223
|
msglen, msgcode = header.unpack("NC")
|
167
224
|
if msglen == 1
|
168
225
|
case MESSAGE_CODES[msgcode]
|
169
|
-
when :PingResp,
|
226
|
+
when :PingResp,
|
227
|
+
:SetClientIdResp,
|
228
|
+
:PutResp,
|
229
|
+
:DelResp,
|
230
|
+
:SetBucketResp,
|
231
|
+
:ResetBucketResp
|
170
232
|
true
|
171
|
-
when :ListBucketsResp,
|
233
|
+
when :ListBucketsResp,
|
234
|
+
:ListKeysResp,
|
235
|
+
:IndexResp
|
172
236
|
[]
|
173
237
|
when :GetResp
|
174
238
|
raise Riak::ProtobuffsFailedRequest.new(:not_found, t('not_found'))
|
239
|
+
when :CounterGetResp,
|
240
|
+
:CounterUpdateResp
|
241
|
+
0
|
175
242
|
else
|
176
243
|
false
|
177
244
|
end
|
@@ -200,17 +267,25 @@ module Riak
|
|
200
267
|
RpbListKeysResp.decode(message)
|
201
268
|
when :GetBucketResp
|
202
269
|
res = RpbGetBucketResp.decode(message)
|
203
|
-
|
270
|
+
res.props.to_hash.stringify_keys
|
204
271
|
when :MapRedResp
|
205
272
|
RpbMapRedResp.decode(message)
|
206
273
|
when :IndexResp
|
207
274
|
res = RpbIndexResp.decode(message)
|
208
|
-
res
|
275
|
+
IndexCollection.new_from_protobuf res
|
209
276
|
when :SearchQueryResp
|
210
277
|
res = RpbSearchQueryResp.decode(message)
|
211
278
|
{ 'docs' => res.docs.map {|d| decode_doc(d) },
|
212
279
|
'max_score' => res.max_score,
|
213
280
|
'num_found' => res.num_found }
|
281
|
+
when :CSBucketResp
|
282
|
+
res = RpbCSBucketResp.decode message
|
283
|
+
when :CounterUpdateResp
|
284
|
+
res = RpbCounterUpdateResp.decode message
|
285
|
+
res.value || nil
|
286
|
+
when :CounterGetResp
|
287
|
+
res = RpbCounterGetResp.decode message
|
288
|
+
res.value || 0
|
214
289
|
end
|
215
290
|
end
|
216
291
|
rescue SystemCallError, SocketError => e
|
@@ -218,6 +293,46 @@ module Riak
|
|
218
293
|
raise
|
219
294
|
end
|
220
295
|
|
296
|
+
def streaming_list_buckets(options = {})
|
297
|
+
request = RpbListBucketsReq.new(options.merge(stream: true))
|
298
|
+
write_protobuff :ListBucketsReq, request
|
299
|
+
loop do
|
300
|
+
header = socket.read 5
|
301
|
+
raise SocketError, "Unexpected EOF on PBC socket" if header.nil?
|
302
|
+
len, code = header.unpack 'NC'
|
303
|
+
if MESSAGE_CODES[code] != :ListBucketsResp
|
304
|
+
raise SocketError, "Unexpected non-ListBucketsResp during streaming list buckets"
|
305
|
+
end
|
306
|
+
|
307
|
+
message = socket.read(len - 1)
|
308
|
+
section = RpbListBucketsResp.decode message
|
309
|
+
yield section.buckets
|
310
|
+
|
311
|
+
return if section.done
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
def decode_index_response
|
316
|
+
loop do
|
317
|
+
header = socket.read(5)
|
318
|
+
raise SocketError, "Unexpected EOF on PBC socket" if header.nil?
|
319
|
+
msglen, msgcode = header.unpack("NC")
|
320
|
+
code = MESSAGE_CODES[msgcode]
|
321
|
+
raise SocketError, "Expected IndexResp, got #{code}" unless code == :IndexResp
|
322
|
+
|
323
|
+
message = RpbIndexResp.decode socket.read msglen - 1
|
324
|
+
|
325
|
+
if !block_given?
|
326
|
+
return IndexCollection.new_from_protobuf(message)
|
327
|
+
end
|
328
|
+
|
329
|
+
content = message.keys || message.results || []
|
330
|
+
yield content
|
331
|
+
|
332
|
+
return if message.done
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
221
336
|
def decode_doc(doc)
|
222
337
|
Hash[doc.properties.map {|p| [ force_utf8(p.key), force_utf8(p.value) ] }]
|
223
338
|
end
|
@@ -226,6 +341,24 @@ module Riak
|
|
226
341
|
# Search returns strings that should always be valid UTF-8
|
227
342
|
ObjectMethods::ENCODING ? str.force_encoding('UTF-8') : str
|
228
343
|
end
|
344
|
+
|
345
|
+
def normalize_hooks(message)
|
346
|
+
message.dup.tap do |o|
|
347
|
+
%w{chash_keyfun linkfun}.each do |k|
|
348
|
+
o[k] = {'mod' => message[k].module, 'fun' => message[k].function}
|
349
|
+
end
|
350
|
+
%w{precommit postcommit}.each do |k|
|
351
|
+
orig = message[k]
|
352
|
+
o[k] = orig.map do |hook|
|
353
|
+
if hook.modfun
|
354
|
+
{'mod' => hook.modfun.module, 'fun' => hook.modfun.function}
|
355
|
+
else
|
356
|
+
hook.name
|
357
|
+
end
|
358
|
+
end
|
359
|
+
end
|
360
|
+
end
|
361
|
+
end
|
229
362
|
end
|
230
363
|
end
|
231
364
|
end
|
@@ -14,7 +14,8 @@ module Riak
|
|
14
14
|
1 => Gem::Version.new("1.0.0"),
|
15
15
|
1.1 => Gem::Version.new("1.1.0"),
|
16
16
|
1.2 => Gem::Version.new("1.2.0"),
|
17
|
-
1.3 => Gem::Version.new("1.3.0")
|
17
|
+
1.3 => Gem::Version.new("1.3.0"),
|
18
|
+
1.4 => Gem::Version.new("1.4.0")
|
18
19
|
}.freeze
|
19
20
|
|
20
21
|
# @return [String] the version of the Riak node
|
@@ -78,6 +79,30 @@ module Riak
|
|
78
79
|
at_least? VERSION[1.3]
|
79
80
|
end
|
80
81
|
|
82
|
+
# @return [true,false] whether secondary indexes support
|
83
|
+
# pagination
|
84
|
+
def index_pagination?
|
85
|
+
at_least? VERSION[1.4]
|
86
|
+
end
|
87
|
+
|
88
|
+
# @return [true,false] whether secondary indexes support
|
89
|
+
# return_terms
|
90
|
+
def index_return_terms?
|
91
|
+
at_least? VERSION[1.4]
|
92
|
+
end
|
93
|
+
|
94
|
+
# @return [true,false] whether secondary indexes support
|
95
|
+
# streaming
|
96
|
+
def index_streaming?
|
97
|
+
at_least? VERSION[1.4]
|
98
|
+
end
|
99
|
+
|
100
|
+
# @return [true,false] whether timeouts are accepted for
|
101
|
+
# object CRUD, key listing, and bucket listing
|
102
|
+
def key_object_bucket_timeouts?
|
103
|
+
at_least? VERSION[1.4]
|
104
|
+
end
|
105
|
+
|
81
106
|
protected
|
82
107
|
# @return [true,false] whether the server version is the same or
|
83
108
|
# newer than the requested version
|
@@ -10,6 +10,7 @@ require 'riak/client/http_backend/transport_methods'
|
|
10
10
|
require 'riak/client/http_backend/object_methods'
|
11
11
|
require 'riak/client/http_backend/configuration'
|
12
12
|
require 'riak/client/http_backend/key_streamer'
|
13
|
+
require 'riak/client/http_backend/bucket_streamer'
|
13
14
|
require 'riak/client/feature_detection'
|
14
15
|
|
15
16
|
module Riak
|
@@ -115,6 +116,38 @@ module Riak
|
|
115
116
|
delete([204, 404], object_path(bucket, key, options), headers)
|
116
117
|
end
|
117
118
|
|
119
|
+
# Fetches a counter
|
120
|
+
# @param [Bucket, String] bucket the bucket where the counter exists
|
121
|
+
# @param [String] key the key for the counter
|
122
|
+
# @param [Hash] options unused
|
123
|
+
def get_counter(bucket, key, options={})
|
124
|
+
bucket = bucket.name if bucket.is_a? Bucket
|
125
|
+
response = get([200, 404], counter_path(bucket, key, options))
|
126
|
+
case response[:code]
|
127
|
+
when 200
|
128
|
+
return response[:body].to_i
|
129
|
+
when 404
|
130
|
+
return 0
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
# Updates a counter
|
135
|
+
# @param [Bucket, String] bucket the bucket where the counter exists
|
136
|
+
# @param [String] key the key for the counter
|
137
|
+
# @param [Integer] amount how much to increment the counter
|
138
|
+
# @param [Hash] options unused
|
139
|
+
def post_counter(bucket, key, amount, options={})
|
140
|
+
bucket = bucket.name if bucket.is_a? Bucket
|
141
|
+
response = post([200, 204], counter_path(bucket, key, options), amount.to_s)
|
142
|
+
case response[:code]
|
143
|
+
when 200
|
144
|
+
return response[:body].to_i
|
145
|
+
when 204
|
146
|
+
return 0 if options[:return_value]
|
147
|
+
return nil
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
118
151
|
# Fetches bucket properties
|
119
152
|
# @param [Bucket, String] bucket the bucket properties to fetch
|
120
153
|
# @return [Hash] bucket properties
|
@@ -155,12 +188,14 @@ module Riak
|
|
155
188
|
# @yield [Array<String>] a list of keys from the current
|
156
189
|
# streamed chunk
|
157
190
|
# @return [Array<String>] the list of keys, if no block was given
|
158
|
-
def list_keys(bucket, &block)
|
191
|
+
def list_keys(bucket, options={}, &block)
|
159
192
|
bucket = bucket.name if Bucket === bucket
|
160
193
|
if block_given?
|
161
|
-
|
194
|
+
stream_opts = options.merge keys: 'stream'
|
195
|
+
get(200, key_list_path(bucket, stream_opts), {}, &KeyStreamer.new(block))
|
162
196
|
else
|
163
|
-
|
197
|
+
list_opts = options.merge keys: true
|
198
|
+
response = get(200, key_list_path(bucket, list_opts))
|
164
199
|
obj = JSON.parse(response[:body])
|
165
200
|
obj && obj['keys'].map {|k| unescape(k) }
|
166
201
|
end
|
@@ -168,7 +203,12 @@ module Riak
|
|
168
203
|
|
169
204
|
# Lists known buckets
|
170
205
|
# @return [Array<String>] the list of buckets
|
171
|
-
def list_buckets
|
206
|
+
def list_buckets(&block)
|
207
|
+
if block_given?
|
208
|
+
get(200, bucket_list_path(stream: true), &BucketStreamer.new(block))
|
209
|
+
return
|
210
|
+
end
|
211
|
+
|
172
212
|
response = get(200, bucket_list_path)
|
173
213
|
JSON.parse(response[:body])['buckets']
|
174
214
|
end
|
@@ -234,19 +274,28 @@ module Riak
|
|
234
274
|
# @param [String, Integer, Range] query the equality query or
|
235
275
|
# range query to perform
|
236
276
|
# @return [Array<String>] a list of keys matching the query
|
237
|
-
def get_index(bucket, index, query)
|
277
|
+
def get_index(bucket, index, query, options={})
|
238
278
|
bucket = bucket.name if Bucket === bucket
|
239
279
|
path = case query
|
240
280
|
when Range
|
241
281
|
raise ArgumentError, t('invalid_index_query', :value => query.inspect) unless String === query.begin || Integer === query.end
|
242
|
-
index_range_path(bucket, index, query.begin, query.end)
|
282
|
+
index_range_path(bucket, index, query.begin, query.end, options)
|
243
283
|
when String, Integer
|
244
|
-
index_eq_path(bucket, index, query)
|
284
|
+
index_eq_path(bucket, index, query, options)
|
245
285
|
else
|
246
286
|
raise ArgumentError, t('invalid_index_query', :value => query.inspect)
|
247
287
|
end
|
248
|
-
|
249
|
-
|
288
|
+
if block_given?
|
289
|
+
parser = Riak::Util::Multipart::StreamParser.new do |response|
|
290
|
+
result = JSON.parse response[:body]
|
291
|
+
|
292
|
+
yield result['keys'] || result['results'] || []
|
293
|
+
end
|
294
|
+
get(200, path, &parser)
|
295
|
+
else
|
296
|
+
response = get(200, path)
|
297
|
+
Riak::IndexCollection.new_from_json response[:body]
|
298
|
+
end
|
250
299
|
end
|
251
300
|
|
252
301
|
# (Riak Search) Performs a search query
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'riak/util/escape'
|
2
|
+
require 'riak/json'
|
3
|
+
|
4
|
+
module Riak
|
5
|
+
class Client
|
6
|
+
class HTTPBackend
|
7
|
+
class ChunkedJsonStreamer
|
8
|
+
include Util::Escape
|
9
|
+
|
10
|
+
def initialize(block)
|
11
|
+
@buffer = ""
|
12
|
+
@block = block
|
13
|
+
end
|
14
|
+
|
15
|
+
def accept(chunk)
|
16
|
+
@buffer << chunk
|
17
|
+
consume
|
18
|
+
end
|
19
|
+
|
20
|
+
def to_proc
|
21
|
+
method(:accept).to_proc
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
def consume
|
26
|
+
while @buffer =~ /\}\{/
|
27
|
+
stream($~.pre_match + '}')
|
28
|
+
@buffer = '{' + $~.post_match
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def stream(str)
|
33
|
+
obj = JSON.parse(str) rescue nil
|
34
|
+
if obj && get_values(obj)
|
35
|
+
@block.call get_values(obj).map(&method(:maybe_unescape))
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
@@ -29,7 +29,10 @@ module Riak
|
|
29
29
|
|
30
30
|
# @return [URI] a URL path for the "buckets list" resource
|
31
31
|
def bucket_list_path(options={})
|
32
|
-
if new_scheme?
|
32
|
+
if options[:stream] && new_scheme?
|
33
|
+
options.delete :stream
|
34
|
+
path(riak_kv_wm_buckets, options.merge(buckets: 'stream'))
|
35
|
+
elsif new_scheme?
|
33
36
|
path(riak_kv_wm_buckets, options.merge(:buckets => true))
|
34
37
|
else
|
35
38
|
path(riak_kv_wm_raw, options.merge(:buckets => true))
|
@@ -71,6 +74,19 @@ module Riak
|
|
71
74
|
end
|
72
75
|
end
|
73
76
|
|
77
|
+
# @return [URI] a URL path for the "counter" resource
|
78
|
+
# @param [String] bucket the bucket of the counter
|
79
|
+
# @param [String] key the key of the counter
|
80
|
+
def counter_path(bucket, key, options={})
|
81
|
+
path([
|
82
|
+
riak_kv_wm_buckets,
|
83
|
+
escape(bucket),
|
84
|
+
"counters",
|
85
|
+
escape(key),
|
86
|
+
options
|
87
|
+
].compact)
|
88
|
+
end
|
89
|
+
|
74
90
|
# @return [URI] a URL path for the "link-walking" resource
|
75
91
|
# @param [String] bucket the bucket of the origin object
|
76
92
|
# @param [String] key the key of the origin object
|