riak-client 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/Gemfile +1 -7
  4. data/README.markdown +66 -0
  5. data/RELEASE_NOTES.md +27 -0
  6. data/lib/riak/bucket.rb +24 -5
  7. data/lib/riak/client.rb +42 -7
  8. data/lib/riak/client/beefcake/message_codes.rb +56 -0
  9. data/lib/riak/client/beefcake/messages.rb +190 -18
  10. data/lib/riak/client/beefcake_protobuffs_backend.rb +143 -10
  11. data/lib/riak/client/feature_detection.rb +26 -1
  12. data/lib/riak/client/http_backend.rb +58 -9
  13. data/lib/riak/client/http_backend/bucket_streamer.rb +15 -0
  14. data/lib/riak/client/http_backend/chunked_json_streamer.rb +42 -0
  15. data/lib/riak/client/http_backend/configuration.rb +17 -1
  16. data/lib/riak/client/http_backend/key_streamer.rb +4 -32
  17. data/lib/riak/client/protobuffs_backend.rb +12 -34
  18. data/lib/riak/counter.rb +101 -0
  19. data/lib/riak/index_collection.rb +71 -0
  20. data/lib/riak/list_buckets.rb +28 -0
  21. data/lib/riak/locale/en.yml +14 -0
  22. data/lib/riak/multiget.rb +123 -0
  23. data/lib/riak/node.rb +2 -0
  24. data/lib/riak/node/configuration.rb +32 -21
  25. data/lib/riak/node/defaults.rb +2 -0
  26. data/lib/riak/node/generation.rb +19 -7
  27. data/lib/riak/node/version.rb +2 -16
  28. data/lib/riak/robject.rb +1 -0
  29. data/lib/riak/secondary_index.rb +67 -0
  30. data/lib/riak/version.rb +1 -1
  31. data/riak-client.gemspec +3 -2
  32. data/spec/integration/riak/counters_spec.rb +51 -0
  33. data/spec/integration/riak/http_backends_spec.rb +24 -14
  34. data/spec/integration/riak/node_spec.rb +6 -28
  35. data/spec/riak/beefcake_protobuffs_backend_spec.rb +84 -0
  36. data/spec/riak/bucket_spec.rb +55 -5
  37. data/spec/riak/client_spec.rb +34 -0
  38. data/spec/riak/counter_spec.rb +122 -0
  39. data/spec/riak/index_collection_spec.rb +50 -0
  40. data/spec/riak/list_buckets_spec.rb +41 -0
  41. data/spec/riak/multiget_spec.rb +76 -0
  42. data/spec/riak/robject_spec.rb +4 -1
  43. data/spec/riak/secondary_index_spec.rb +225 -0
  44. data/spec/spec_helper.rb +1 -0
  45. data/spec/support/sometimes.rb +2 -2
  46. data/spec/support/unified_backend_examples.rb +4 -0
  47. metadata +41 -47
@@ -77,11 +77,41 @@ module Riak
77
77
  decode_response
78
78
  end
79
79
 
80
+ def get_counter(bucket, key, options={})
81
+ bucket = bucket.name if bucket.is_a? Bucket
82
+
83
+ options = normalize_quorums(options)
84
+ options[:bucket] = bucket
85
+ options[:key] = key
86
+
87
+ request = RpbCounterGetReq.new options
88
+ write_protobuff :CounterGetReq, request
89
+
90
+ decode_response
91
+ end
92
+
93
+ def post_counter(bucket, key, amount, options={})
94
+ bucket = bucket.name if bucket.is_a? Bucket
95
+
96
+ options = normalize_quorums(options)
97
+ options[:bucket] = bucket
98
+ options[:key] = key
99
+ # TODO: raise if ammount doesn't fit in sint64
100
+ options[:amount] = amount
101
+
102
+ request = RpbCounterUpdateReq.new options
103
+ write_protobuff :CounterUpdateReq, request
104
+
105
+ decode_response
106
+ end
107
+
80
108
  def get_bucket_props(bucket)
81
109
  bucket = bucket.name if Bucket === bucket
82
110
  req = RpbGetBucketReq.new(:bucket => maybe_encode(bucket))
83
111
  write_protobuff(:GetBucketReq, req)
84
- decode_response
112
+ resp = normalize_quorums decode_response
113
+ normalized = normalize_hooks resp
114
+ normalized.stringify_keys
85
115
  end
86
116
 
87
117
  def set_bucket_props(bucket, props)
@@ -92,9 +122,16 @@ module Riak
92
122
  decode_response
93
123
  end
94
124
 
95
- def list_keys(bucket, &block)
125
+ def reset_bucket_props(bucket)
126
+ bucket = bucket.name if Bucket === bucket
127
+ req = RpbResetBucketReq.new(:bucket => maybe_encode(bucket))
128
+ write_protobuff(:ResetBucketReq)
129
+ decode_response
130
+ end
131
+
132
+ def list_keys(bucket, options={}, &block)
96
133
  bucket = bucket.name if Bucket === bucket
97
- req = RpbListKeysReq.new(:bucket => maybe_encode(bucket))
134
+ req = RpbListKeysReq.new(options.merge(:bucket => maybe_encode(bucket)))
98
135
  write_protobuff(:ListKeysReq, req)
99
136
  keys = []
100
137
  while msg = decode_response
@@ -108,6 +145,21 @@ module Riak
108
145
  block_given? || keys
109
146
  end
110
147
 
148
+ # override the simple list_buckets
149
+ def list_buckets(options={}, &blk)
150
+ if block_given?
151
+ return streaming_list_buckets options, &blk
152
+ end
153
+
154
+ raise t("streaming_bucket_list_without_block") if options[:stream]
155
+
156
+ request = RpbListBucketsReq.new options
157
+
158
+ write_protobuff :ListBucketsReq, request
159
+
160
+ decode_response
161
+ end
162
+
111
163
  def mapred(mr, &block)
112
164
  raise MapReduceError.new(t("empty_map_reduce_query")) if mr.query.empty? && !mapred_phaseless?
113
165
  req = RpbMapRedReq.new(:request => mr.to_json, :content_type => "application/json")
@@ -124,7 +176,7 @@ module Riak
124
176
  block_given? || results.report
125
177
  end
126
178
 
127
- def get_index(bucket, index, query)
179
+ def get_index(bucket, index, query, query_options={}, &block)
128
180
  return super unless pb_indexes?
129
181
  bucket = bucket.name if Bucket === bucket
130
182
  if Range === query
@@ -139,9 +191,14 @@ module Riak
139
191
  :key => query.to_s
140
192
  }
141
193
  end
142
- req = RpbIndexReq.new(options.merge(:bucket => bucket, :index => index))
194
+
195
+ options.merge!(:bucket => bucket, :index => index)
196
+ options.merge!(query_options)
197
+ options[:stream] = block_given?
198
+
199
+ req = RpbIndexReq.new(options)
143
200
  write_protobuff(:IndexReq, req)
144
- decode_response
201
+ decode_index_response(&block)
145
202
  end
146
203
 
147
204
  def search(index, query, options={})
@@ -166,12 +223,22 @@ module Riak
166
223
  msglen, msgcode = header.unpack("NC")
167
224
  if msglen == 1
168
225
  case MESSAGE_CODES[msgcode]
169
- when :PingResp, :SetClientIdResp, :PutResp, :DelResp, :SetBucketResp
226
+ when :PingResp,
227
+ :SetClientIdResp,
228
+ :PutResp,
229
+ :DelResp,
230
+ :SetBucketResp,
231
+ :ResetBucketResp
170
232
  true
171
- when :ListBucketsResp, :ListKeysResp
233
+ when :ListBucketsResp,
234
+ :ListKeysResp,
235
+ :IndexResp
172
236
  []
173
237
  when :GetResp
174
238
  raise Riak::ProtobuffsFailedRequest.new(:not_found, t('not_found'))
239
+ when :CounterGetResp,
240
+ :CounterUpdateResp
241
+ 0
175
242
  else
176
243
  false
177
244
  end
@@ -200,17 +267,25 @@ module Riak
200
267
  RpbListKeysResp.decode(message)
201
268
  when :GetBucketResp
202
269
  res = RpbGetBucketResp.decode(message)
203
- {'n_val' => res.props.n_val, 'allow_mult' => res.props.allow_mult}
270
+ res.props.to_hash.stringify_keys
204
271
  when :MapRedResp
205
272
  RpbMapRedResp.decode(message)
206
273
  when :IndexResp
207
274
  res = RpbIndexResp.decode(message)
208
- res.keys
275
+ IndexCollection.new_from_protobuf res
209
276
  when :SearchQueryResp
210
277
  res = RpbSearchQueryResp.decode(message)
211
278
  { 'docs' => res.docs.map {|d| decode_doc(d) },
212
279
  'max_score' => res.max_score,
213
280
  'num_found' => res.num_found }
281
+ when :CSBucketResp
282
+ res = RpbCSBucketResp.decode message
283
+ when :CounterUpdateResp
284
+ res = RpbCounterUpdateResp.decode message
285
+ res.value || nil
286
+ when :CounterGetResp
287
+ res = RpbCounterGetResp.decode message
288
+ res.value || 0
214
289
  end
215
290
  end
216
291
  rescue SystemCallError, SocketError => e
@@ -218,6 +293,46 @@ module Riak
218
293
  raise
219
294
  end
220
295
 
296
+ def streaming_list_buckets(options = {})
297
+ request = RpbListBucketsReq.new(options.merge(stream: true))
298
+ write_protobuff :ListBucketsReq, request
299
+ loop do
300
+ header = socket.read 5
301
+ raise SocketError, "Unexpected EOF on PBC socket" if header.nil?
302
+ len, code = header.unpack 'NC'
303
+ if MESSAGE_CODES[code] != :ListBucketsResp
304
+ raise SocketError, "Unexpected non-ListBucketsResp during streaming list buckets"
305
+ end
306
+
307
+ message = socket.read(len - 1)
308
+ section = RpbListBucketsResp.decode message
309
+ yield section.buckets
310
+
311
+ return if section.done
312
+ end
313
+ end
314
+
315
+ def decode_index_response
316
+ loop do
317
+ header = socket.read(5)
318
+ raise SocketError, "Unexpected EOF on PBC socket" if header.nil?
319
+ msglen, msgcode = header.unpack("NC")
320
+ code = MESSAGE_CODES[msgcode]
321
+ raise SocketError, "Expected IndexResp, got #{code}" unless code == :IndexResp
322
+
323
+ message = RpbIndexResp.decode socket.read msglen - 1
324
+
325
+ if !block_given?
326
+ return IndexCollection.new_from_protobuf(message)
327
+ end
328
+
329
+ content = message.keys || message.results || []
330
+ yield content
331
+
332
+ return if message.done
333
+ end
334
+ end
335
+
221
336
  def decode_doc(doc)
222
337
  Hash[doc.properties.map {|p| [ force_utf8(p.key), force_utf8(p.value) ] }]
223
338
  end
@@ -226,6 +341,24 @@ module Riak
226
341
  # Search returns strings that should always be valid UTF-8
227
342
  ObjectMethods::ENCODING ? str.force_encoding('UTF-8') : str
228
343
  end
344
+
345
+ def normalize_hooks(message)
346
+ message.dup.tap do |o|
347
+ %w{chash_keyfun linkfun}.each do |k|
348
+ o[k] = {'mod' => message[k].module, 'fun' => message[k].function}
349
+ end
350
+ %w{precommit postcommit}.each do |k|
351
+ orig = message[k]
352
+ o[k] = orig.map do |hook|
353
+ if hook.modfun
354
+ {'mod' => hook.modfun.module, 'fun' => hook.modfun.function}
355
+ else
356
+ hook.name
357
+ end
358
+ end
359
+ end
360
+ end
361
+ end
229
362
  end
230
363
  end
231
364
  end
@@ -14,7 +14,8 @@ module Riak
14
14
  1 => Gem::Version.new("1.0.0"),
15
15
  1.1 => Gem::Version.new("1.1.0"),
16
16
  1.2 => Gem::Version.new("1.2.0"),
17
- 1.3 => Gem::Version.new("1.3.0")
17
+ 1.3 => Gem::Version.new("1.3.0"),
18
+ 1.4 => Gem::Version.new("1.4.0")
18
19
  }.freeze
19
20
 
20
21
  # @return [String] the version of the Riak node
@@ -78,6 +79,30 @@ module Riak
78
79
  at_least? VERSION[1.3]
79
80
  end
80
81
 
82
+ # @return [true,false] whether secondary indexes support
83
+ # pagination
84
+ def index_pagination?
85
+ at_least? VERSION[1.4]
86
+ end
87
+
88
+ # @return [true,false] whether secondary indexes support
89
+ # return_terms
90
+ def index_return_terms?
91
+ at_least? VERSION[1.4]
92
+ end
93
+
94
+ # @return [true,false] whether secondary indexes support
95
+ # streaming
96
+ def index_streaming?
97
+ at_least? VERSION[1.4]
98
+ end
99
+
100
+ # @return [true,false] whether timeouts are accepted for
101
+ # object CRUD, key listing, and bucket listing
102
+ def key_object_bucket_timeouts?
103
+ at_least? VERSION[1.4]
104
+ end
105
+
81
106
  protected
82
107
  # @return [true,false] whether the server version is the same or
83
108
  # newer than the requested version
@@ -10,6 +10,7 @@ require 'riak/client/http_backend/transport_methods'
10
10
  require 'riak/client/http_backend/object_methods'
11
11
  require 'riak/client/http_backend/configuration'
12
12
  require 'riak/client/http_backend/key_streamer'
13
+ require 'riak/client/http_backend/bucket_streamer'
13
14
  require 'riak/client/feature_detection'
14
15
 
15
16
  module Riak
@@ -115,6 +116,38 @@ module Riak
115
116
  delete([204, 404], object_path(bucket, key, options), headers)
116
117
  end
117
118
 
119
+ # Fetches a counter
120
+ # @param [Bucket, String] bucket the bucket where the counter exists
121
+ # @param [String] key the key for the counter
122
+ # @param [Hash] options unused
123
+ def get_counter(bucket, key, options={})
124
+ bucket = bucket.name if bucket.is_a? Bucket
125
+ response = get([200, 404], counter_path(bucket, key, options))
126
+ case response[:code]
127
+ when 200
128
+ return response[:body].to_i
129
+ when 404
130
+ return 0
131
+ end
132
+ end
133
+
134
+ # Updates a counter
135
+ # @param [Bucket, String] bucket the bucket where the counter exists
136
+ # @param [String] key the key for the counter
137
+ # @param [Integer] amount how much to increment the counter
138
+ # @param [Hash] options unused
139
+ def post_counter(bucket, key, amount, options={})
140
+ bucket = bucket.name if bucket.is_a? Bucket
141
+ response = post([200, 204], counter_path(bucket, key, options), amount.to_s)
142
+ case response[:code]
143
+ when 200
144
+ return response[:body].to_i
145
+ when 204
146
+ return 0 if options[:return_value]
147
+ return nil
148
+ end
149
+ end
150
+
118
151
  # Fetches bucket properties
119
152
  # @param [Bucket, String] bucket the bucket properties to fetch
120
153
  # @return [Hash] bucket properties
@@ -155,12 +188,14 @@ module Riak
155
188
  # @yield [Array<String>] a list of keys from the current
156
189
  # streamed chunk
157
190
  # @return [Array<String>] the list of keys, if no block was given
158
- def list_keys(bucket, &block)
191
+ def list_keys(bucket, options={}, &block)
159
192
  bucket = bucket.name if Bucket === bucket
160
193
  if block_given?
161
- get(200, key_list_path(bucket, :keys => 'stream'), {}, &KeyStreamer.new(block))
194
+ stream_opts = options.merge keys: 'stream'
195
+ get(200, key_list_path(bucket, stream_opts), {}, &KeyStreamer.new(block))
162
196
  else
163
- response = get(200, key_list_path(bucket))
197
+ list_opts = options.merge keys: true
198
+ response = get(200, key_list_path(bucket, list_opts))
164
199
  obj = JSON.parse(response[:body])
165
200
  obj && obj['keys'].map {|k| unescape(k) }
166
201
  end
@@ -168,7 +203,12 @@ module Riak
168
203
 
169
204
  # Lists known buckets
170
205
  # @return [Array<String>] the list of buckets
171
- def list_buckets
206
+ def list_buckets(&block)
207
+ if block_given?
208
+ get(200, bucket_list_path(stream: true), &BucketStreamer.new(block))
209
+ return
210
+ end
211
+
172
212
  response = get(200, bucket_list_path)
173
213
  JSON.parse(response[:body])['buckets']
174
214
  end
@@ -234,19 +274,28 @@ module Riak
234
274
  # @param [String, Integer, Range] query the equality query or
235
275
  # range query to perform
236
276
  # @return [Array<String>] a list of keys matching the query
237
- def get_index(bucket, index, query)
277
+ def get_index(bucket, index, query, options={})
238
278
  bucket = bucket.name if Bucket === bucket
239
279
  path = case query
240
280
  when Range
241
281
  raise ArgumentError, t('invalid_index_query', :value => query.inspect) unless String === query.begin || Integer === query.end
242
- index_range_path(bucket, index, query.begin, query.end)
282
+ index_range_path(bucket, index, query.begin, query.end, options)
243
283
  when String, Integer
244
- index_eq_path(bucket, index, query)
284
+ index_eq_path(bucket, index, query, options)
245
285
  else
246
286
  raise ArgumentError, t('invalid_index_query', :value => query.inspect)
247
287
  end
248
- response = get(200, path)
249
- JSON.parse(response[:body])['keys']
288
+ if block_given?
289
+ parser = Riak::Util::Multipart::StreamParser.new do |response|
290
+ result = JSON.parse response[:body]
291
+
292
+ yield result['keys'] || result['results'] || []
293
+ end
294
+ get(200, path, &parser)
295
+ else
296
+ response = get(200, path)
297
+ Riak::IndexCollection.new_from_json response[:body]
298
+ end
250
299
  end
251
300
 
252
301
  # (Riak Search) Performs a search query
@@ -0,0 +1,15 @@
1
+ require 'riak/client/http_backend/chunked_json_streamer'
2
+
3
+ module Riak
4
+ class Client
5
+ class HTTPBackend
6
+ # @private
7
+ class BucketStreamer < ChunkedJsonStreamer
8
+ def get_values(obj)
9
+ obj['buckets']
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
15
+
@@ -0,0 +1,42 @@
1
+ require 'riak/util/escape'
2
+ require 'riak/json'
3
+
4
+ module Riak
5
+ class Client
6
+ class HTTPBackend
7
+ class ChunkedJsonStreamer
8
+ include Util::Escape
9
+
10
+ def initialize(block)
11
+ @buffer = ""
12
+ @block = block
13
+ end
14
+
15
+ def accept(chunk)
16
+ @buffer << chunk
17
+ consume
18
+ end
19
+
20
+ def to_proc
21
+ method(:accept).to_proc
22
+ end
23
+
24
+ private
25
+ def consume
26
+ while @buffer =~ /\}\{/
27
+ stream($~.pre_match + '}')
28
+ @buffer = '{' + $~.post_match
29
+ end
30
+ end
31
+
32
+ def stream(str)
33
+ obj = JSON.parse(str) rescue nil
34
+ if obj && get_values(obj)
35
+ @block.call get_values(obj).map(&method(:maybe_unescape))
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
42
+
@@ -29,7 +29,10 @@ module Riak
29
29
 
30
30
  # @return [URI] a URL path for the "buckets list" resource
31
31
  def bucket_list_path(options={})
32
- if new_scheme?
32
+ if options[:stream] && new_scheme?
33
+ options.delete :stream
34
+ path(riak_kv_wm_buckets, options.merge(buckets: 'stream'))
35
+ elsif new_scheme?
33
36
  path(riak_kv_wm_buckets, options.merge(:buckets => true))
34
37
  else
35
38
  path(riak_kv_wm_raw, options.merge(:buckets => true))
@@ -71,6 +74,19 @@ module Riak
71
74
  end
72
75
  end
73
76
 
77
+ # @return [URI] a URL path for the "counter" resource
78
+ # @param [String] bucket the bucket of the counter
79
+ # @param [String] key the key of the counter
80
+ def counter_path(bucket, key, options={})
81
+ path([
82
+ riak_kv_wm_buckets,
83
+ escape(bucket),
84
+ "counters",
85
+ escape(key),
86
+ options
87
+ ].compact)
88
+ end
89
+
74
90
  # @return [URI] a URL path for the "link-walking" resource
75
91
  # @param [String] bucket the bucket of the origin object
76
92
  # @param [String] key the key of the origin object