riak-client 1.0.5 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/.document +5 -0
  2. data/.gitignore +4 -3
  3. data/.rspec +1 -0
  4. data/Gemfile +1 -0
  5. data/RELEASE_NOTES.md +47 -0
  6. data/Rakefile +0 -1
  7. data/erl_src/riak_kv_test_backend.erl +34 -0
  8. data/lib/riak/client.rb +3 -1
  9. data/lib/riak/client/beefcake/messages.rb +49 -1
  10. data/lib/riak/client/beefcake/object_methods.rb +14 -21
  11. data/lib/riak/client/beefcake_protobuffs_backend.rb +58 -12
  12. data/lib/riak/client/decaying.rb +31 -23
  13. data/lib/riak/client/feature_detection.rb +88 -0
  14. data/lib/riak/client/http_backend.rb +27 -6
  15. data/lib/riak/client/http_backend/configuration.rb +13 -0
  16. data/lib/riak/client/http_backend/object_methods.rb +33 -25
  17. data/lib/riak/client/node.rb +7 -2
  18. data/lib/riak/client/protobuffs_backend.rb +54 -3
  19. data/lib/riak/client/search.rb +2 -2
  20. data/lib/riak/conflict.rb +13 -0
  21. data/lib/riak/locale/en.yml +2 -0
  22. data/lib/riak/map_reduce.rb +1 -1
  23. data/lib/riak/map_reduce/filter_builder.rb +2 -2
  24. data/lib/riak/map_reduce/results.rb +49 -0
  25. data/lib/riak/node/console.rb +17 -16
  26. data/lib/riak/node/generation.rb +9 -0
  27. data/lib/riak/rcontent.rb +168 -0
  28. data/lib/riak/robject.rb +37 -157
  29. data/lib/riak/util/escape.rb +5 -1
  30. data/lib/riak/version.rb +1 -1
  31. data/riak-client.gemspec +37 -5
  32. data/spec/fixtures/multipart-basic-conflict.txt +15 -0
  33. data/spec/fixtures/munchausen.txt +1033 -0
  34. data/spec/integration/riak/cluster_spec.rb +1 -1
  35. data/spec/integration/riak/http_backends_spec.rb +23 -2
  36. data/spec/integration/riak/node_spec.rb +2 -2
  37. data/spec/integration/riak/protobuffs_backends_spec.rb +17 -2
  38. data/spec/integration/riak/test_server_spec.rb +1 -1
  39. data/spec/integration/riak/threading_spec.rb +3 -3
  40. data/spec/riak/beefcake_protobuffs_backend_spec.rb +58 -25
  41. data/spec/riak/escape_spec.rb +3 -0
  42. data/spec/riak/feature_detection_spec.rb +61 -0
  43. data/spec/riak/http_backend/object_methods_spec.rb +4 -13
  44. data/spec/riak/http_backend_spec.rb +6 -5
  45. data/spec/riak/map_reduce_spec.rb +0 -5
  46. data/spec/riak/robject_spec.rb +12 -11
  47. data/spec/spec_helper.rb +3 -1
  48. data/spec/support/riak_test.rb +77 -0
  49. data/spec/support/search_corpus_setup.rb +18 -0
  50. data/spec/support/sometimes.rb +1 -1
  51. data/spec/support/test_server.rb +1 -1
  52. data/spec/support/unified_backend_examples.rb +53 -7
  53. data/spec/support/version_filter.rb +4 -11
  54. metadata +56 -22
  55. data/lib/riak/client/pool.rb +0 -180
  56. data/spec/riak/pool_spec.rb +0 -306
@@ -0,0 +1,88 @@
1
+ module Riak
2
+ class Client
3
+ # Methods that can be used to determine whether certain features
4
+ # are supported by the Riak node to which the client backend is
5
+ # connected.
6
+ #
7
+ # Backends must implement the "get_server_version" method,
8
+ # returning a string representing the Riak node's version. This is
9
+ # implemented on HTTP using the stats resource, and on Protocol
10
+ # Buffers using the RpbGetServerInfoReq message.
11
+ module FeatureDetection
12
+ # Constants representing Riak versions
13
+ VERSION = {
14
+ 1 => Gem::Version.new("1.0.0"),
15
+ 1.1 => Gem::Version.new("1.1.0"),
16
+ 1.2 => Gem::Version.new("1.2.0")
17
+ }.freeze
18
+
19
+ # @return [String] the version of the Riak node
20
+ # @abstract
21
+ def get_server_version
22
+ raise NotImplementedError
23
+ end
24
+
25
+ # @return [Gem::Version] the version of the Riak node to which
26
+ # this backend is connected
27
+ def server_version
28
+ @server_version ||= Gem::Version.new(get_server_version)
29
+ end
30
+
31
+ # @return [true,false] whether MapReduce requests can be submitted without
32
+ # phases.
33
+ def mapred_phaseless?
34
+ at_least? VERSION[1.1]
35
+ end
36
+
37
+ # @return [true,false] whether secondary index queries are
38
+ # supported over Protocol Buffers
39
+ def pb_indexes?
40
+ at_least? VERSION[1.2]
41
+ end
42
+
43
+ # @return [true,false] whether search queries are supported over
44
+ # Protocol Buffers
45
+ def pb_search?
46
+ at_least? VERSION[1.2]
47
+ end
48
+
49
+ # @return [true,false] whether conditional fetch/store semantics
50
+ # are supported over Protocol Buffers
51
+ def pb_conditionals?
52
+ at_least? VERSION[1]
53
+ end
54
+
55
+ # @return [true,false] whether additional quorums and FSM
56
+ # controls are available, e.g. primary quorums, basic_quorum,
57
+ # notfound_ok
58
+ def quorum_controls?
59
+ at_least? VERSION[1]
60
+ end
61
+
62
+ # @return [true,false] whether "not found" responses might
63
+ # include vclocks
64
+ def tombstone_vclocks?
65
+ at_least? VERSION[1]
66
+ end
67
+
68
+ # @return [true,false] whether partial-fetches (vclock and
69
+ # metadata only) are supported over Protocol Buffers
70
+ def pb_head?
71
+ at_least? VERSION[1]
72
+ end
73
+
74
+ protected
75
+ # @return [true,false] whether the server version is the same or
76
+ # newer than the requested version
77
+ def at_least?(version)
78
+ server_version >= version
79
+ end
80
+
81
+ # Backends should call this when their connection is interrupted
82
+ # or reset so as to facilitate rolling upgrades
83
+ def reset_server_version
84
+ @server_version = nil
85
+ end
86
+ end
87
+ end
88
+ end
@@ -10,6 +10,7 @@ require 'riak/client/http_backend/transport_methods'
10
10
  require 'riak/client/http_backend/object_methods'
11
11
  require 'riak/client/http_backend/configuration'
12
12
  require 'riak/client/http_backend/key_streamer'
13
+ require 'riak/client/feature_detection'
13
14
 
14
15
  module Riak
15
16
  class Client
@@ -21,6 +22,7 @@ module Riak
21
22
  class HTTPBackend
22
23
  include Util::Escape
23
24
  include Util::Translation
25
+ include FeatureDetection
24
26
 
25
27
  include TransportMethods
26
28
  include ObjectMethods
@@ -162,6 +164,7 @@ module Riak
162
164
  # @return [Array<Object>] the list of results, if no block was
163
165
  # given
164
166
  def mapred(mr)
167
+ raise MapReduceError.new(t("empty_map_reduce_query")) if mr.query.empty? && !mapred_phaseless?
165
168
  if block_given?
166
169
  parser = Riak::Util::Multipart::StreamParser.new do |response|
167
170
  result = JSON.parse(response[:body])
@@ -170,12 +173,13 @@ module Riak
170
173
  post(200, mapred_path({:chunked => true}), mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"}, &parser)
171
174
  nil
172
175
  else
173
- response = post(200, mapred_path, mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"})
174
- begin
175
- JSON.parse(response[:body])
176
- rescue
177
- response
176
+ results = MapReduce::Results.new(mr)
177
+ parser = Riak::Util::Multipart::StreamParser.new do |response|
178
+ result = JSON.parse(response[:body])
179
+ results.add result['phase'], result['data']
178
180
  end
181
+ post(200, mapred_path({:chunked => true}), mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"}, &parser)
182
+ results.report
179
183
  end
180
184
  end
181
185
 
@@ -238,7 +242,7 @@ module Riak
238
242
  def search(index, query, options={})
239
243
  response = get(200, solr_select_path(index, query, options.stringify_keys))
240
244
  if response[:headers]['content-type'].include?("application/json")
241
- JSON.parse(response[:body])
245
+ normalize_search_response JSON.parse(response[:body])
242
246
  else
243
247
  response[:body]
244
248
  end
@@ -315,6 +319,23 @@ module Riak
315
319
  response[:headers]["location"].first.split("/").last
316
320
  end
317
321
  end
322
+
323
+ private
324
+ def normalize_search_response(json)
325
+ {}.tap do |result|
326
+ if json['response']
327
+ result['num_found'] = json['response']['numFound']
328
+ result['max_score'] = json['response']['maxScore'].to_f
329
+ result['docs'] = json['response']['docs'].map do |d|
330
+ if d['fields']
331
+ d['fields'].merge('id' => d['id'])
332
+ else
333
+ d
334
+ end
335
+ end
336
+ end
337
+ end
338
+ end
318
339
  end
319
340
  end
320
341
  end
@@ -157,6 +157,19 @@ module Riak
157
157
  !riak_kv_wm_buckets.nil?
158
158
  end
159
159
 
160
+ def get_server_version
161
+ begin
162
+ # Attempt to grab the server version from the stats resource
163
+ stats = send(:stats)
164
+ stats['riak_kv_version']
165
+ rescue FailedRequest
166
+ # If stats is disabled, we can't assume the Riak version
167
+ # is >= 1.1. However, we can assume the new URL scheme is
168
+ # at least version 1.0.
169
+ new_scheme? ? "1.0.0" : "0.14.0"
170
+ end
171
+ end
172
+
160
173
  def riak_kv_wm_buckets
161
174
  server_config[:riak_kv_wm_buckets]
162
175
  end
@@ -52,28 +52,15 @@ module Riak
52
52
  # @param [Hash] response a response from {Riak::Client::HTTPBackend}
53
53
  def load_object(robject, response)
54
54
  extract_header(robject, response, "location", :key) {|v| URI.unescape(v.match(%r{.*/(.*?)(\?.*)?$})[1]) }
55
- extract_header(robject, response, "content-type", :content_type)
56
55
  extract_header(robject, response, "x-riak-vclock", :vclock)
57
- extract_header(robject, response, "link", :links) {|v| Set.new(Link.parse(v)) }
58
- extract_header(robject, response, "etag", :etag)
59
- extract_header(robject, response, "last-modified", :last_modified) {|v| Time.httpdate(v) }
60
- robject.meta = response[:headers].inject({}) do |h,(k,v)|
61
- if k =~ /x-riak-meta-(.*)/i
62
- h[$1] = v
63
- end
64
- h
65
- end
66
- robject.indexes = response[:headers].inject(Hash.new {|h,k| h[k] = Set.new }) do |h,(k,v)|
67
- if k =~ /x-riak-index-((?:.*)_(?:int|bin))$/i
68
- key = $1
69
- h[key].merge Array(v).map {|vals| vals.split(/,\s*/).map {|i| key =~ /int$/ ? i.to_i : i } }.flatten
70
- end
71
- h
56
+ case response[:code] && response[:code].to_i
57
+ when 304
58
+ # Resulted from a reload, don't modify anything
59
+ when 300
60
+ robject.siblings = extract_siblings(robject, response[:body], response[:headers]['content-type'].first)
61
+ else
62
+ robject.siblings = [ load_content(response, RContent.new(robject)) ]
72
63
  end
73
- robject.conflict = (response[:code] && response[:code].to_i == 300 && robject.content_type =~ /multipart\/mixed/)
74
- robject.siblings = robject.conflict? ? extract_siblings(robject, response[:body]) : nil
75
- robject.raw_data = response[:body] if response[:body].present? && !robject.conflict?
76
-
77
64
  robject.conflict? ? robject.attempt_conflict_resolution : robject
78
65
  end
79
66
 
@@ -92,13 +79,34 @@ module Riak
92
79
  end
93
80
  end
94
81
 
95
- def extract_siblings(robject, data)
96
- Util::Multipart.parse(data, Util::Multipart.extract_boundary(robject.content_type)).map do |part|
97
- RObject.new(robject.bucket, robject.key) do |sibling|
98
- load_object(sibling, part)
99
- sibling.vclock = robject.vclock
82
+ def extract_siblings(robject, data, content_type)
83
+ Util::Multipart.parse(data, Util::Multipart.extract_boundary(content_type)).map do |part|
84
+ RContent.new(robject) do |sibling|
85
+ load_content(part, sibling)
86
+ end
87
+ end
88
+ end
89
+
90
+ def load_content(response, rcontent)
91
+ extract_header(rcontent, response, "link", :links) {|v| Set.new(Link.parse(v)) }
92
+ extract_header(rcontent, response, "etag", :etag)
93
+ extract_header(rcontent, response, "last-modified", :last_modified) {|v| Time.httpdate(v) }
94
+ extract_header(rcontent, response, "content-type", :content_type)
95
+ rcontent.meta = response[:headers].inject({}) do |h,(k,v)|
96
+ if k =~ /x-riak-meta-(.*)/i
97
+ h[$1] = v
98
+ end
99
+ h
100
+ end
101
+ rcontent.indexes = response[:headers].inject(Hash.new {|h,k| h[k] = Set.new }) do |h,(k,v)|
102
+ if k =~ /x-riak-index-((?:.*)_(?:int|bin))$/i
103
+ key = $1
104
+ h[key].merge Array(v).map {|vals| vals.split(/,\s*/).map {|i| key =~ /int$/ ? i.to_i : i } }.flatten
100
105
  end
106
+ h
101
107
  end
108
+ rcontent.raw_data = response[:body] if response[:body].present?
109
+ rcontent
102
110
  end
103
111
  end
104
112
  end
@@ -79,7 +79,12 @@ module Riak
79
79
  # Enables or disables SSL on this node to be utilized by the HTTP
80
80
  # Backends
81
81
  def ssl=(value)
82
- @ssl_options = Hash === value ? value : {}
82
+ case value
83
+ when TrueClass
84
+ @ssl_options ||= {}
85
+ when Hash
86
+ (@ssl_options ||= {}).merge!(value)
87
+ end
83
88
  value ? ssl_enable : ssl_disable
84
89
  end
85
90
 
@@ -89,7 +94,7 @@ module Riak
89
94
  end
90
95
 
91
96
  def inspect
92
- "<#Node #{@host}:#{@http_port}:#{@pb_port}>"
97
+ "#<Node #{@host}:#{@http_port}:#{@pb_port}>"
93
98
  end
94
99
 
95
100
  protected
@@ -3,12 +3,14 @@ require 'socket'
3
3
  require 'base64'
4
4
  require 'digest/sha1'
5
5
  require 'riak/util/translation'
6
+ require 'riak/client/feature_detection'
6
7
 
7
8
  module Riak
8
9
  class Client
9
10
  class ProtobuffsBackend
10
11
  include Util::Translation
11
12
  include Util::Escape
13
+ include FeatureDetection
12
14
 
13
15
  # Message Codes Enum
14
16
  MESSAGE_CODES = %W[
@@ -37,6 +39,10 @@ module Riak
37
39
  SetBucketResp
38
40
  MapRedReq
39
41
  MapRedResp
42
+ IndexReq
43
+ IndexResp
44
+ SearchQueryReq
45
+ SearchQueryResp
40
46
  ].map {|s| s.intern }.freeze
41
47
 
42
48
  def self.simple(method, code)
@@ -65,9 +71,32 @@ module Riak
65
71
  # range query to perform
66
72
  # @return [Array<String>] a list of keys matching the query
67
73
  def get_index(bucket, index, query)
68
- mapred(Riak::MapReduce.new(client).
69
- index(bucket, index, query).
70
- reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)).map {|p| p.last }
74
+ mr = Riak::MapReduce.new(client).index(bucket, index, query)
75
+ unless mapred_phaseless?
76
+ mr.reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)
77
+ end
78
+ mapred(mr).map {|p| p.last }
79
+ end
80
+
81
+ # Performs search query via emulation through MapReduce. This
82
+ # has more limited capabilites than native queries. Essentially,
83
+ # only the 'id' field of matched documents will ever be
84
+ # returned, the 'fl' and other options have no effect.
85
+ # @param [String] index the index to query
86
+ # @param [String] query the Lucene-style search query
87
+ # @param [Hash] options ignored in MapReduce emulation
88
+ # @return [Hash] the search results
89
+ def search(index, query, options={})
90
+ mr = Riak::MapReduce.new(client).search(index || 'search', query)
91
+ unless mapred_phaseless?
92
+ mr.reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)
93
+ end
94
+ docs = mapred(mr).map {|d| {'id' => d[1] } }
95
+ # Since we don't get this information back from the MapReduce,
96
+ # we have to fake the max_score and num_found.
97
+ { 'docs' => docs,
98
+ 'num_found' => docs.size,
99
+ 'max_score' => 0.0 }
71
100
  end
72
101
 
73
102
  # Gracefully shuts down this connection.
@@ -76,6 +105,10 @@ module Riak
76
105
  end
77
106
 
78
107
  private
108
+ def get_server_version
109
+ server_info[:server_version]
110
+ end
111
+
79
112
  # Implemented by subclasses
80
113
  def decode_response
81
114
  raise NotImplementedError
@@ -94,6 +127,7 @@ module Riak
94
127
  end
95
128
 
96
129
  def reset_socket
130
+ reset_server_version
97
131
  @socket.close if @socket && !@socket.closed?
98
132
  @socket = nil
99
133
  end
@@ -106,6 +140,23 @@ module Riak
106
140
  "default" => UINTMAX - 4
107
141
  }.freeze
108
142
 
143
+ def prune_unsupported_options(req,options={})
144
+ unless quorum_controls?
145
+ [:notfound_ok, :basic_quorum, :pr, :pw].each {|k| options.delete k }
146
+ end
147
+ unless pb_head?
148
+ [:head, :return_head].each {|k| options.delete k }
149
+ end
150
+ unless tombstone_vclocks?
151
+ options.delete :deletedvclock
152
+ options.delete :vclock if req == :DelReq
153
+ end
154
+ unless pb_conditionals?
155
+ [:if_not_modified, :if_none_match, :if_modified].each {|k| options.delete k }
156
+ end
157
+ options
158
+ end
159
+
109
160
  def normalize_quorums(options={})
110
161
  options.dup.tap do |o|
111
162
  [:r, :pr, :w, :pw, :dw, :rw].each do |k|
@@ -20,8 +20,8 @@ module Riak
20
20
  def search(*args)
21
21
  options = args.extract_options!
22
22
  index, query = args[-2], args[-1] # Allows nil index, while keeping it as firstargument
23
- http do |h|
24
- h.search(index, query, options)
23
+ backend do |b|
24
+ b.search(index, query, options)
25
25
  end
26
26
  end
27
27
  alias :select :search
@@ -0,0 +1,13 @@
1
+ require 'riak/util/translation'
2
+
3
+ module Riak
4
+ # Raised when an object that is in conflict (i.e. has siblings) is
5
+ # stored or manipulated as if it had a single value.
6
+ class Conflict < StandardError
7
+ include Util::Translation
8
+
9
+ def initialize(robject)
10
+ super t('object_in_conflict', :robject => robject.inspect)
11
+ end
12
+ end
13
+ end
@@ -36,8 +36,10 @@ en:
36
36
  module_function_pair_required: "function must have two elements when an array"
37
37
  not_found: "The requested object was not found."
38
38
  no_pipes: "Could not find or open pipes for Riak console in %{path}."
39
+ object_in_conflict: "The object is in conflict (has siblings) and cannot be treated singly or saved: %{robject}"
39
40
  port_invalid: "port must be an integer between 0 and 65535"
40
41
  protobuffs_failed_request: "Expected success from Riak but received %{code}. %{body}"
42
+ protobuffs_configuration: "The %{backend} Protobuffs backend cannot be used. Please check its requirements."
41
43
  request_body_type: "Request body must be a String or respond to :read."
42
44
  search_unsupported: "Riak server does not support search."
43
45
  search_docs_require_id: "Search index documents must include the 'id' field."
@@ -9,6 +9,7 @@ require 'riak/failed_request'
9
9
  require 'riak/map_reduce_error'
10
10
  require 'riak/map_reduce/phase'
11
11
  require 'riak/map_reduce/filter_builder'
12
+ require 'riak/map_reduce/results'
12
13
 
13
14
  module Riak
14
15
  # Class for invoking map-reduce jobs using the HTTP interface.
@@ -213,7 +214,6 @@ module Riak
213
214
  # @yieldparam [Array] data a list of results from the phase
214
215
  # @return [nil] nothing
215
216
  def run(&block)
216
- raise MapReduceError.new(t("empty_map_reduce_query")) if @query.empty?
217
217
  @client.mapred(self, &block)
218
218
  rescue FailedRequest => fr
219
219
  if fr.server_error? && fr.is_json?
@@ -46,8 +46,8 @@ module Riak
46
46
  # FilterBuilder.new do
47
47
  # string_to_int
48
48
  # AND do
49
- # seq { greater_than_eq 50 }
50
- # seq { neq 100 }
49
+ # greater_than_eq 50
50
+ # neq 100
51
51
  # end
52
52
  # end
53
53
  LOGICAL_OPERATIONS = %w{and or not}