riak-client 1.0.5 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.gitignore +4 -3
- data/.rspec +1 -0
- data/Gemfile +1 -0
- data/RELEASE_NOTES.md +47 -0
- data/Rakefile +0 -1
- data/erl_src/riak_kv_test_backend.erl +34 -0
- data/lib/riak/client.rb +3 -1
- data/lib/riak/client/beefcake/messages.rb +49 -1
- data/lib/riak/client/beefcake/object_methods.rb +14 -21
- data/lib/riak/client/beefcake_protobuffs_backend.rb +58 -12
- data/lib/riak/client/decaying.rb +31 -23
- data/lib/riak/client/feature_detection.rb +88 -0
- data/lib/riak/client/http_backend.rb +27 -6
- data/lib/riak/client/http_backend/configuration.rb +13 -0
- data/lib/riak/client/http_backend/object_methods.rb +33 -25
- data/lib/riak/client/node.rb +7 -2
- data/lib/riak/client/protobuffs_backend.rb +54 -3
- data/lib/riak/client/search.rb +2 -2
- data/lib/riak/conflict.rb +13 -0
- data/lib/riak/locale/en.yml +2 -0
- data/lib/riak/map_reduce.rb +1 -1
- data/lib/riak/map_reduce/filter_builder.rb +2 -2
- data/lib/riak/map_reduce/results.rb +49 -0
- data/lib/riak/node/console.rb +17 -16
- data/lib/riak/node/generation.rb +9 -0
- data/lib/riak/rcontent.rb +168 -0
- data/lib/riak/robject.rb +37 -157
- data/lib/riak/util/escape.rb +5 -1
- data/lib/riak/version.rb +1 -1
- data/riak-client.gemspec +37 -5
- data/spec/fixtures/multipart-basic-conflict.txt +15 -0
- data/spec/fixtures/munchausen.txt +1033 -0
- data/spec/integration/riak/cluster_spec.rb +1 -1
- data/spec/integration/riak/http_backends_spec.rb +23 -2
- data/spec/integration/riak/node_spec.rb +2 -2
- data/spec/integration/riak/protobuffs_backends_spec.rb +17 -2
- data/spec/integration/riak/test_server_spec.rb +1 -1
- data/spec/integration/riak/threading_spec.rb +3 -3
- data/spec/riak/beefcake_protobuffs_backend_spec.rb +58 -25
- data/spec/riak/escape_spec.rb +3 -0
- data/spec/riak/feature_detection_spec.rb +61 -0
- data/spec/riak/http_backend/object_methods_spec.rb +4 -13
- data/spec/riak/http_backend_spec.rb +6 -5
- data/spec/riak/map_reduce_spec.rb +0 -5
- data/spec/riak/robject_spec.rb +12 -11
- data/spec/spec_helper.rb +3 -1
- data/spec/support/riak_test.rb +77 -0
- data/spec/support/search_corpus_setup.rb +18 -0
- data/spec/support/sometimes.rb +1 -1
- data/spec/support/test_server.rb +1 -1
- data/spec/support/unified_backend_examples.rb +53 -7
- data/spec/support/version_filter.rb +4 -11
- metadata +56 -22
- data/lib/riak/client/pool.rb +0 -180
- data/spec/riak/pool_spec.rb +0 -306
@@ -0,0 +1,88 @@
|
|
1
|
+
module Riak
|
2
|
+
class Client
|
3
|
+
# Methods that can be used to determine whether certain features
|
4
|
+
# are supported by the Riak node to which the client backend is
|
5
|
+
# connected.
|
6
|
+
#
|
7
|
+
# Backends must implement the "get_server_version" method,
|
8
|
+
# returning a string representing the Riak node's version. This is
|
9
|
+
# implemented on HTTP using the stats resource, and on Protocol
|
10
|
+
# Buffers using the RpbGetServerInfoReq message.
|
11
|
+
module FeatureDetection
|
12
|
+
# Constants representing Riak versions
|
13
|
+
VERSION = {
|
14
|
+
1 => Gem::Version.new("1.0.0"),
|
15
|
+
1.1 => Gem::Version.new("1.1.0"),
|
16
|
+
1.2 => Gem::Version.new("1.2.0")
|
17
|
+
}.freeze
|
18
|
+
|
19
|
+
# @return [String] the version of the Riak node
|
20
|
+
# @abstract
|
21
|
+
def get_server_version
|
22
|
+
raise NotImplementedError
|
23
|
+
end
|
24
|
+
|
25
|
+
# @return [Gem::Version] the version of the Riak node to which
|
26
|
+
# this backend is connected
|
27
|
+
def server_version
|
28
|
+
@server_version ||= Gem::Version.new(get_server_version)
|
29
|
+
end
|
30
|
+
|
31
|
+
# @return [true,false] whether MapReduce requests can be submitted without
|
32
|
+
# phases.
|
33
|
+
def mapred_phaseless?
|
34
|
+
at_least? VERSION[1.1]
|
35
|
+
end
|
36
|
+
|
37
|
+
# @return [true,false] whether secondary index queries are
|
38
|
+
# supported over Protocol Buffers
|
39
|
+
def pb_indexes?
|
40
|
+
at_least? VERSION[1.2]
|
41
|
+
end
|
42
|
+
|
43
|
+
# @return [true,false] whether search queries are supported over
|
44
|
+
# Protocol Buffers
|
45
|
+
def pb_search?
|
46
|
+
at_least? VERSION[1.2]
|
47
|
+
end
|
48
|
+
|
49
|
+
# @return [true,false] whether conditional fetch/store semantics
|
50
|
+
# are supported over Protocol Buffers
|
51
|
+
def pb_conditionals?
|
52
|
+
at_least? VERSION[1]
|
53
|
+
end
|
54
|
+
|
55
|
+
# @return [true,false] whether additional quorums and FSM
|
56
|
+
# controls are available, e.g. primary quorums, basic_quorum,
|
57
|
+
# notfound_ok
|
58
|
+
def quorum_controls?
|
59
|
+
at_least? VERSION[1]
|
60
|
+
end
|
61
|
+
|
62
|
+
# @return [true,false] whether "not found" responses might
|
63
|
+
# include vclocks
|
64
|
+
def tombstone_vclocks?
|
65
|
+
at_least? VERSION[1]
|
66
|
+
end
|
67
|
+
|
68
|
+
# @return [true,false] whether partial-fetches (vclock and
|
69
|
+
# metadata only) are supported over Protocol Buffers
|
70
|
+
def pb_head?
|
71
|
+
at_least? VERSION[1]
|
72
|
+
end
|
73
|
+
|
74
|
+
protected
|
75
|
+
# @return [true,false] whether the server version is the same or
|
76
|
+
# newer than the requested version
|
77
|
+
def at_least?(version)
|
78
|
+
server_version >= version
|
79
|
+
end
|
80
|
+
|
81
|
+
# Backends should call this when their connection is interrupted
|
82
|
+
# or reset so as to facilitate rolling upgrades
|
83
|
+
def reset_server_version
|
84
|
+
@server_version = nil
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -10,6 +10,7 @@ require 'riak/client/http_backend/transport_methods'
|
|
10
10
|
require 'riak/client/http_backend/object_methods'
|
11
11
|
require 'riak/client/http_backend/configuration'
|
12
12
|
require 'riak/client/http_backend/key_streamer'
|
13
|
+
require 'riak/client/feature_detection'
|
13
14
|
|
14
15
|
module Riak
|
15
16
|
class Client
|
@@ -21,6 +22,7 @@ module Riak
|
|
21
22
|
class HTTPBackend
|
22
23
|
include Util::Escape
|
23
24
|
include Util::Translation
|
25
|
+
include FeatureDetection
|
24
26
|
|
25
27
|
include TransportMethods
|
26
28
|
include ObjectMethods
|
@@ -162,6 +164,7 @@ module Riak
|
|
162
164
|
# @return [Array<Object>] the list of results, if no block was
|
163
165
|
# given
|
164
166
|
def mapred(mr)
|
167
|
+
raise MapReduceError.new(t("empty_map_reduce_query")) if mr.query.empty? && !mapred_phaseless?
|
165
168
|
if block_given?
|
166
169
|
parser = Riak::Util::Multipart::StreamParser.new do |response|
|
167
170
|
result = JSON.parse(response[:body])
|
@@ -170,12 +173,13 @@ module Riak
|
|
170
173
|
post(200, mapred_path({:chunked => true}), mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"}, &parser)
|
171
174
|
nil
|
172
175
|
else
|
173
|
-
|
174
|
-
|
175
|
-
JSON.parse(response[:body])
|
176
|
-
|
177
|
-
response
|
176
|
+
results = MapReduce::Results.new(mr)
|
177
|
+
parser = Riak::Util::Multipart::StreamParser.new do |response|
|
178
|
+
result = JSON.parse(response[:body])
|
179
|
+
results.add result['phase'], result['data']
|
178
180
|
end
|
181
|
+
post(200, mapred_path({:chunked => true}), mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"}, &parser)
|
182
|
+
results.report
|
179
183
|
end
|
180
184
|
end
|
181
185
|
|
@@ -238,7 +242,7 @@ module Riak
|
|
238
242
|
def search(index, query, options={})
|
239
243
|
response = get(200, solr_select_path(index, query, options.stringify_keys))
|
240
244
|
if response[:headers]['content-type'].include?("application/json")
|
241
|
-
JSON.parse(response[:body])
|
245
|
+
normalize_search_response JSON.parse(response[:body])
|
242
246
|
else
|
243
247
|
response[:body]
|
244
248
|
end
|
@@ -315,6 +319,23 @@ module Riak
|
|
315
319
|
response[:headers]["location"].first.split("/").last
|
316
320
|
end
|
317
321
|
end
|
322
|
+
|
323
|
+
private
|
324
|
+
def normalize_search_response(json)
|
325
|
+
{}.tap do |result|
|
326
|
+
if json['response']
|
327
|
+
result['num_found'] = json['response']['numFound']
|
328
|
+
result['max_score'] = json['response']['maxScore'].to_f
|
329
|
+
result['docs'] = json['response']['docs'].map do |d|
|
330
|
+
if d['fields']
|
331
|
+
d['fields'].merge('id' => d['id'])
|
332
|
+
else
|
333
|
+
d
|
334
|
+
end
|
335
|
+
end
|
336
|
+
end
|
337
|
+
end
|
338
|
+
end
|
318
339
|
end
|
319
340
|
end
|
320
341
|
end
|
@@ -157,6 +157,19 @@ module Riak
|
|
157
157
|
!riak_kv_wm_buckets.nil?
|
158
158
|
end
|
159
159
|
|
160
|
+
def get_server_version
|
161
|
+
begin
|
162
|
+
# Attempt to grab the server version from the stats resource
|
163
|
+
stats = send(:stats)
|
164
|
+
stats['riak_kv_version']
|
165
|
+
rescue FailedRequest
|
166
|
+
# If stats is disabled, we can't assume the Riak version
|
167
|
+
# is >= 1.1. However, we can assume the new URL scheme is
|
168
|
+
# at least version 1.0.
|
169
|
+
new_scheme? ? "1.0.0" : "0.14.0"
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
160
173
|
def riak_kv_wm_buckets
|
161
174
|
server_config[:riak_kv_wm_buckets]
|
162
175
|
end
|
@@ -52,28 +52,15 @@ module Riak
|
|
52
52
|
# @param [Hash] response a response from {Riak::Client::HTTPBackend}
|
53
53
|
def load_object(robject, response)
|
54
54
|
extract_header(robject, response, "location", :key) {|v| URI.unescape(v.match(%r{.*/(.*?)(\?.*)?$})[1]) }
|
55
|
-
extract_header(robject, response, "content-type", :content_type)
|
56
55
|
extract_header(robject, response, "x-riak-vclock", :vclock)
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
h
|
65
|
-
end
|
66
|
-
robject.indexes = response[:headers].inject(Hash.new {|h,k| h[k] = Set.new }) do |h,(k,v)|
|
67
|
-
if k =~ /x-riak-index-((?:.*)_(?:int|bin))$/i
|
68
|
-
key = $1
|
69
|
-
h[key].merge Array(v).map {|vals| vals.split(/,\s*/).map {|i| key =~ /int$/ ? i.to_i : i } }.flatten
|
70
|
-
end
|
71
|
-
h
|
56
|
+
case response[:code] && response[:code].to_i
|
57
|
+
when 304
|
58
|
+
# Resulted from a reload, don't modify anything
|
59
|
+
when 300
|
60
|
+
robject.siblings = extract_siblings(robject, response[:body], response[:headers]['content-type'].first)
|
61
|
+
else
|
62
|
+
robject.siblings = [ load_content(response, RContent.new(robject)) ]
|
72
63
|
end
|
73
|
-
robject.conflict = (response[:code] && response[:code].to_i == 300 && robject.content_type =~ /multipart\/mixed/)
|
74
|
-
robject.siblings = robject.conflict? ? extract_siblings(robject, response[:body]) : nil
|
75
|
-
robject.raw_data = response[:body] if response[:body].present? && !robject.conflict?
|
76
|
-
|
77
64
|
robject.conflict? ? robject.attempt_conflict_resolution : robject
|
78
65
|
end
|
79
66
|
|
@@ -92,13 +79,34 @@ module Riak
|
|
92
79
|
end
|
93
80
|
end
|
94
81
|
|
95
|
-
def extract_siblings(robject, data)
|
96
|
-
Util::Multipart.parse(data, Util::Multipart.extract_boundary(
|
97
|
-
|
98
|
-
|
99
|
-
|
82
|
+
def extract_siblings(robject, data, content_type)
|
83
|
+
Util::Multipart.parse(data, Util::Multipart.extract_boundary(content_type)).map do |part|
|
84
|
+
RContent.new(robject) do |sibling|
|
85
|
+
load_content(part, sibling)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def load_content(response, rcontent)
|
91
|
+
extract_header(rcontent, response, "link", :links) {|v| Set.new(Link.parse(v)) }
|
92
|
+
extract_header(rcontent, response, "etag", :etag)
|
93
|
+
extract_header(rcontent, response, "last-modified", :last_modified) {|v| Time.httpdate(v) }
|
94
|
+
extract_header(rcontent, response, "content-type", :content_type)
|
95
|
+
rcontent.meta = response[:headers].inject({}) do |h,(k,v)|
|
96
|
+
if k =~ /x-riak-meta-(.*)/i
|
97
|
+
h[$1] = v
|
98
|
+
end
|
99
|
+
h
|
100
|
+
end
|
101
|
+
rcontent.indexes = response[:headers].inject(Hash.new {|h,k| h[k] = Set.new }) do |h,(k,v)|
|
102
|
+
if k =~ /x-riak-index-((?:.*)_(?:int|bin))$/i
|
103
|
+
key = $1
|
104
|
+
h[key].merge Array(v).map {|vals| vals.split(/,\s*/).map {|i| key =~ /int$/ ? i.to_i : i } }.flatten
|
100
105
|
end
|
106
|
+
h
|
101
107
|
end
|
108
|
+
rcontent.raw_data = response[:body] if response[:body].present?
|
109
|
+
rcontent
|
102
110
|
end
|
103
111
|
end
|
104
112
|
end
|
data/lib/riak/client/node.rb
CHANGED
@@ -79,7 +79,12 @@ module Riak
|
|
79
79
|
# Enables or disables SSL on this node to be utilized by the HTTP
|
80
80
|
# Backends
|
81
81
|
def ssl=(value)
|
82
|
-
|
82
|
+
case value
|
83
|
+
when TrueClass
|
84
|
+
@ssl_options ||= {}
|
85
|
+
when Hash
|
86
|
+
(@ssl_options ||= {}).merge!(value)
|
87
|
+
end
|
83
88
|
value ? ssl_enable : ssl_disable
|
84
89
|
end
|
85
90
|
|
@@ -89,7 +94,7 @@ module Riak
|
|
89
94
|
end
|
90
95
|
|
91
96
|
def inspect
|
92
|
-
"
|
97
|
+
"#<Node #{@host}:#{@http_port}:#{@pb_port}>"
|
93
98
|
end
|
94
99
|
|
95
100
|
protected
|
@@ -3,12 +3,14 @@ require 'socket'
|
|
3
3
|
require 'base64'
|
4
4
|
require 'digest/sha1'
|
5
5
|
require 'riak/util/translation'
|
6
|
+
require 'riak/client/feature_detection'
|
6
7
|
|
7
8
|
module Riak
|
8
9
|
class Client
|
9
10
|
class ProtobuffsBackend
|
10
11
|
include Util::Translation
|
11
12
|
include Util::Escape
|
13
|
+
include FeatureDetection
|
12
14
|
|
13
15
|
# Message Codes Enum
|
14
16
|
MESSAGE_CODES = %W[
|
@@ -37,6 +39,10 @@ module Riak
|
|
37
39
|
SetBucketResp
|
38
40
|
MapRedReq
|
39
41
|
MapRedResp
|
42
|
+
IndexReq
|
43
|
+
IndexResp
|
44
|
+
SearchQueryReq
|
45
|
+
SearchQueryResp
|
40
46
|
].map {|s| s.intern }.freeze
|
41
47
|
|
42
48
|
def self.simple(method, code)
|
@@ -65,9 +71,32 @@ module Riak
|
|
65
71
|
# range query to perform
|
66
72
|
# @return [Array<String>] a list of keys matching the query
|
67
73
|
def get_index(bucket, index, query)
|
68
|
-
|
69
|
-
|
70
|
-
|
74
|
+
mr = Riak::MapReduce.new(client).index(bucket, index, query)
|
75
|
+
unless mapred_phaseless?
|
76
|
+
mr.reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)
|
77
|
+
end
|
78
|
+
mapred(mr).map {|p| p.last }
|
79
|
+
end
|
80
|
+
|
81
|
+
# Performs search query via emulation through MapReduce. This
|
82
|
+
# has more limited capabilites than native queries. Essentially,
|
83
|
+
# only the 'id' field of matched documents will ever be
|
84
|
+
# returned, the 'fl' and other options have no effect.
|
85
|
+
# @param [String] index the index to query
|
86
|
+
# @param [String] query the Lucene-style search query
|
87
|
+
# @param [Hash] options ignored in MapReduce emulation
|
88
|
+
# @return [Hash] the search results
|
89
|
+
def search(index, query, options={})
|
90
|
+
mr = Riak::MapReduce.new(client).search(index || 'search', query)
|
91
|
+
unless mapred_phaseless?
|
92
|
+
mr.reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)
|
93
|
+
end
|
94
|
+
docs = mapred(mr).map {|d| {'id' => d[1] } }
|
95
|
+
# Since we don't get this information back from the MapReduce,
|
96
|
+
# we have to fake the max_score and num_found.
|
97
|
+
{ 'docs' => docs,
|
98
|
+
'num_found' => docs.size,
|
99
|
+
'max_score' => 0.0 }
|
71
100
|
end
|
72
101
|
|
73
102
|
# Gracefully shuts down this connection.
|
@@ -76,6 +105,10 @@ module Riak
|
|
76
105
|
end
|
77
106
|
|
78
107
|
private
|
108
|
+
def get_server_version
|
109
|
+
server_info[:server_version]
|
110
|
+
end
|
111
|
+
|
79
112
|
# Implemented by subclasses
|
80
113
|
def decode_response
|
81
114
|
raise NotImplementedError
|
@@ -94,6 +127,7 @@ module Riak
|
|
94
127
|
end
|
95
128
|
|
96
129
|
def reset_socket
|
130
|
+
reset_server_version
|
97
131
|
@socket.close if @socket && !@socket.closed?
|
98
132
|
@socket = nil
|
99
133
|
end
|
@@ -106,6 +140,23 @@ module Riak
|
|
106
140
|
"default" => UINTMAX - 4
|
107
141
|
}.freeze
|
108
142
|
|
143
|
+
def prune_unsupported_options(req,options={})
|
144
|
+
unless quorum_controls?
|
145
|
+
[:notfound_ok, :basic_quorum, :pr, :pw].each {|k| options.delete k }
|
146
|
+
end
|
147
|
+
unless pb_head?
|
148
|
+
[:head, :return_head].each {|k| options.delete k }
|
149
|
+
end
|
150
|
+
unless tombstone_vclocks?
|
151
|
+
options.delete :deletedvclock
|
152
|
+
options.delete :vclock if req == :DelReq
|
153
|
+
end
|
154
|
+
unless pb_conditionals?
|
155
|
+
[:if_not_modified, :if_none_match, :if_modified].each {|k| options.delete k }
|
156
|
+
end
|
157
|
+
options
|
158
|
+
end
|
159
|
+
|
109
160
|
def normalize_quorums(options={})
|
110
161
|
options.dup.tap do |o|
|
111
162
|
[:r, :pr, :w, :pw, :dw, :rw].each do |k|
|
data/lib/riak/client/search.rb
CHANGED
@@ -20,8 +20,8 @@ module Riak
|
|
20
20
|
def search(*args)
|
21
21
|
options = args.extract_options!
|
22
22
|
index, query = args[-2], args[-1] # Allows nil index, while keeping it as firstargument
|
23
|
-
|
24
|
-
|
23
|
+
backend do |b|
|
24
|
+
b.search(index, query, options)
|
25
25
|
end
|
26
26
|
end
|
27
27
|
alias :select :search
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'riak/util/translation'
|
2
|
+
|
3
|
+
module Riak
|
4
|
+
# Raised when an object that is in conflict (i.e. has siblings) is
|
5
|
+
# stored or manipulated as if it had a single value.
|
6
|
+
class Conflict < StandardError
|
7
|
+
include Util::Translation
|
8
|
+
|
9
|
+
def initialize(robject)
|
10
|
+
super t('object_in_conflict', :robject => robject.inspect)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
data/lib/riak/locale/en.yml
CHANGED
@@ -36,8 +36,10 @@ en:
|
|
36
36
|
module_function_pair_required: "function must have two elements when an array"
|
37
37
|
not_found: "The requested object was not found."
|
38
38
|
no_pipes: "Could not find or open pipes for Riak console in %{path}."
|
39
|
+
object_in_conflict: "The object is in conflict (has siblings) and cannot be treated singly or saved: %{robject}"
|
39
40
|
port_invalid: "port must be an integer between 0 and 65535"
|
40
41
|
protobuffs_failed_request: "Expected success from Riak but received %{code}. %{body}"
|
42
|
+
protobuffs_configuration: "The %{backend} Protobuffs backend cannot be used. Please check its requirements."
|
41
43
|
request_body_type: "Request body must be a String or respond to :read."
|
42
44
|
search_unsupported: "Riak server does not support search."
|
43
45
|
search_docs_require_id: "Search index documents must include the 'id' field."
|
data/lib/riak/map_reduce.rb
CHANGED
@@ -9,6 +9,7 @@ require 'riak/failed_request'
|
|
9
9
|
require 'riak/map_reduce_error'
|
10
10
|
require 'riak/map_reduce/phase'
|
11
11
|
require 'riak/map_reduce/filter_builder'
|
12
|
+
require 'riak/map_reduce/results'
|
12
13
|
|
13
14
|
module Riak
|
14
15
|
# Class for invoking map-reduce jobs using the HTTP interface.
|
@@ -213,7 +214,6 @@ module Riak
|
|
213
214
|
# @yieldparam [Array] data a list of results from the phase
|
214
215
|
# @return [nil] nothing
|
215
216
|
def run(&block)
|
216
|
-
raise MapReduceError.new(t("empty_map_reduce_query")) if @query.empty?
|
217
217
|
@client.mapred(self, &block)
|
218
218
|
rescue FailedRequest => fr
|
219
219
|
if fr.server_error? && fr.is_json?
|