riak-client 0.9.8 → 1.0.0.beta
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +32 -0
- data/Gemfile +17 -11
- data/Guardfile +14 -0
- data/Rakefile +18 -44
- data/erl_src/riak_kv_test_backend.beam +0 -0
- data/erl_src/riak_kv_test_backend.erl +461 -128
- data/erl_src/riak_search_test_backend.beam +0 -0
- data/erl_src/riak_search_test_backend.erl +175 -0
- data/lib/active_support/cache/riak_store.rb +0 -13
- data/lib/riak.rb +11 -16
- data/lib/riak/bucket.rb +59 -41
- data/lib/riak/cache_store.rb +1 -14
- data/lib/riak/client.rb +145 -73
- data/lib/riak/client/beefcake/messages.rb +36 -31
- data/lib/riak/client/beefcake/object_methods.rb +27 -19
- data/lib/riak/client/beefcake_protobuffs_backend.rb +27 -33
- data/lib/riak/client/excon_backend.rb +0 -13
- data/lib/riak/client/http_backend.rb +95 -60
- data/lib/riak/client/http_backend/configuration.rb +144 -19
- data/lib/riak/client/http_backend/key_streamer.rb +1 -14
- data/lib/riak/client/http_backend/object_methods.rb +16 -16
- data/lib/riak/client/http_backend/request_headers.rb +0 -13
- data/lib/riak/client/http_backend/transport_methods.rb +26 -56
- data/lib/riak/client/net_http_backend.rb +11 -13
- data/lib/riak/client/protobuffs_backend.rb +21 -19
- data/lib/riak/client/pump.rb +1 -15
- data/lib/riak/client/search.rb +85 -0
- data/lib/riak/cluster.rb +151 -0
- data/lib/riak/core_ext.rb +1 -0
- data/lib/riak/core_ext/deep_dup.rb +13 -0
- data/lib/riak/core_ext/json.rb +15 -0
- data/lib/riak/core_ext/stringify_keys.rb +1 -1
- data/lib/riak/core_ext/symbolize_keys.rb +1 -1
- data/lib/riak/encoding.rb +6 -0
- data/lib/riak/failed_request.rb +2 -15
- data/lib/riak/i18n.rb +0 -13
- data/lib/riak/json.rb +19 -8
- data/lib/riak/link.rb +18 -20
- data/lib/riak/locale/en.yml +13 -16
- data/lib/riak/map_reduce.rb +40 -20
- data/lib/riak/map_reduce/filter_builder.rb +14 -18
- data/lib/riak/map_reduce/phase.rb +0 -13
- data/lib/riak/map_reduce_error.rb +0 -13
- data/lib/riak/node.rb +38 -0
- data/lib/riak/node/configuration.rb +286 -0
- data/lib/riak/node/console.rb +139 -0
- data/lib/riak/node/control.rb +207 -0
- data/lib/riak/node/defaults.rb +70 -0
- data/lib/riak/node/generation.rb +99 -0
- data/lib/riak/node/log.rb +34 -0
- data/lib/riak/node/version.rb +37 -0
- data/lib/riak/robject.rb +45 -41
- data/lib/riak/search.rb +2 -161
- data/lib/riak/serializers.rb +74 -0
- data/lib/riak/stamp.rb +77 -0
- data/lib/riak/test_server.rb +56 -220
- data/lib/riak/util/escape.rb +58 -17
- data/lib/riak/util/headers.rb +2 -15
- data/lib/riak/util/multipart.rb +0 -13
- data/lib/riak/util/multipart/stream_parser.rb +0 -13
- data/lib/riak/util/tcp_socket_extensions.rb +1 -14
- data/lib/riak/util/translation.rb +0 -13
- data/lib/riak/version.rb +3 -0
- data/lib/riak/walk_spec.rb +0 -13
- data/riak-client.gemspec +27 -47
- data/spec/fixtures/multipart-with-marked-tombstones.txt +17 -0
- data/spec/fixtures/multipart-with-unmarked-tombstone.txt +16 -0
- data/spec/integration/riak/cache_store_spec.rb +2 -40
- data/spec/integration/riak/cluster_spec.rb +88 -0
- data/spec/integration/riak/http_backends_spec.rb +6 -30
- data/spec/integration/riak/node_spec.rb +184 -0
- data/spec/integration/riak/protobuffs_backends_spec.rb +2 -26
- data/spec/integration/riak/test_server_spec.rb +31 -167
- data/spec/riak/beefcake_protobuffs_backend_spec.rb +5 -4
- data/spec/riak/bucket_spec.rb +26 -36
- data/spec/riak/client_spec.rb +44 -38
- data/spec/riak/escape_spec.rb +56 -30
- data/spec/riak/excon_backend_spec.rb +4 -17
- data/spec/riak/headers_spec.rb +1 -14
- data/spec/riak/http_backend/configuration_spec.rb +211 -34
- data/spec/riak/http_backend/object_methods_spec.rb +52 -18
- data/spec/riak/http_backend/transport_methods_spec.rb +5 -38
- data/spec/riak/http_backend_spec.rb +84 -78
- data/spec/riak/link_spec.rb +19 -18
- data/spec/riak/map_reduce/filter_builder_spec.rb +1 -14
- data/spec/riak/map_reduce/phase_spec.rb +1 -14
- data/spec/riak/map_reduce_spec.rb +141 -43
- data/spec/riak/multipart_spec.rb +1 -14
- data/spec/riak/net_http_backend_spec.rb +2 -15
- data/spec/riak/robject_spec.rb +129 -97
- data/spec/riak/search_spec.rb +45 -62
- data/spec/riak/serializers_spec.rb +93 -0
- data/spec/riak/stamp_spec.rb +54 -0
- data/spec/riak/stream_parser_spec.rb +3 -16
- data/spec/riak/walk_spec_spec.rb +1 -14
- data/spec/spec_helper.rb +22 -27
- data/spec/support/http_backend_implementation_examples.rb +49 -79
- data/spec/support/integration_setup.rb +10 -0
- data/spec/support/mock_server.rb +0 -14
- data/spec/support/mocks.rb +0 -13
- data/spec/support/test_server.rb +30 -0
- data/spec/support/test_server.yml.example +14 -2
- data/spec/support/unified_backend_examples.rb +36 -27
- metadata +100 -31
- data/lib/riak/client/curb_backend.rb +0 -89
- data/spec/riak/curb_backend_spec.rb +0 -76
data/lib/riak/json.rb
CHANGED
@@ -1,11 +1,6 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
require 'yajl/json_gem'
|
5
|
-
rescue LoadError
|
6
|
-
require 'json'
|
7
|
-
end
|
8
|
-
end
|
1
|
+
require 'multi_json'
|
2
|
+
MultiJson.engine # Force loading of an engine
|
3
|
+
require 'riak/core_ext/json'
|
9
4
|
|
10
5
|
module Riak
|
11
6
|
class << self
|
@@ -14,4 +9,20 @@ module Riak
|
|
14
9
|
attr_accessor :json_options
|
15
10
|
end
|
16
11
|
self.json_options = {:max_nesting => 20}
|
12
|
+
|
13
|
+
# JSON module for internal use inside riak-client
|
14
|
+
module JSON
|
15
|
+
class << self
|
16
|
+
# Parse a JSON string
|
17
|
+
def parse(str)
|
18
|
+
MultiJson.decode(str, Riak.json_options)
|
19
|
+
end
|
20
|
+
|
21
|
+
# Generate a JSON string
|
22
|
+
def encode(obj)
|
23
|
+
MultiJson.encode(obj)
|
24
|
+
end
|
25
|
+
alias :dump :encode
|
26
|
+
end
|
27
|
+
end
|
17
28
|
end
|
data/lib/riak/link.rb
CHANGED
@@ -1,16 +1,3 @@
|
|
1
|
-
# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
1
|
|
15
2
|
require 'riak/util/translation'
|
16
3
|
require 'riak/util/escape'
|
@@ -34,7 +21,10 @@ module Riak
|
|
34
21
|
attr_accessor :key
|
35
22
|
|
36
23
|
%w{bucket key}.each do |m|
|
37
|
-
|
24
|
+
define_method("#{m}=") { |value|
|
25
|
+
@url = nil
|
26
|
+
instance_variable_set("@#{m}", value)
|
27
|
+
}
|
38
28
|
end
|
39
29
|
|
40
30
|
# @param [String] header_string the string value of the Link: HTTP header from a Riak response
|
@@ -62,20 +52,28 @@ module Riak
|
|
62
52
|
end
|
63
53
|
|
64
54
|
# @return [String] the URL (relative or absolute) of the related resource
|
65
|
-
def url
|
66
|
-
@
|
55
|
+
def url(new_scheme=false)
|
56
|
+
if @bucket
|
57
|
+
if new_scheme
|
58
|
+
"/buckets/#{escape(bucket)}" + (key.blank? ? "" : "/keys/#{escape(key)}")
|
59
|
+
else
|
60
|
+
"/riak/#{escape(bucket)}" + (key.blank? ? "" : "/#{escape(key)}")
|
61
|
+
end
|
62
|
+
else
|
63
|
+
@url
|
64
|
+
end
|
67
65
|
end
|
68
66
|
|
69
67
|
def url=(value)
|
70
68
|
@url = value
|
71
|
-
@bucket = unescape($1) if value =~ %r{^/[^/]+/([^/]+)/?}
|
72
|
-
@key = unescape($1) if value =~ %r{^/[^/]+/[^/]+/([^/]+)/?}
|
69
|
+
@bucket = unescape($1) if value =~ %r{^/buckets/([^/]+)/?} || value =~ %r{^/[^/]+/([^/]+)/?}
|
70
|
+
@key = unescape($1) if value =~ %r{^/buckets/[^/]+/keys/([^/]+)/?} || value =~ %r{^/[^/]+/[^/]+/([^/]+)/?}
|
73
71
|
end
|
74
72
|
|
75
73
|
def inspect; to_s; end
|
76
74
|
|
77
|
-
def to_s
|
78
|
-
%Q[<#{url}>; riaktag="#{tag}"]
|
75
|
+
def to_s(new_scheme=false)
|
76
|
+
%Q[<#{url(new_scheme)}>; riaktag="#{tag}"]
|
79
77
|
end
|
80
78
|
|
81
79
|
def hash
|
data/lib/riak/locale/en.yml
CHANGED
@@ -1,27 +1,18 @@
|
|
1
|
-
# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
1
|
en:
|
15
2
|
riak:
|
3
|
+
backwards_clock: "System clock moved backwards, ID generation will fail for %{delay} more milliseconds."
|
16
4
|
bucket_link_conversion: "Can't convert a bucket link to a walk spec"
|
17
5
|
client_type: "invalid argument %{client} is not a Riak::Client"
|
6
|
+
conflict_resolver_invalid: "The given resolver (%{resolver}) did not respond to :call"
|
18
7
|
content_type_undefined: "content_type is not defined!"
|
19
8
|
deprecated:
|
20
9
|
port: "DEPRECATION: Riak::Client#port has been deprecated, use #http_port or #pb_port for the appropriate protocol.\n%{backtrace}"
|
10
|
+
search: "DEPRECATION: Riak Search features are included in the main client, you no longer need to require 'riak/search'.\n%{backtrace}"
|
21
11
|
empty_map_reduce_query: "Specify one or more query phases to your MapReduce."
|
22
12
|
failed_request: "Client request failed."
|
23
13
|
filter_needs_block: "Filter %{filter} expects a block."
|
24
14
|
filter_arity_mismatch: "Filter %{filter} expects %{expected} arguments but %{received} were given."
|
15
|
+
full_bucket_mapred: "Full-bucket MapReduce, including key filters, invokes list-keys which is an expensive operation that should not be used in production.\n %{backtrace}"
|
25
16
|
hash_type: "invalid argument %{hash} is not a Hash"
|
26
17
|
http_configuration: "The %{backend} HTTP backend cannot be used. Please check its requirements."
|
27
18
|
http_failed_request: "Expected %{expected} from Riak but received %{code}. %{body}"
|
@@ -29,23 +20,29 @@ en:
|
|
29
20
|
protocol_invalid: "'%{invalid}' is not a valid protocol, valid values are %{valid}"
|
30
21
|
invalid_basic_auth: "basic auth must be set using 'user:pass'"
|
31
22
|
invalid_client_id: "Invalid client ID, must be a string or between 0 and %{max_id}"
|
23
|
+
invalid_io_object: "Invalid IO-like object assigned to RObject#data. It should be assigned to raw_data instead."
|
32
24
|
invalid_function_value: "invalid value for function: %{value}"
|
33
25
|
invalid_options: "Invalid configuration options given."
|
34
26
|
invalid_phase_type: "type must be :map, :reduce, or :link"
|
35
|
-
invalid_response: "Expected %{expected} but received %{received} from Riak %{extra}"
|
36
27
|
invalid_ssl_verify_mode: "%{invalid} is not a valid :verify_mode option for SSL. Valid options are 'peer' and 'none'."
|
28
|
+
invalid_index_query: "%{value} is not a valid index query term, only Strings, Integers, and Ranges of those are allowed."
|
29
|
+
indexes_unsupported: "Riak server does not support secondary indexes."
|
37
30
|
loading_bucket: "while loading bucket '%{name}'"
|
31
|
+
list_buckets: "Riak::Client#buckets is an expensive operation that should not be used in production.\n %{backtrace}"
|
32
|
+
list_keys: "Riak::Bucket#keys is an expensive operation that should not be used in production.\n %{backtrace}"
|
38
33
|
missing_block: "A block must be given."
|
39
34
|
missing_host_and_port: "You must specify a host and port, or use the defaults of 127.0.0.1:8098"
|
40
35
|
module_function_pair_required: "function must have two elements when an array"
|
41
36
|
not_found: "The requested object was not found."
|
42
|
-
|
37
|
+
no_pipes: "Could not find or open pipes for Riak console in %{path}."
|
43
38
|
port_invalid: "port must be an integer between 0 and 65535"
|
44
39
|
protobuffs_failed_request: "Expected success from Riak but received %{code}. %{body}"
|
45
40
|
request_body_type: "Request body must be a String or respond to :read."
|
46
|
-
|
41
|
+
search_unsupported: "Riak server does not support search."
|
47
42
|
search_docs_require_id: "Search index documents must include the 'id' field."
|
48
43
|
search_remove_requires_id_or_query: "Search index documents to be removed must have 'id' or 'query' keys."
|
44
|
+
serializer_not_implemented: "No serializer has been registered for content type %{content_type}"
|
45
|
+
source_and_root_required: "Riak::Node configuration must include :source and :root keys."
|
49
46
|
stale_write_prevented: "Stale write prevented by client."
|
50
47
|
stored_function_invalid: "function must have :bucket and :key when a hash"
|
51
48
|
string_type: "invalid_argument %{string} is not a String"
|
data/lib/riak/map_reduce.rb
CHANGED
@@ -1,17 +1,3 @@
|
|
1
|
-
# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
|
-
|
15
1
|
require 'riak/util/translation'
|
16
2
|
require 'riak/util/escape'
|
17
3
|
require 'riak/json'
|
@@ -81,20 +67,23 @@ module Riak
|
|
81
67
|
p = params.first
|
82
68
|
case p
|
83
69
|
when Bucket
|
84
|
-
|
70
|
+
warn(t('full_bucket_mapred', :backtrace => caller.join("\n "))) unless Riak.disable_list_keys_warnings
|
71
|
+
@inputs = maybe_escape(p.name)
|
85
72
|
when RObject
|
86
|
-
@inputs << [
|
73
|
+
@inputs << [maybe_escape(p.bucket.name), maybe_escape(p.key)]
|
87
74
|
when String
|
88
|
-
|
75
|
+
warn(t('full_bucket_mapred', :backtrace => caller.join("\n "))) unless Riak.disable_list_keys_warnings
|
76
|
+
@inputs = maybe_escape(p)
|
89
77
|
end
|
90
78
|
when 2..3
|
91
79
|
bucket = params.shift
|
92
80
|
bucket = bucket.name if Bucket === bucket
|
93
81
|
if Array === params.first
|
94
|
-
|
82
|
+
warn(t('full_bucket_mapred', :backtrace => caller.join("\n "))) unless Riak.disable_list_keys_warnings
|
83
|
+
@inputs = {:bucket => maybe_escape(bucket), :key_filters => params.first }
|
95
84
|
else
|
96
85
|
key = params.shift
|
97
|
-
@inputs << params.unshift(
|
86
|
+
@inputs << params.unshift(maybe_escape(key)).unshift(maybe_escape(bucket))
|
98
87
|
end
|
99
88
|
end
|
100
89
|
self
|
@@ -112,6 +101,37 @@ module Riak
|
|
112
101
|
add(bucket, FilterBuilder.new(&block).to_a)
|
113
102
|
end
|
114
103
|
|
104
|
+
# (Riak Search) Use a search query to start a map/reduce job.
|
105
|
+
# @param [String, Bucket] bucket the bucket/index to search
|
106
|
+
# @param [String] query the query to run
|
107
|
+
# @return [MapReduce] self
|
108
|
+
def search(bucket, query)
|
109
|
+
bucket = bucket.name if bucket.respond_to?(:name)
|
110
|
+
@inputs = {:module => "riak_search", :function => "mapred_search", :arg => [bucket, query]}
|
111
|
+
self
|
112
|
+
end
|
113
|
+
|
114
|
+
# (Secondary Indexes) Use a secondary index query to start a
|
115
|
+
# map/reduce job.
|
116
|
+
# @param [String, Bucket] bucket the bucket whose index to query
|
117
|
+
# @param [String] index the index to query
|
118
|
+
# @param [String, Integer, Range] query the value of the index, or
|
119
|
+
# a range of values (of Strings or Integers)
|
120
|
+
# @return [MapReduce] self
|
121
|
+
def index(bucket, index, query)
|
122
|
+
bucket = bucket.name if bucket.respond_to?(:name)
|
123
|
+
case query
|
124
|
+
when String, Fixnum
|
125
|
+
@inputs = {:bucket => maybe_escape(bucket), :index => index, :key => query}
|
126
|
+
when Range
|
127
|
+
raise ArgumentError, t('invalid_index_query', :value => query.inspect) unless String === query.begin || Integer === query.begin
|
128
|
+
@inputs = {:bucket => maybe_escape(bucket), :index => index, :start => query.begin, :end => query.end}
|
129
|
+
else
|
130
|
+
raise ArgumentError, t('invalid_index_query', :value => query.inspect)
|
131
|
+
end
|
132
|
+
self
|
133
|
+
end
|
134
|
+
|
115
135
|
# Add a map phase to the job.
|
116
136
|
# @overload map(function)
|
117
137
|
# @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module,function] pair
|
@@ -194,7 +214,7 @@ module Riak
|
|
194
214
|
# @return [nil] nothing
|
195
215
|
def run(&block)
|
196
216
|
raise MapReduceError.new(t("empty_map_reduce_query")) if @query.empty?
|
197
|
-
@client.
|
217
|
+
@client.mapred(self, &block)
|
198
218
|
rescue FailedRequest => fr
|
199
219
|
if fr.server_error? && fr.is_json?
|
200
220
|
raise MapReduceError.new(fr.body)
|
@@ -1,16 +1,3 @@
|
|
1
|
-
# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
1
|
|
15
2
|
require 'riak/util/translation'
|
16
3
|
|
@@ -66,15 +53,24 @@ module Riak
|
|
66
53
|
LOGICAL_OPERATIONS = %w{and or not}
|
67
54
|
|
68
55
|
FILTERS.each do |f,arity|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
56
|
+
arities = [arity].flatten
|
57
|
+
|
58
|
+
define_method(f) { |*args|
|
59
|
+
unless arities.include?(-1) or arities.include?(args.size)
|
60
|
+
raise ArgumentError.new t("filter_arity_mismatch",
|
61
|
+
:filter => f,
|
62
|
+
:expected => arities,
|
63
|
+
:received => args.size
|
64
|
+
)
|
73
65
|
end
|
74
|
-
|
66
|
+
|
67
|
+
@filters << [f, *args]
|
68
|
+
}
|
75
69
|
end
|
76
70
|
|
77
71
|
LOGICAL_OPERATIONS.each do |op|
|
72
|
+
# NB: string eval is needed here because in ruby 1.8, blocks can't yield to
|
73
|
+
# other blocks
|
78
74
|
class_eval <<-CODE
|
79
75
|
def _#{op}(&block)
|
80
76
|
raise ArgumentError.new(t('filter_needs_block', :filter => '#{op}')) unless block_given?
|
@@ -1,16 +1,3 @@
|
|
1
|
-
# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
1
|
|
15
2
|
require 'riak/json'
|
16
3
|
require 'riak/util/translation'
|
@@ -1,16 +1,3 @@
|
|
1
|
-
# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
1
|
require 'riak/util/translation'
|
15
2
|
|
16
3
|
module Riak
|
data/lib/riak/node.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'riak/util/translation'
|
2
|
+
require 'riak/node/defaults'
|
3
|
+
require 'riak/node/configuration'
|
4
|
+
require 'riak/node/generation'
|
5
|
+
require 'riak/node/control'
|
6
|
+
require 'riak/node/version'
|
7
|
+
require 'riak/node/log'
|
8
|
+
|
9
|
+
module Riak
|
10
|
+
# A Node encapsulates the generation and management of standalone
|
11
|
+
# Riak nodes. It is used by the {TestServer} to start and manage an
|
12
|
+
# in-memory node for supporting integration test suites.
|
13
|
+
class Node
|
14
|
+
include Util::Translation
|
15
|
+
|
16
|
+
# Creates a new Riak node. Unlike {#new}, this will also generate
|
17
|
+
# the node if it does not exist on disk. Equivalent to {::new}
|
18
|
+
# followed by {#create}.
|
19
|
+
# @see #new
|
20
|
+
def self.create(configuration={})
|
21
|
+
new(configuration).tap do |node|
|
22
|
+
node.create
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# Creates the template for a Riak node. To generate the node after
|
27
|
+
# initialization, use {#create}.
|
28
|
+
def initialize(configuration={})
|
29
|
+
set_defaults
|
30
|
+
configure configuration
|
31
|
+
end
|
32
|
+
|
33
|
+
protected
|
34
|
+
def debug(msg)
|
35
|
+
$stderr.puts msg if ENV["DEBUG_RIAK_NODE"]
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,286 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module Riak
|
5
|
+
class Node
|
6
|
+
# The directories (and accessor methods) that will be created
|
7
|
+
# under the generated node.
|
8
|
+
NODE_DIRECTORIES = [:bin, :etc, :log, :data, :ring, :pipe]
|
9
|
+
|
10
|
+
NODE_DIRECTORIES.each do |dir|
|
11
|
+
# Makes accessor methods for all the node directories that
|
12
|
+
# return Pathname objects.
|
13
|
+
class_eval %Q{
|
14
|
+
def #{dir}
|
15
|
+
root + '#{dir}'
|
16
|
+
end
|
17
|
+
}
|
18
|
+
end
|
19
|
+
|
20
|
+
# @return [Hash] the contents of the Erlang environment, which will
|
21
|
+
# be created into the app.config file.
|
22
|
+
attr_reader :env
|
23
|
+
|
24
|
+
# @return [Hash] the command-line switches for the Erlang virtual
|
25
|
+
# machine, which will be created into the vm.args file
|
26
|
+
attr_reader :vm
|
27
|
+
|
28
|
+
# @return [Hash] the configuration that was passed to the Node
|
29
|
+
# when initialized
|
30
|
+
attr_reader :configuration
|
31
|
+
|
32
|
+
# @return [Array<Pathname>] where user Erlang code will be loaded from
|
33
|
+
def erlang_sources
|
34
|
+
env[:riak_kv][:add_paths].map {|p| Pathname.new(p) }
|
35
|
+
end
|
36
|
+
|
37
|
+
# @return [Pathname] where user Javascript code will be loaded from
|
38
|
+
def javascript_source
|
39
|
+
Pathname.new(env[:riak_kv][:js_source_dir])
|
40
|
+
end
|
41
|
+
|
42
|
+
# @return [Fixnum] the size of the ring, i.e. number of partitions
|
43
|
+
def ring_size
|
44
|
+
env[:riak_core][:ring_creation_size]
|
45
|
+
end
|
46
|
+
|
47
|
+
# @return [Fixnum] The port used for handing off data to other nodes.
|
48
|
+
def handoff_port
|
49
|
+
env[:riak_core][:handoff_port]
|
50
|
+
end
|
51
|
+
|
52
|
+
# @return [Fixnum] The port to which the HTTP API is connected.
|
53
|
+
def http_port
|
54
|
+
# We'll only support 0.14 and later, which uses http rather than web_ip/web_port
|
55
|
+
env[:riak_core][:http][0][1]
|
56
|
+
end
|
57
|
+
|
58
|
+
# @return [Fixnum] the port to which the Protocol Buffers API is connected.
|
59
|
+
def pb_port
|
60
|
+
env[:riak_kv][:pb_port]
|
61
|
+
end
|
62
|
+
|
63
|
+
# @return [String] the interface to which the HTTP API is connected
|
64
|
+
def http_ip
|
65
|
+
env[:riak_core][:http][0][0]
|
66
|
+
end
|
67
|
+
|
68
|
+
# @return [String] the interface to which the Protocol Buffers API is connected
|
69
|
+
def pb_ip
|
70
|
+
env[:riak_kv][:pb_ip]
|
71
|
+
end
|
72
|
+
|
73
|
+
# @return [Symbol] the storage backend for Riak Search.
|
74
|
+
def search_backend
|
75
|
+
env[:riak_search][:search_backend]
|
76
|
+
end
|
77
|
+
|
78
|
+
# @return [Symbol] the storage backend for Riak KV.
|
79
|
+
def kv_backend
|
80
|
+
env[:riak_kv][:storage_backend]
|
81
|
+
end
|
82
|
+
|
83
|
+
# @return [String] the name of the Riak node as seen by distributed Erlang
|
84
|
+
# communication. AKA "-name" in vm.args.
|
85
|
+
def name
|
86
|
+
vm['-name']
|
87
|
+
end
|
88
|
+
|
89
|
+
# @return [String] the cookie/shared secret used for connecting
|
90
|
+
# a cluster
|
91
|
+
def cookie
|
92
|
+
vm['-setcookie']
|
93
|
+
end
|
94
|
+
|
95
|
+
# The source of the Riak installation from where the {Node} will
|
96
|
+
# be generated. This should point to the directory that contains
|
97
|
+
# the 'riak[search]' and 'riak[search]-admin' scripts.
|
98
|
+
# @return [Pathname] the source Riak installation
|
99
|
+
attr_reader :source
|
100
|
+
|
101
|
+
# The root directory of the {Node}, where all files are placed
|
102
|
+
# after generation.
|
103
|
+
# @return [Pathname] the root directory of the node
|
104
|
+
attr_reader :root
|
105
|
+
|
106
|
+
# The script for starting, stopping and pinging the Node.
|
107
|
+
# @return [Pathname] the path to the control script
|
108
|
+
def control_script
|
109
|
+
@control_script ||= root + 'bin' + control_script_name
|
110
|
+
end
|
111
|
+
|
112
|
+
# The name of the 'riak' or 'riaksearch' control script.
|
113
|
+
# @return [String] 'riak' or 'riaksearch'
|
114
|
+
def control_script_name
|
115
|
+
@control_script_name ||= (source + 'riaksearch').exist? ? 'riaksearch' : 'riak'
|
116
|
+
end
|
117
|
+
|
118
|
+
# The script for controlling non-lifecycle features of Riak like
|
119
|
+
# joining, leaving, status, ringready, etc.
|
120
|
+
# @return [Pathname] the path to the administrative script
|
121
|
+
def admin_script
|
122
|
+
@admin_script ||= root + 'bin' + "#{control_script_name}-admin"
|
123
|
+
end
|
124
|
+
|
125
|
+
# The "manifest" file where the node configuration will be
|
126
|
+
# written.
|
127
|
+
# @return [Pathname] the path to the manifest
|
128
|
+
def manifest
|
129
|
+
root + '.node.yml'
|
130
|
+
end
|
131
|
+
|
132
|
+
protected
|
133
|
+
# Populates the proper node configuration from the input config.
|
134
|
+
def configure(hash)
|
135
|
+
raise ArgumentError, t('source_and_root_required') unless hash[:source] && hash[:root]
|
136
|
+
@configuration = hash
|
137
|
+
configure_paths
|
138
|
+
configure_manifest
|
139
|
+
configure_settings
|
140
|
+
configure_logging
|
141
|
+
configure_data
|
142
|
+
configure_ports(hash[:interface], hash[:min_port])
|
143
|
+
configure_name(hash[:interface])
|
144
|
+
end
|
145
|
+
|
146
|
+
# Reads the manifest if it exists, overrides the passed configuration.
|
147
|
+
def configure_manifest
|
148
|
+
@configuration = YAML.load_file(manifest.to_s) if exist?
|
149
|
+
end
|
150
|
+
|
151
|
+
# Sets the data directories for the various on-disk backends and
|
152
|
+
# the ring state.
|
153
|
+
def configure_data
|
154
|
+
[:bitcask, :eleveldb, :merge_index].each {|k| env[k] ||= {} }
|
155
|
+
env[:bitcask][:data_root] ||= (data + 'bitcask').expand_path.to_s
|
156
|
+
env[:eleveldb][:data_root] ||= (data + 'leveldb').expand_path.to_s
|
157
|
+
env[:merge_index][:data_root] ||= (data + 'merge_index').expand_path.to_s
|
158
|
+
env[:riak_core][:ring_state_dir] ||= ring.expand_path.to_s
|
159
|
+
NODE_DIRECTORIES.each do |dir|
|
160
|
+
next if [:ring, :pipe].include?(dir)
|
161
|
+
env[:riak_core][:"platform_#{dir}_dir"] ||= send(dir).to_s
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
# Sets directories and handlers for logging.
|
166
|
+
def configure_logging
|
167
|
+
if env[:lager]
|
168
|
+
env[:lager][:handlers] = {
|
169
|
+
:lager_file_backend => [
|
170
|
+
Tuple[(log+"error.log").expand_path.to_s, :error],
|
171
|
+
Tuple[(log+"console.log").expand_path.to_s, :info]
|
172
|
+
]
|
173
|
+
}
|
174
|
+
env[:lager][:crash_log] = (log+"crash.log").to_s
|
175
|
+
else
|
176
|
+
# TODO: Need a better way to detect this, the defaults point
|
177
|
+
# to 1.0-style configs. Maybe there should be some kind of
|
178
|
+
# detection routine.
|
179
|
+
# Use sasl error logger for 0.14.
|
180
|
+
env[:riak_err] ||= {
|
181
|
+
:term_max_size => 65536,
|
182
|
+
:fmt_max_bytes => 65536
|
183
|
+
}
|
184
|
+
env[:sasl] = {
|
185
|
+
:sasl_error_logger => Tuple[:file, (log+"sasl-error.log").expand_path.to_s],
|
186
|
+
:errlog_type => :error,
|
187
|
+
:error_logger_mf_dir => (log+"sasl").expand_path.to_s,
|
188
|
+
:error_logger_mf_maxbytes => 10485760,
|
189
|
+
:error_logger_mf_maxfiles => 5
|
190
|
+
}
|
191
|
+
end
|
192
|
+
vm['-env ERL_CRASH_DUMP'] = (log + 'erl_crash.dump').to_s
|
193
|
+
end
|
194
|
+
|
195
|
+
# Sets the node name and cookie for distributed Erlang.
|
196
|
+
def configure_name(interface)
|
197
|
+
interface ||= "127.0.0.1"
|
198
|
+
vm["-name"] ||= configuration[:name] || "riak#{rand(1000000).to_s}@#{interface}"
|
199
|
+
vm["-setcookie"] ||= configuration[:cookie] || "#{rand(100000).to_s}_#{rand(1000000).to_s}"
|
200
|
+
end
|
201
|
+
|
202
|
+
# Merges input configuration with the defaults.
|
203
|
+
def configure_settings
|
204
|
+
@env = deep_merge(env.dup, configuration[:env]) if configuration[:env]
|
205
|
+
@vm = vm.merge(configuration[:vm]) if configuration[:vm]
|
206
|
+
end
|
207
|
+
|
208
|
+
# Sets the source directory and root directory of the generated node.
|
209
|
+
def configure_paths
|
210
|
+
@source = Pathname.new(configuration[:source]).expand_path
|
211
|
+
@root = Pathname.new(configuration[:root]).expand_path
|
212
|
+
end
|
213
|
+
|
214
|
+
# Sets ports and interfaces for http, protocol buffers, and handoff.
|
215
|
+
def configure_ports(interface, min_port)
|
216
|
+
interface ||= "127.0.0.1"
|
217
|
+
min_port ||= 8080
|
218
|
+
unless env[:riak_core][:http]
|
219
|
+
env[:riak_core][:http] = [Tuple[interface, min_port]]
|
220
|
+
min_port += 1
|
221
|
+
end
|
222
|
+
env[:riak_core][:http] = env[:riak_core][:http].map {|pair| Tuple[*pair] }
|
223
|
+
env[:riak_kv][:pb_ip] = interface unless env[:riak_kv][:pb_ip]
|
224
|
+
unless env[:riak_kv][:pb_port]
|
225
|
+
env[:riak_kv][:pb_port] = min_port
|
226
|
+
min_port += 1
|
227
|
+
end
|
228
|
+
unless env[:riak_core][:handoff_port]
|
229
|
+
env[:riak_core][:handoff_port] = min_port
|
230
|
+
min_port += 1
|
231
|
+
end
|
232
|
+
end
|
233
|
+
|
234
|
+
# Implements a deep-merge of two {Hash} instances.
|
235
|
+
# @param [Hash] source the original hash
|
236
|
+
# @param [Hash] target the new hash
|
237
|
+
# @return [Hash] a {Hash} whose {Hash} values have also been merged
|
238
|
+
def deep_merge(source, target)
|
239
|
+
source.merge(target) do |key, old_val, new_val|
|
240
|
+
if Hash === old_val && Hash === new_val
|
241
|
+
deep_merge(old_val, new_val)
|
242
|
+
else
|
243
|
+
new_val
|
244
|
+
end
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
248
|
+
# This class lets us specify that some settings should be emitted
|
249
|
+
# as Erlang tuples, even though the first element is not
|
250
|
+
# necessarily a Symbol.
|
251
|
+
class Tuple < Array; end
|
252
|
+
|
253
|
+
# Recursively converts a {Hash} into an Erlang configuration
|
254
|
+
# string that is appropriate for the app.config file.
|
255
|
+
# @param [Hash] hash a collection of configuration values
|
256
|
+
# @param [Fixnum] depth the current nesting level of
|
257
|
+
# generation/indentation
|
258
|
+
# @return [String] Erlang proplists in a String for use in
|
259
|
+
# app.config
|
260
|
+
def to_erlang_config(hash, depth = 1)
|
261
|
+
padding = ' ' * depth
|
262
|
+
parent_padding = ' ' * (depth-1)
|
263
|
+
values = hash.map do |k,v|
|
264
|
+
"{#{k}, #{value_to_erlang(v, depth)}}"
|
265
|
+
end.join(",\n#{padding}")
|
266
|
+
"[\n#{padding}#{values}\n#{parent_padding}]"
|
267
|
+
end
|
268
|
+
|
269
|
+
# Converts a value to an Erlang term. Mutually recursive with
|
270
|
+
# {#to_erlang_config}.
|
271
|
+
def value_to_erlang(v, depth=1)
|
272
|
+
case v
|
273
|
+
when Hash
|
274
|
+
to_erlang_config(v, depth+1)
|
275
|
+
when String
|
276
|
+
"\"#{v}\""
|
277
|
+
when Tuple
|
278
|
+
"{" << v.map {|i| value_to_erlang(i, depth+1) }.join(", ") << "}"
|
279
|
+
when Array
|
280
|
+
"[" << v.map {|i| value_to_erlang(i, depth+1) }.join(", ") << "]"
|
281
|
+
else
|
282
|
+
v.to_s
|
283
|
+
end
|
284
|
+
end
|
285
|
+
end
|
286
|
+
end
|