riak-client 0.9.8 → 1.0.0.beta

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. data/.gitignore +32 -0
  2. data/Gemfile +17 -11
  3. data/Guardfile +14 -0
  4. data/Rakefile +18 -44
  5. data/erl_src/riak_kv_test_backend.beam +0 -0
  6. data/erl_src/riak_kv_test_backend.erl +461 -128
  7. data/erl_src/riak_search_test_backend.beam +0 -0
  8. data/erl_src/riak_search_test_backend.erl +175 -0
  9. data/lib/active_support/cache/riak_store.rb +0 -13
  10. data/lib/riak.rb +11 -16
  11. data/lib/riak/bucket.rb +59 -41
  12. data/lib/riak/cache_store.rb +1 -14
  13. data/lib/riak/client.rb +145 -73
  14. data/lib/riak/client/beefcake/messages.rb +36 -31
  15. data/lib/riak/client/beefcake/object_methods.rb +27 -19
  16. data/lib/riak/client/beefcake_protobuffs_backend.rb +27 -33
  17. data/lib/riak/client/excon_backend.rb +0 -13
  18. data/lib/riak/client/http_backend.rb +95 -60
  19. data/lib/riak/client/http_backend/configuration.rb +144 -19
  20. data/lib/riak/client/http_backend/key_streamer.rb +1 -14
  21. data/lib/riak/client/http_backend/object_methods.rb +16 -16
  22. data/lib/riak/client/http_backend/request_headers.rb +0 -13
  23. data/lib/riak/client/http_backend/transport_methods.rb +26 -56
  24. data/lib/riak/client/net_http_backend.rb +11 -13
  25. data/lib/riak/client/protobuffs_backend.rb +21 -19
  26. data/lib/riak/client/pump.rb +1 -15
  27. data/lib/riak/client/search.rb +85 -0
  28. data/lib/riak/cluster.rb +151 -0
  29. data/lib/riak/core_ext.rb +1 -0
  30. data/lib/riak/core_ext/deep_dup.rb +13 -0
  31. data/lib/riak/core_ext/json.rb +15 -0
  32. data/lib/riak/core_ext/stringify_keys.rb +1 -1
  33. data/lib/riak/core_ext/symbolize_keys.rb +1 -1
  34. data/lib/riak/encoding.rb +6 -0
  35. data/lib/riak/failed_request.rb +2 -15
  36. data/lib/riak/i18n.rb +0 -13
  37. data/lib/riak/json.rb +19 -8
  38. data/lib/riak/link.rb +18 -20
  39. data/lib/riak/locale/en.yml +13 -16
  40. data/lib/riak/map_reduce.rb +40 -20
  41. data/lib/riak/map_reduce/filter_builder.rb +14 -18
  42. data/lib/riak/map_reduce/phase.rb +0 -13
  43. data/lib/riak/map_reduce_error.rb +0 -13
  44. data/lib/riak/node.rb +38 -0
  45. data/lib/riak/node/configuration.rb +286 -0
  46. data/lib/riak/node/console.rb +139 -0
  47. data/lib/riak/node/control.rb +207 -0
  48. data/lib/riak/node/defaults.rb +70 -0
  49. data/lib/riak/node/generation.rb +99 -0
  50. data/lib/riak/node/log.rb +34 -0
  51. data/lib/riak/node/version.rb +37 -0
  52. data/lib/riak/robject.rb +45 -41
  53. data/lib/riak/search.rb +2 -161
  54. data/lib/riak/serializers.rb +74 -0
  55. data/lib/riak/stamp.rb +77 -0
  56. data/lib/riak/test_server.rb +56 -220
  57. data/lib/riak/util/escape.rb +58 -17
  58. data/lib/riak/util/headers.rb +2 -15
  59. data/lib/riak/util/multipart.rb +0 -13
  60. data/lib/riak/util/multipart/stream_parser.rb +0 -13
  61. data/lib/riak/util/tcp_socket_extensions.rb +1 -14
  62. data/lib/riak/util/translation.rb +0 -13
  63. data/lib/riak/version.rb +3 -0
  64. data/lib/riak/walk_spec.rb +0 -13
  65. data/riak-client.gemspec +27 -47
  66. data/spec/fixtures/multipart-with-marked-tombstones.txt +17 -0
  67. data/spec/fixtures/multipart-with-unmarked-tombstone.txt +16 -0
  68. data/spec/integration/riak/cache_store_spec.rb +2 -40
  69. data/spec/integration/riak/cluster_spec.rb +88 -0
  70. data/spec/integration/riak/http_backends_spec.rb +6 -30
  71. data/spec/integration/riak/node_spec.rb +184 -0
  72. data/spec/integration/riak/protobuffs_backends_spec.rb +2 -26
  73. data/spec/integration/riak/test_server_spec.rb +31 -167
  74. data/spec/riak/beefcake_protobuffs_backend_spec.rb +5 -4
  75. data/spec/riak/bucket_spec.rb +26 -36
  76. data/spec/riak/client_spec.rb +44 -38
  77. data/spec/riak/escape_spec.rb +56 -30
  78. data/spec/riak/excon_backend_spec.rb +4 -17
  79. data/spec/riak/headers_spec.rb +1 -14
  80. data/spec/riak/http_backend/configuration_spec.rb +211 -34
  81. data/spec/riak/http_backend/object_methods_spec.rb +52 -18
  82. data/spec/riak/http_backend/transport_methods_spec.rb +5 -38
  83. data/spec/riak/http_backend_spec.rb +84 -78
  84. data/spec/riak/link_spec.rb +19 -18
  85. data/spec/riak/map_reduce/filter_builder_spec.rb +1 -14
  86. data/spec/riak/map_reduce/phase_spec.rb +1 -14
  87. data/spec/riak/map_reduce_spec.rb +141 -43
  88. data/spec/riak/multipart_spec.rb +1 -14
  89. data/spec/riak/net_http_backend_spec.rb +2 -15
  90. data/spec/riak/robject_spec.rb +129 -97
  91. data/spec/riak/search_spec.rb +45 -62
  92. data/spec/riak/serializers_spec.rb +93 -0
  93. data/spec/riak/stamp_spec.rb +54 -0
  94. data/spec/riak/stream_parser_spec.rb +3 -16
  95. data/spec/riak/walk_spec_spec.rb +1 -14
  96. data/spec/spec_helper.rb +22 -27
  97. data/spec/support/http_backend_implementation_examples.rb +49 -79
  98. data/spec/support/integration_setup.rb +10 -0
  99. data/spec/support/mock_server.rb +0 -14
  100. data/spec/support/mocks.rb +0 -13
  101. data/spec/support/test_server.rb +30 -0
  102. data/spec/support/test_server.yml.example +14 -2
  103. data/spec/support/unified_backend_examples.rb +36 -27
  104. metadata +100 -31
  105. data/lib/riak/client/curb_backend.rb +0 -89
  106. data/spec/riak/curb_backend_spec.rb +0 -76
@@ -1,11 +1,6 @@
1
- # Load JSON
2
- unless defined? JSON
3
- begin
4
- require 'yajl/json_gem'
5
- rescue LoadError
6
- require 'json'
7
- end
8
- end
1
+ require 'multi_json'
2
+ MultiJson.engine # Force loading of an engine
3
+ require 'riak/core_ext/json'
9
4
 
10
5
  module Riak
11
6
  class << self
@@ -14,4 +9,20 @@ module Riak
14
9
  attr_accessor :json_options
15
10
  end
16
11
  self.json_options = {:max_nesting => 20}
12
+
13
+ # JSON module for internal use inside riak-client
14
+ module JSON
15
+ class << self
16
+ # Parse a JSON string
17
+ def parse(str)
18
+ MultiJson.decode(str, Riak.json_options)
19
+ end
20
+
21
+ # Generate a JSON string
22
+ def encode(obj)
23
+ MultiJson.encode(obj)
24
+ end
25
+ alias :dump :encode
26
+ end
27
+ end
17
28
  end
@@ -1,16 +1,3 @@
1
- # Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
1
 
15
2
  require 'riak/util/translation'
16
3
  require 'riak/util/escape'
@@ -34,7 +21,10 @@ module Riak
34
21
  attr_accessor :key
35
22
 
36
23
  %w{bucket key}.each do |m|
37
- class_eval %{ def #{m}=(value); @url = nil; @#{m} = value; end }
24
+ define_method("#{m}=") { |value|
25
+ @url = nil
26
+ instance_variable_set("@#{m}", value)
27
+ }
38
28
  end
39
29
 
40
30
  # @param [String] header_string the string value of the Link: HTTP header from a Riak response
@@ -62,20 +52,28 @@ module Riak
62
52
  end
63
53
 
64
54
  # @return [String] the URL (relative or absolute) of the related resource
65
- def url
66
- @url ||= "/riak/#{escape(bucket)}" + (key.blank? ? "" : "/#{escape(key)}")
55
+ def url(new_scheme=false)
56
+ if @bucket
57
+ if new_scheme
58
+ "/buckets/#{escape(bucket)}" + (key.blank? ? "" : "/keys/#{escape(key)}")
59
+ else
60
+ "/riak/#{escape(bucket)}" + (key.blank? ? "" : "/#{escape(key)}")
61
+ end
62
+ else
63
+ @url
64
+ end
67
65
  end
68
66
 
69
67
  def url=(value)
70
68
  @url = value
71
- @bucket = unescape($1) if value =~ %r{^/[^/]+/([^/]+)/?}
72
- @key = unescape($1) if value =~ %r{^/[^/]+/[^/]+/([^/]+)/?}
69
+ @bucket = unescape($1) if value =~ %r{^/buckets/([^/]+)/?} || value =~ %r{^/[^/]+/([^/]+)/?}
70
+ @key = unescape($1) if value =~ %r{^/buckets/[^/]+/keys/([^/]+)/?} || value =~ %r{^/[^/]+/[^/]+/([^/]+)/?}
73
71
  end
74
72
 
75
73
  def inspect; to_s; end
76
74
 
77
- def to_s
78
- %Q[<#{url}>; riaktag="#{tag}"]
75
+ def to_s(new_scheme=false)
76
+ %Q[<#{url(new_scheme)}>; riaktag="#{tag}"]
79
77
  end
80
78
 
81
79
  def hash
@@ -1,27 +1,18 @@
1
- # Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
1
  en:
15
2
  riak:
3
+ backwards_clock: "System clock moved backwards, ID generation will fail for %{delay} more milliseconds."
16
4
  bucket_link_conversion: "Can't convert a bucket link to a walk spec"
17
5
  client_type: "invalid argument %{client} is not a Riak::Client"
6
+ conflict_resolver_invalid: "The given resolver (%{resolver}) did not respond to :call"
18
7
  content_type_undefined: "content_type is not defined!"
19
8
  deprecated:
20
9
  port: "DEPRECATION: Riak::Client#port has been deprecated, use #http_port or #pb_port for the appropriate protocol.\n%{backtrace}"
10
+ search: "DEPRECATION: Riak Search features are included in the main client, you no longer need to require 'riak/search'.\n%{backtrace}"
21
11
  empty_map_reduce_query: "Specify one or more query phases to your MapReduce."
22
12
  failed_request: "Client request failed."
23
13
  filter_needs_block: "Filter %{filter} expects a block."
24
14
  filter_arity_mismatch: "Filter %{filter} expects %{expected} arguments but %{received} were given."
15
+ full_bucket_mapred: "Full-bucket MapReduce, including key filters, invokes list-keys which is an expensive operation that should not be used in production.\n %{backtrace}"
25
16
  hash_type: "invalid argument %{hash} is not a Hash"
26
17
  http_configuration: "The %{backend} HTTP backend cannot be used. Please check its requirements."
27
18
  http_failed_request: "Expected %{expected} from Riak but received %{code}. %{body}"
@@ -29,23 +20,29 @@ en:
29
20
  protocol_invalid: "'%{invalid}' is not a valid protocol, valid values are %{valid}"
30
21
  invalid_basic_auth: "basic auth must be set using 'user:pass'"
31
22
  invalid_client_id: "Invalid client ID, must be a string or between 0 and %{max_id}"
23
+ invalid_io_object: "Invalid IO-like object assigned to RObject#data. It should be assigned to raw_data instead."
32
24
  invalid_function_value: "invalid value for function: %{value}"
33
25
  invalid_options: "Invalid configuration options given."
34
26
  invalid_phase_type: "type must be :map, :reduce, or :link"
35
- invalid_response: "Expected %{expected} but received %{received} from Riak %{extra}"
36
27
  invalid_ssl_verify_mode: "%{invalid} is not a valid :verify_mode option for SSL. Valid options are 'peer' and 'none'."
28
+ invalid_index_query: "%{value} is not a valid index query term, only Strings, Integers, and Ranges of those are allowed."
29
+ indexes_unsupported: "Riak server does not support secondary indexes."
37
30
  loading_bucket: "while loading bucket '%{name}'"
31
+ list_buckets: "Riak::Client#buckets is an expensive operation that should not be used in production.\n %{backtrace}"
32
+ list_keys: "Riak::Bucket#keys is an expensive operation that should not be used in production.\n %{backtrace}"
38
33
  missing_block: "A block must be given."
39
34
  missing_host_and_port: "You must specify a host and port, or use the defaults of 127.0.0.1:8098"
40
35
  module_function_pair_required: "function must have two elements when an array"
41
36
  not_found: "The requested object was not found."
42
- path_and_body_required: "You must supply both a resource path and a body."
37
+ no_pipes: "Could not find or open pipes for Riak console in %{path}."
43
38
  port_invalid: "port must be an integer between 0 and 65535"
44
39
  protobuffs_failed_request: "Expected success from Riak but received %{code}. %{body}"
45
40
  request_body_type: "Request body must be a String or respond to :read."
46
- resource_path_short: "Resource path too short"
41
+ search_unsupported: "Riak server does not support search."
47
42
  search_docs_require_id: "Search index documents must include the 'id' field."
48
43
  search_remove_requires_id_or_query: "Search index documents to be removed must have 'id' or 'query' keys."
44
+ serializer_not_implemented: "No serializer has been registered for content type %{content_type}"
45
+ source_and_root_required: "Riak::Node configuration must include :source and :root keys."
49
46
  stale_write_prevented: "Stale write prevented by client."
50
47
  stored_function_invalid: "function must have :bucket and :key when a hash"
51
48
  string_type: "invalid_argument %{string} is not a String"
@@ -1,17 +1,3 @@
1
- # Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
-
15
1
  require 'riak/util/translation'
16
2
  require 'riak/util/escape'
17
3
  require 'riak/json'
@@ -81,20 +67,23 @@ module Riak
81
67
  p = params.first
82
68
  case p
83
69
  when Bucket
84
- @inputs = escape(p.name)
70
+ warn(t('full_bucket_mapred', :backtrace => caller.join("\n "))) unless Riak.disable_list_keys_warnings
71
+ @inputs = maybe_escape(p.name)
85
72
  when RObject
86
- @inputs << [escape(p.bucket.name), escape(p.key)]
73
+ @inputs << [maybe_escape(p.bucket.name), maybe_escape(p.key)]
87
74
  when String
88
- @inputs = escape(p)
75
+ warn(t('full_bucket_mapred', :backtrace => caller.join("\n "))) unless Riak.disable_list_keys_warnings
76
+ @inputs = maybe_escape(p)
89
77
  end
90
78
  when 2..3
91
79
  bucket = params.shift
92
80
  bucket = bucket.name if Bucket === bucket
93
81
  if Array === params.first
94
- @inputs = {:bucket => escape(bucket), :key_filters => params.first }
82
+ warn(t('full_bucket_mapred', :backtrace => caller.join("\n "))) unless Riak.disable_list_keys_warnings
83
+ @inputs = {:bucket => maybe_escape(bucket), :key_filters => params.first }
95
84
  else
96
85
  key = params.shift
97
- @inputs << params.unshift(escape(key)).unshift(escape(bucket))
86
+ @inputs << params.unshift(maybe_escape(key)).unshift(maybe_escape(bucket))
98
87
  end
99
88
  end
100
89
  self
@@ -112,6 +101,37 @@ module Riak
112
101
  add(bucket, FilterBuilder.new(&block).to_a)
113
102
  end
114
103
 
104
+ # (Riak Search) Use a search query to start a map/reduce job.
105
+ # @param [String, Bucket] bucket the bucket/index to search
106
+ # @param [String] query the query to run
107
+ # @return [MapReduce] self
108
+ def search(bucket, query)
109
+ bucket = bucket.name if bucket.respond_to?(:name)
110
+ @inputs = {:module => "riak_search", :function => "mapred_search", :arg => [bucket, query]}
111
+ self
112
+ end
113
+
114
+ # (Secondary Indexes) Use a secondary index query to start a
115
+ # map/reduce job.
116
+ # @param [String, Bucket] bucket the bucket whose index to query
117
+ # @param [String] index the index to query
118
+ # @param [String, Integer, Range] query the value of the index, or
119
+ # a range of values (of Strings or Integers)
120
+ # @return [MapReduce] self
121
+ def index(bucket, index, query)
122
+ bucket = bucket.name if bucket.respond_to?(:name)
123
+ case query
124
+ when String, Fixnum
125
+ @inputs = {:bucket => maybe_escape(bucket), :index => index, :key => query}
126
+ when Range
127
+ raise ArgumentError, t('invalid_index_query', :value => query.inspect) unless String === query.begin || Integer === query.begin
128
+ @inputs = {:bucket => maybe_escape(bucket), :index => index, :start => query.begin, :end => query.end}
129
+ else
130
+ raise ArgumentError, t('invalid_index_query', :value => query.inspect)
131
+ end
132
+ self
133
+ end
134
+
115
135
  # Add a map phase to the job.
116
136
  # @overload map(function)
117
137
  # @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module,function] pair
@@ -194,7 +214,7 @@ module Riak
194
214
  # @return [nil] nothing
195
215
  def run(&block)
196
216
  raise MapReduceError.new(t("empty_map_reduce_query")) if @query.empty?
197
- @client.backend.mapred(self, &block)
217
+ @client.mapred(self, &block)
198
218
  rescue FailedRequest => fr
199
219
  if fr.server_error? && fr.is_json?
200
220
  raise MapReduceError.new(fr.body)
@@ -1,16 +1,3 @@
1
- # Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
1
 
15
2
  require 'riak/util/translation'
16
3
 
@@ -66,15 +53,24 @@ module Riak
66
53
  LOGICAL_OPERATIONS = %w{and or not}
67
54
 
68
55
  FILTERS.each do |f,arity|
69
- class_eval <<-CODE
70
- def #{f}(*args)
71
- raise ArgumentError.new(t("filter_arity_mismatch", :filter => :#{f}, :expected => #{arity.inspect}, :received => args.size)) unless #{arity.inspect} == -1 || Array(#{arity.inspect}).include?(args.size)
72
- @filters << ([:#{f}] + args)
56
+ arities = [arity].flatten
57
+
58
+ define_method(f) { |*args|
59
+ unless arities.include?(-1) or arities.include?(args.size)
60
+ raise ArgumentError.new t("filter_arity_mismatch",
61
+ :filter => f,
62
+ :expected => arities,
63
+ :received => args.size
64
+ )
73
65
  end
74
- CODE
66
+
67
+ @filters << [f, *args]
68
+ }
75
69
  end
76
70
 
77
71
  LOGICAL_OPERATIONS.each do |op|
72
+ # NB: string eval is needed here because in ruby 1.8, blocks can't yield to
73
+ # other blocks
78
74
  class_eval <<-CODE
79
75
  def _#{op}(&block)
80
76
  raise ArgumentError.new(t('filter_needs_block', :filter => '#{op}')) unless block_given?
@@ -1,16 +1,3 @@
1
- # Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
1
 
15
2
  require 'riak/json'
16
3
  require 'riak/util/translation'
@@ -1,16 +1,3 @@
1
- # Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
1
  require 'riak/util/translation'
15
2
 
16
3
  module Riak
@@ -0,0 +1,38 @@
1
+ require 'riak/util/translation'
2
+ require 'riak/node/defaults'
3
+ require 'riak/node/configuration'
4
+ require 'riak/node/generation'
5
+ require 'riak/node/control'
6
+ require 'riak/node/version'
7
+ require 'riak/node/log'
8
+
9
+ module Riak
10
+ # A Node encapsulates the generation and management of standalone
11
+ # Riak nodes. It is used by the {TestServer} to start and manage an
12
+ # in-memory node for supporting integration test suites.
13
+ class Node
14
+ include Util::Translation
15
+
16
+ # Creates a new Riak node. Unlike {#new}, this will also generate
17
+ # the node if it does not exist on disk. Equivalent to {::new}
18
+ # followed by {#create}.
19
+ # @see #new
20
+ def self.create(configuration={})
21
+ new(configuration).tap do |node|
22
+ node.create
23
+ end
24
+ end
25
+
26
+ # Creates the template for a Riak node. To generate the node after
27
+ # initialization, use {#create}.
28
+ def initialize(configuration={})
29
+ set_defaults
30
+ configure configuration
31
+ end
32
+
33
+ protected
34
+ def debug(msg)
35
+ $stderr.puts msg if ENV["DEBUG_RIAK_NODE"]
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,286 @@
1
+ require 'pathname'
2
+ require 'yaml'
3
+
4
+ module Riak
5
+ class Node
6
+ # The directories (and accessor methods) that will be created
7
+ # under the generated node.
8
+ NODE_DIRECTORIES = [:bin, :etc, :log, :data, :ring, :pipe]
9
+
10
+ NODE_DIRECTORIES.each do |dir|
11
+ # Makes accessor methods for all the node directories that
12
+ # return Pathname objects.
13
+ class_eval %Q{
14
+ def #{dir}
15
+ root + '#{dir}'
16
+ end
17
+ }
18
+ end
19
+
20
+ # @return [Hash] the contents of the Erlang environment, which will
21
+ # be created into the app.config file.
22
+ attr_reader :env
23
+
24
+ # @return [Hash] the command-line switches for the Erlang virtual
25
+ # machine, which will be created into the vm.args file
26
+ attr_reader :vm
27
+
28
+ # @return [Hash] the configuration that was passed to the Node
29
+ # when initialized
30
+ attr_reader :configuration
31
+
32
+ # @return [Array<Pathname>] where user Erlang code will be loaded from
33
+ def erlang_sources
34
+ env[:riak_kv][:add_paths].map {|p| Pathname.new(p) }
35
+ end
36
+
37
+ # @return [Pathname] where user Javascript code will be loaded from
38
+ def javascript_source
39
+ Pathname.new(env[:riak_kv][:js_source_dir])
40
+ end
41
+
42
+ # @return [Fixnum] the size of the ring, i.e. number of partitions
43
+ def ring_size
44
+ env[:riak_core][:ring_creation_size]
45
+ end
46
+
47
+ # @return [Fixnum] The port used for handing off data to other nodes.
48
+ def handoff_port
49
+ env[:riak_core][:handoff_port]
50
+ end
51
+
52
+ # @return [Fixnum] The port to which the HTTP API is connected.
53
+ def http_port
54
+ # We'll only support 0.14 and later, which uses http rather than web_ip/web_port
55
+ env[:riak_core][:http][0][1]
56
+ end
57
+
58
+ # @return [Fixnum] the port to which the Protocol Buffers API is connected.
59
+ def pb_port
60
+ env[:riak_kv][:pb_port]
61
+ end
62
+
63
+ # @return [String] the interface to which the HTTP API is connected
64
+ def http_ip
65
+ env[:riak_core][:http][0][0]
66
+ end
67
+
68
+ # @return [String] the interface to which the Protocol Buffers API is connected
69
+ def pb_ip
70
+ env[:riak_kv][:pb_ip]
71
+ end
72
+
73
+ # @return [Symbol] the storage backend for Riak Search.
74
+ def search_backend
75
+ env[:riak_search][:search_backend]
76
+ end
77
+
78
+ # @return [Symbol] the storage backend for Riak KV.
79
+ def kv_backend
80
+ env[:riak_kv][:storage_backend]
81
+ end
82
+
83
+ # @return [String] the name of the Riak node as seen by distributed Erlang
84
+ # communication. AKA "-name" in vm.args.
85
+ def name
86
+ vm['-name']
87
+ end
88
+
89
+ # @return [String] the cookie/shared secret used for connecting
90
+ # a cluster
91
+ def cookie
92
+ vm['-setcookie']
93
+ end
94
+
95
+ # The source of the Riak installation from where the {Node} will
96
+ # be generated. This should point to the directory that contains
97
+ # the 'riak[search]' and 'riak[search]-admin' scripts.
98
+ # @return [Pathname] the source Riak installation
99
+ attr_reader :source
100
+
101
+ # The root directory of the {Node}, where all files are placed
102
+ # after generation.
103
+ # @return [Pathname] the root directory of the node
104
+ attr_reader :root
105
+
106
+ # The script for starting, stopping and pinging the Node.
107
+ # @return [Pathname] the path to the control script
108
+ def control_script
109
+ @control_script ||= root + 'bin' + control_script_name
110
+ end
111
+
112
+ # The name of the 'riak' or 'riaksearch' control script.
113
+ # @return [String] 'riak' or 'riaksearch'
114
+ def control_script_name
115
+ @control_script_name ||= (source + 'riaksearch').exist? ? 'riaksearch' : 'riak'
116
+ end
117
+
118
+ # The script for controlling non-lifecycle features of Riak like
119
+ # joining, leaving, status, ringready, etc.
120
+ # @return [Pathname] the path to the administrative script
121
+ def admin_script
122
+ @admin_script ||= root + 'bin' + "#{control_script_name}-admin"
123
+ end
124
+
125
+ # The "manifest" file where the node configuration will be
126
+ # written.
127
+ # @return [Pathname] the path to the manifest
128
+ def manifest
129
+ root + '.node.yml'
130
+ end
131
+
132
+ protected
133
+ # Populates the proper node configuration from the input config.
134
+ def configure(hash)
135
+ raise ArgumentError, t('source_and_root_required') unless hash[:source] && hash[:root]
136
+ @configuration = hash
137
+ configure_paths
138
+ configure_manifest
139
+ configure_settings
140
+ configure_logging
141
+ configure_data
142
+ configure_ports(hash[:interface], hash[:min_port])
143
+ configure_name(hash[:interface])
144
+ end
145
+
146
+ # Reads the manifest if it exists, overrides the passed configuration.
147
+ def configure_manifest
148
+ @configuration = YAML.load_file(manifest.to_s) if exist?
149
+ end
150
+
151
+ # Sets the data directories for the various on-disk backends and
152
+ # the ring state.
153
+ def configure_data
154
+ [:bitcask, :eleveldb, :merge_index].each {|k| env[k] ||= {} }
155
+ env[:bitcask][:data_root] ||= (data + 'bitcask').expand_path.to_s
156
+ env[:eleveldb][:data_root] ||= (data + 'leveldb').expand_path.to_s
157
+ env[:merge_index][:data_root] ||= (data + 'merge_index').expand_path.to_s
158
+ env[:riak_core][:ring_state_dir] ||= ring.expand_path.to_s
159
+ NODE_DIRECTORIES.each do |dir|
160
+ next if [:ring, :pipe].include?(dir)
161
+ env[:riak_core][:"platform_#{dir}_dir"] ||= send(dir).to_s
162
+ end
163
+ end
164
+
165
+ # Sets directories and handlers for logging.
166
+ def configure_logging
167
+ if env[:lager]
168
+ env[:lager][:handlers] = {
169
+ :lager_file_backend => [
170
+ Tuple[(log+"error.log").expand_path.to_s, :error],
171
+ Tuple[(log+"console.log").expand_path.to_s, :info]
172
+ ]
173
+ }
174
+ env[:lager][:crash_log] = (log+"crash.log").to_s
175
+ else
176
+ # TODO: Need a better way to detect this, the defaults point
177
+ # to 1.0-style configs. Maybe there should be some kind of
178
+ # detection routine.
179
+ # Use sasl error logger for 0.14.
180
+ env[:riak_err] ||= {
181
+ :term_max_size => 65536,
182
+ :fmt_max_bytes => 65536
183
+ }
184
+ env[:sasl] = {
185
+ :sasl_error_logger => Tuple[:file, (log+"sasl-error.log").expand_path.to_s],
186
+ :errlog_type => :error,
187
+ :error_logger_mf_dir => (log+"sasl").expand_path.to_s,
188
+ :error_logger_mf_maxbytes => 10485760,
189
+ :error_logger_mf_maxfiles => 5
190
+ }
191
+ end
192
+ vm['-env ERL_CRASH_DUMP'] = (log + 'erl_crash.dump').to_s
193
+ end
194
+
195
+ # Sets the node name and cookie for distributed Erlang.
196
+ def configure_name(interface)
197
+ interface ||= "127.0.0.1"
198
+ vm["-name"] ||= configuration[:name] || "riak#{rand(1000000).to_s}@#{interface}"
199
+ vm["-setcookie"] ||= configuration[:cookie] || "#{rand(100000).to_s}_#{rand(1000000).to_s}"
200
+ end
201
+
202
+ # Merges input configuration with the defaults.
203
+ def configure_settings
204
+ @env = deep_merge(env.dup, configuration[:env]) if configuration[:env]
205
+ @vm = vm.merge(configuration[:vm]) if configuration[:vm]
206
+ end
207
+
208
+ # Sets the source directory and root directory of the generated node.
209
+ def configure_paths
210
+ @source = Pathname.new(configuration[:source]).expand_path
211
+ @root = Pathname.new(configuration[:root]).expand_path
212
+ end
213
+
214
+ # Sets ports and interfaces for http, protocol buffers, and handoff.
215
+ def configure_ports(interface, min_port)
216
+ interface ||= "127.0.0.1"
217
+ min_port ||= 8080
218
+ unless env[:riak_core][:http]
219
+ env[:riak_core][:http] = [Tuple[interface, min_port]]
220
+ min_port += 1
221
+ end
222
+ env[:riak_core][:http] = env[:riak_core][:http].map {|pair| Tuple[*pair] }
223
+ env[:riak_kv][:pb_ip] = interface unless env[:riak_kv][:pb_ip]
224
+ unless env[:riak_kv][:pb_port]
225
+ env[:riak_kv][:pb_port] = min_port
226
+ min_port += 1
227
+ end
228
+ unless env[:riak_core][:handoff_port]
229
+ env[:riak_core][:handoff_port] = min_port
230
+ min_port += 1
231
+ end
232
+ end
233
+
234
+ # Implements a deep-merge of two {Hash} instances.
235
+ # @param [Hash] source the original hash
236
+ # @param [Hash] target the new hash
237
+ # @return [Hash] a {Hash} whose {Hash} values have also been merged
238
+ def deep_merge(source, target)
239
+ source.merge(target) do |key, old_val, new_val|
240
+ if Hash === old_val && Hash === new_val
241
+ deep_merge(old_val, new_val)
242
+ else
243
+ new_val
244
+ end
245
+ end
246
+ end
247
+
248
+ # This class lets us specify that some settings should be emitted
249
+ # as Erlang tuples, even though the first element is not
250
+ # necessarily a Symbol.
251
+ class Tuple < Array; end
252
+
253
+ # Recursively converts a {Hash} into an Erlang configuration
254
+ # string that is appropriate for the app.config file.
255
+ # @param [Hash] hash a collection of configuration values
256
+ # @param [Fixnum] depth the current nesting level of
257
+ # generation/indentation
258
+ # @return [String] Erlang proplists in a String for use in
259
+ # app.config
260
+ def to_erlang_config(hash, depth = 1)
261
+ padding = ' ' * depth
262
+ parent_padding = ' ' * (depth-1)
263
+ values = hash.map do |k,v|
264
+ "{#{k}, #{value_to_erlang(v, depth)}}"
265
+ end.join(",\n#{padding}")
266
+ "[\n#{padding}#{values}\n#{parent_padding}]"
267
+ end
268
+
269
+ # Converts a value to an Erlang term. Mutually recursive with
270
+ # {#to_erlang_config}.
271
+ def value_to_erlang(v, depth=1)
272
+ case v
273
+ when Hash
274
+ to_erlang_config(v, depth+1)
275
+ when String
276
+ "\"#{v}\""
277
+ when Tuple
278
+ "{" << v.map {|i| value_to_erlang(i, depth+1) }.join(", ") << "}"
279
+ when Array
280
+ "[" << v.map {|i| value_to_erlang(i, depth+1) }.join(", ") << "]"
281
+ else
282
+ v.to_s
283
+ end
284
+ end
285
+ end
286
+ end