riak-client 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/Gemfile +1 -7
  4. data/README.markdown +66 -0
  5. data/RELEASE_NOTES.md +27 -0
  6. data/lib/riak/bucket.rb +24 -5
  7. data/lib/riak/client.rb +42 -7
  8. data/lib/riak/client/beefcake/message_codes.rb +56 -0
  9. data/lib/riak/client/beefcake/messages.rb +190 -18
  10. data/lib/riak/client/beefcake_protobuffs_backend.rb +143 -10
  11. data/lib/riak/client/feature_detection.rb +26 -1
  12. data/lib/riak/client/http_backend.rb +58 -9
  13. data/lib/riak/client/http_backend/bucket_streamer.rb +15 -0
  14. data/lib/riak/client/http_backend/chunked_json_streamer.rb +42 -0
  15. data/lib/riak/client/http_backend/configuration.rb +17 -1
  16. data/lib/riak/client/http_backend/key_streamer.rb +4 -32
  17. data/lib/riak/client/protobuffs_backend.rb +12 -34
  18. data/lib/riak/counter.rb +101 -0
  19. data/lib/riak/index_collection.rb +71 -0
  20. data/lib/riak/list_buckets.rb +28 -0
  21. data/lib/riak/locale/en.yml +14 -0
  22. data/lib/riak/multiget.rb +123 -0
  23. data/lib/riak/node.rb +2 -0
  24. data/lib/riak/node/configuration.rb +32 -21
  25. data/lib/riak/node/defaults.rb +2 -0
  26. data/lib/riak/node/generation.rb +19 -7
  27. data/lib/riak/node/version.rb +2 -16
  28. data/lib/riak/robject.rb +1 -0
  29. data/lib/riak/secondary_index.rb +67 -0
  30. data/lib/riak/version.rb +1 -1
  31. data/riak-client.gemspec +3 -2
  32. data/spec/integration/riak/counters_spec.rb +51 -0
  33. data/spec/integration/riak/http_backends_spec.rb +24 -14
  34. data/spec/integration/riak/node_spec.rb +6 -28
  35. data/spec/riak/beefcake_protobuffs_backend_spec.rb +84 -0
  36. data/spec/riak/bucket_spec.rb +55 -5
  37. data/spec/riak/client_spec.rb +34 -0
  38. data/spec/riak/counter_spec.rb +122 -0
  39. data/spec/riak/index_collection_spec.rb +50 -0
  40. data/spec/riak/list_buckets_spec.rb +41 -0
  41. data/spec/riak/multiget_spec.rb +76 -0
  42. data/spec/riak/robject_spec.rb +4 -1
  43. data/spec/riak/secondary_index_spec.rb +225 -0
  44. data/spec/spec_helper.rb +1 -0
  45. data/spec/support/sometimes.rb +2 -2
  46. data/spec/support/unified_backend_examples.rb +4 -0
  47. metadata +41 -47
@@ -1,3 +1,5 @@
1
+ require 'fileutils'
2
+
1
3
  require 'riak/util/translation'
2
4
  require 'riak/node/defaults'
3
5
  require 'riak/node/configuration'
@@ -3,16 +3,20 @@ require 'yaml'
3
3
 
4
4
  module Riak
5
5
  class Node
6
+
7
+ # do not copy these directories to the test node
8
+ NODE_DIR_SKIP_LIST = [:data, :pipe]
9
+
6
10
  # The directories (and accessor methods) that will be created
7
11
  # under the generated node.
8
- NODE_DIRECTORIES = [:bin, :etc, :log, :data, :ring, :pipe]
12
+ NODE_DIRECTORIES = [:bin, :etc, :log, :data, :pipe]
9
13
 
14
+ # Makes accessor methods for all the node directories that
15
+ # return Pathname objects.
10
16
  NODE_DIRECTORIES.each do |dir|
11
- # Makes accessor methods for all the node directories that
12
- # return Pathname objects.
13
17
  class_eval %Q{
14
18
  def #{dir}
15
- root + '#{dir}'
19
+ root + '#{dir}/'
16
20
  end
17
21
  }
18
22
  end
@@ -55,9 +59,14 @@ module Riak
55
59
  env[:riak_core][:http][0][1]
56
60
  end
57
61
 
62
+ def pb_config_section
63
+ return :riak_kv if version < '1.4.0'
64
+ return :riak_api
65
+ end
66
+
58
67
  # @return [Fixnum] the port to which the Protocol Buffers API is connected.
59
68
  def pb_port
60
- env[:riak_kv][:pb_port]
69
+ env[pb_config_section][:pb_port]
61
70
  end
62
71
 
63
72
  # @return [String] the interface to which the HTTP API is connected
@@ -67,7 +76,7 @@ module Riak
67
76
 
68
77
  # @return [String] the interface to which the Protocol Buffers API is connected
69
78
  def pb_ip
70
- env[:riak_kv][:pb_ip]
79
+ env[pb_config_section][:pb_ip]
71
80
  end
72
81
 
73
82
  # @return [Symbol] the storage backend for Riak Search.
@@ -93,8 +102,7 @@ module Riak
93
102
  end
94
103
 
95
104
  # The source of the Riak installation from where the {Node} will
96
- # be generated. This should point to the directory that contains
97
- # the 'riak[search]' and 'riak[search]-admin' scripts.
105
+ # be generated.
98
106
  # @return [Pathname] the source Riak installation
99
107
  attr_reader :source
100
108
 
@@ -103,16 +111,19 @@ module Riak
103
111
  # @return [Pathname] the root directory of the node
104
112
  attr_reader :root
105
113
 
114
+ def env_script
115
+ @env_script ||= root + 'lib' + 'env.sh'
116
+ end
117
+
106
118
  # The script for starting, stopping and pinging the Node.
107
119
  # @return [Pathname] the path to the control script
108
120
  def control_script
109
121
  @control_script ||= root + 'bin' + control_script_name
110
122
  end
111
123
 
112
- # The name of the 'riak' or 'riaksearch' control script.
113
- # @return [String] 'riak' or 'riaksearch'
124
+ # The name of the 'riak' control script.
114
125
  def control_script_name
115
- @control_script_name ||= (source + 'riaksearch').exist? ? 'riaksearch' : 'riak'
126
+ @control_script_name ||= 'riak'
116
127
  end
117
128
 
118
129
  # The script for controlling non-lifecycle features of Riak like
@@ -155,10 +166,10 @@ module Riak
155
166
  env[:bitcask][:data_root] ||= (data + 'bitcask').expand_path.to_s
156
167
  env[:eleveldb][:data_root] ||= (data + 'leveldb').expand_path.to_s
157
168
  env[:merge_index][:data_root] ||= (data + 'merge_index').expand_path.to_s
158
- env[:riak_core][:ring_state_dir] ||= ring.expand_path.to_s
159
169
  env[:riak_core][:slide_private_dir] ||= (data + 'slide-data').expand_path.to_s
170
+ env[:riak_core][:ring_state_dir] ||= (data + 'ring').expand_path.to_s
171
+
160
172
  NODE_DIRECTORIES.each do |dir|
161
- next if [:ring, :pipe].include?(dir)
162
173
  env[:riak_core][:"platform_#{dir}_dir"] ||= send(dir).to_s
163
174
  end
164
175
  end
@@ -169,11 +180,11 @@ module Riak
169
180
  env[:lager][:handlers] = {
170
181
  :lager_console_backend => :info,
171
182
  :lager_file_backend => [
172
- Tuple[(log+"error.log").expand_path.to_s, :error],
173
- Tuple[(log+"console.log").expand_path.to_s, :info]
183
+ Tuple[(log + "error.log").expand_path.to_s, :error],
184
+ Tuple[(log + "console.log").expand_path.to_s, :info]
174
185
  ]
175
186
  }
176
- env[:lager][:crash_log] = (log+"crash.log").to_s
187
+ env[:lager][:crash_log] = (log + "crash.log").to_s
177
188
  else
178
189
  # TODO: Need a better way to detect this, the defaults point
179
190
  # to 1.0-style configs. Maybe there should be some kind of
@@ -184,9 +195,9 @@ module Riak
184
195
  :fmt_max_bytes => 65536
185
196
  }
186
197
  env[:sasl] = {
187
- :sasl_error_logger => Tuple[:file, (log+"sasl-error.log").expand_path.to_s],
198
+ :sasl_error_logger => Tuple[:file, (log + "sasl-error.log").expand_path.to_s],
188
199
  :errlog_type => :error,
189
- :error_logger_mf_dir => (log+"sasl").expand_path.to_s,
200
+ :error_logger_mf_dir => (log + "sasl").expand_path.to_s,
190
201
  :error_logger_mf_maxbytes => 10485760,
191
202
  :error_logger_mf_maxfiles => 5
192
203
  }
@@ -227,9 +238,9 @@ module Riak
227
238
  min_port += 1
228
239
  end
229
240
  env[:riak_core][:http] = env[:riak_core][:http].map {|pair| Tuple[*pair] }
230
- env[:riak_kv][:pb_ip] = interface unless env[:riak_kv][:pb_ip]
231
- unless env[:riak_kv][:pb_port]
232
- env[:riak_kv][:pb_port] = min_port
241
+ env[pb_config_section][:pb_ip] = interface unless env[pb_config_section][:pb_ip]
242
+ unless env[pb_config_section][:pb_port]
243
+ env[pb_config_section][:pb_port] = min_port
233
244
  min_port += 1
234
245
  end
235
246
  unless env[:riak_core][:handoff_port]
@@ -4,6 +4,8 @@ module Riak
4
4
  class Node
5
5
  # Settings based on Riak 1.1.
6
6
  ENV_DEFAULTS = {
7
+ :riak_api => {
8
+ },
7
9
  :riak_core => {
8
10
  :ring_creation_size => 64
9
11
  },
@@ -17,7 +17,7 @@ module Riak
17
17
  def create
18
18
  unless exist?
19
19
  touch_ssl_distribution_args
20
- create_directories
20
+ copy_directories
21
21
  write_scripts
22
22
  write_vm_args
23
23
  write_app_config
@@ -52,9 +52,17 @@ module Riak
52
52
  root.rmtree if root.exist?
53
53
  end
54
54
 
55
- def create_directories
55
+ def copy_directories
56
56
  root.mkpath
57
- NODE_DIRECTORIES.each {|d| send(d).mkpath }
57
+ raise 'Source is not a directory!' unless base_dir.directory?
58
+
59
+ base_dir.each_child do |dir|
60
+ basename = dir.basename.to_s
61
+ next if NODE_DIR_SKIP_LIST.include? basename.to_sym
62
+ target = Pathname.new("#{root.to_s}")
63
+ target.mkpath
64
+ FileUtils.cp_r(dir.to_s,target)
65
+ end
58
66
  end
59
67
 
60
68
  def write_vm_args
@@ -72,18 +80,22 @@ module Riak
72
80
  end
73
81
 
74
82
  def write_scripts
75
- [control_script, admin_script].each {|s| write_script(s.basename, s) }
83
+ if version >= '1.4.0'
84
+ [env_script].each {|s| write_script(s) }
85
+ else
86
+ [control_script, admin_script].each {|s| write_script(s) }
87
+ end
76
88
  end
77
89
 
78
- def write_script(name, target)
79
- source_script = source + name
90
+ def write_script(target)
91
+ source_script = source.parent + target.relative_path_from(target.parent.parent)
80
92
  target.open('wb') do |f|
81
93
  source_script.readlines.each do |line|
82
94
  line.sub!(/(RUNNER_SCRIPT_DIR=)(.*)/, '\1' + bin.to_s)
83
95
  line.sub!(/(RUNNER_ETC_DIR=)(.*)/, '\1' + etc.to_s)
84
96
  line.sub!(/(RUNNER_USER=)(.*)/, '\1')
85
97
  line.sub!(/(RUNNER_LOG_DIR=)(.*)/, '\1' + log.to_s)
86
- line.sub!(/(PIPE_DIR=)(.*)/, '\1' + pipe.to_s + "/") # PIPE_DIR must have a trailing slash
98
+ line.sub!(/(PIPE_DIR=)(.*)/, '\1' + pipe.to_s)
87
99
  line.sub!(/(PLATFORM_DATA_DIR=)(.*)/, '\1' + data.to_s)
88
100
  line.sub!('grep "$RUNNER_BASE_DIR/.*/[b]eam"', 'grep "$RUNNER_ETC_DIR/app.config"')
89
101
  if line.strip == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}"
@@ -21,23 +21,9 @@ module Riak
21
21
  end
22
22
  end
23
23
 
24
- # Determines the base_dir from source control script
24
+ # Determines the base_dir from source parent
25
25
  def configure_base_dir
26
- # Use the script from the source directory so we don't require
27
- # it to be generated first.
28
- (source + control_script_name).each_line.find {|l| l =~ /^RUNNER_BASE_DIR=(.*)/ }
29
-
30
- # There should only be one matching line, so the contents of $1
31
- # will be the matched path. If there's nothing matched, we
32
- # return nil.
33
- case $1
34
- when '${RUNNER_SCRIPT_DIR%/*}'
35
- source.parent
36
- when String
37
- Pathname.new($1).expand_path
38
- else
39
- nil
40
- end
26
+ source.parent
41
27
  end
42
28
  end
43
29
  end
@@ -126,6 +126,7 @@ module Riak
126
126
  def store(options={})
127
127
  raise Conflict, self if conflict?
128
128
  raise ArgumentError, t("content_type_undefined") unless content_type.present?
129
+ raise ArgumentError, t("zero_length_key") if key == ''
129
130
  @bucket.client.store_object(self, options)
130
131
  self
131
132
  end
@@ -0,0 +1,67 @@
1
+ require 'riak/index_collection'
2
+ module Riak
3
+ class SecondaryIndex
4
+ include Util::Translation
5
+ include Client::FeatureDetection
6
+
7
+ # Create a Riak Secondary Index operation
8
+ # @param [Bucket] the {Riak::Bucket} we'll query against
9
+ # @param [String] the index name
10
+ # @param [String,Integer,Range<String,Integer>] a single value or
11
+ # range of values to query for
12
+ def initialize(bucket, index, query, options={})
13
+ @bucket = bucket
14
+ @client = @bucket.client
15
+ @index = index
16
+ @query = query
17
+ @options = options
18
+
19
+ validate_options
20
+ end
21
+
22
+ def get_server_version
23
+ @client.backend{|b| b.send :get_server_version }
24
+ end
25
+
26
+ # Get the array of matched keys
27
+ def keys(&block)
28
+ @collection ||=
29
+ @client.backend do |b|
30
+ b.get_index @bucket, @index, @query, @options, &block
31
+ end
32
+ end
33
+
34
+ # Get the array of values
35
+ def values
36
+ @values ||= @bucket.get_many(self.keys).values
37
+ end
38
+
39
+ # Get a new SecondaryIndex fetch for the next page
40
+ def next_page
41
+ raise t('index.no_next_page') unless keys.continuation
42
+
43
+ self.class.new(@bucket,
44
+ @index,
45
+ @query,
46
+ @options.merge(:continuation => keys.continuation))
47
+ end
48
+
49
+ # Determine whether a SecondaryIndex fetch has a next page available
50
+ def has_next_page?
51
+ !!keys.continuation
52
+ end
53
+
54
+ private
55
+ def validate_options
56
+ raise t('index.pagination_not_available') if paginated? && !index_pagination?
57
+ raise t('index.return_terms_not_available') if @options[:return_terms] && !index_return_terms?
58
+ raise t('index.include_terms_is_wrong') if @options[:include_terms]
59
+
60
+ # raise t('index.streaming_not_available') if @options[:stream] && !index_streaming?
61
+ end
62
+
63
+ def paginated?
64
+ @options[:continuation] || @options[:max_results]
65
+ end
66
+ end
67
+ end
@@ -1,3 +1,3 @@
1
1
  module Riak
2
- VERSION = "1.2.0"
2
+ VERSION = "1.4.0"
3
3
  end
@@ -7,9 +7,9 @@ Gem::Specification.new do |gem|
7
7
  gem.version = Riak::VERSION
8
8
  gem.summary = %Q{riak-client is a rich client for Riak, the distributed database by Basho.}
9
9
  gem.description = %Q{riak-client is a rich client for Riak, the distributed database by Basho. It supports the full HTTP and Protocol Buffers interfaces including storage operations, bucket configuration, link-walking, secondary indexes and map-reduce.}
10
- gem.email = ["sean@basho.com"]
10
+ gem.email = ["sean@basho.com", 'bryce@basho.com']
11
11
  gem.homepage = "http://github.com/basho/riak-ruby-client"
12
- gem.authors = ["Sean Cribbs"]
12
+ gem.authors = ["Sean Cribbs", 'Bryce Kerley']
13
13
 
14
14
  # Deps
15
15
  gem.add_development_dependency "rspec", "~>2.13.0"
@@ -52,6 +52,7 @@ Gem::Specification.new do |gem|
52
52
  **/*.rbc
53
53
  **/.DS_Store
54
54
  spec/support/test_server.yml
55
+ .ruby-version
55
56
  }
56
57
 
57
58
  files = includes.map {|glob| Dir[glob] }.flatten.select {|f| File.file?(f) }.sort
@@ -0,0 +1,51 @@
1
+ require 'spec_helper'
2
+ require 'riak'
3
+
4
+ describe Riak::Counter, test_server: true, integration: true do
5
+ before :all do
6
+ opts = {
7
+ http_port: test_server.http_port,
8
+ pb_port: test_server.pb_port,
9
+ protocol: 'pbc'
10
+ }
11
+ test_server.start
12
+ @client = Riak::Client.new opts
13
+ @bucket = @client['counter_spec']
14
+ @bucket.allow_mult = true
15
+
16
+ @counter = Riak::Counter.new @bucket, 'counter_spec'
17
+ end
18
+
19
+
20
+ ['pbc', 'http'].each do |protocol|
21
+ describe protocol do
22
+ before :all do
23
+ @client.protocol = protocol
24
+ end
25
+ it 'should read and update' do
26
+ initial = @counter.value
27
+
28
+ @counter.increment
29
+ @counter.increment
30
+
31
+ @counter.value.should == (initial + 2)
32
+
33
+ @counter.decrement 2
34
+
35
+ @counter.value.should == initial
36
+
37
+ 5.times do
38
+ amt = rand(10_000)
39
+
40
+ @counter.increment amt
41
+ @counter.value.should == (initial + amt)
42
+
43
+ @counter.decrement (amt * 2)
44
+ @counter.value.should == (initial - amt)
45
+
46
+ @counter.increment_and_return(amt).should == initial
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -106,29 +106,39 @@ describe "HTTP" do
106
106
  end
107
107
  end
108
108
 
109
- class Reader < Array
109
+ class SizelessReader < Array
110
110
  def read(*args)
111
111
  shift
112
112
  end
113
113
 
114
- def size
115
- join.size
116
- end
117
- end
118
-
119
- class SizelessReader < Reader
120
114
  undef :size
121
115
  end
122
116
 
123
117
  describe 'NetHTTPBackend' do
124
118
  subject { Riak::Client::NetHTTPBackend.new(@client, @client.node) }
125
- let(:file) { File.open(__FILE__) }
126
- let(:sized) { Reader.new(["foo", "bar", "baz"]) }
127
- let(:sizeless) { SizelessReader.new(["foo", "bar", "baz"]) }
128
- it "should set the content-length or transfer-encoding properly on IO uploads" do
129
- lambda { subject.put(204, subject.object_path('nethttp', 'test-file'), file, {"Content-Type" => "text/plain"}) }.should_not raise_error
130
- lambda { subject.put(204, subject.object_path('nethttp', 'test-sized'), sized, {"Content-Type" => "text/plain"}) }.should_not raise_error
131
- lambda { subject.put(204, subject.object_path('nethttp', 'test-sizeless'), sizeless, {"Content-Type" => "text/plain"}) }.should_not raise_error
119
+ shared_examples "IO uploads" do |io|
120
+ it "should upload without error" do
121
+ lambda do
122
+ Timeout::timeout(2) do
123
+ subject.put(
124
+ 204,
125
+ subject.object_path('nethttp', 'test-io'),
126
+ io,
127
+ {'Content-Type' => 'text/plain'}
128
+ )
129
+ end
130
+ end.should_not raise_error
131
+ end
132
+ end
133
+
134
+ context "File" do
135
+ include_examples "IO uploads", File.open(__FILE__)
136
+ end
137
+ context "Sized reader" do
138
+ include_examples "IO uploads", StringIO.new(%w{foo bar baz}.join)
139
+ end
140
+ context "Sizeless reader" do
141
+ include_examples "IO uploads", SizelessReader.new(%w{foo bar baz})
132
142
  end
133
143
  end
134
144
  end
@@ -12,28 +12,6 @@ describe Riak::Node, :test_server => false, :slow => true, :nodegen => true do
12
12
  context "finding the base_dir and version" do
13
13
  its(:base_dir) { should be_directory }
14
14
  its(:version) { should match /^\d+.\d+.\d+$/ }
15
-
16
- context "when the base directory is missing" do
17
- before { Pathname.any_instance.stub(:each_line).and_return([]) }
18
- its(:base_dir) { should be_nil }
19
- its(:version) { should be_nil }
20
- end
21
- end
22
-
23
- context "when the source control script is a symlink" do
24
- let(:symdir) { Pathname.new(".symlinkriak") }
25
- let(:sourcedir) { Pathname.new(test_server_config['source']) }
26
- let(:control_script){ (sourcedir + 'riaksearch').exist? ? 'riaksearch' : 'riak' }
27
-
28
- subject { described_class.new(:root => ".ripplenode", :source => symdir) }
29
-
30
- before do
31
- symdir.mkpath
32
- (symdir + control_script).make_symlink(sourcedir + control_script)
33
- end
34
- after { symdir.rmtree }
35
-
36
- its(:source){ should == sourcedir }
37
15
  end
38
16
 
39
17
  context "creation" do
@@ -64,8 +42,8 @@ describe Riak::Node, :test_server => false, :slow => true, :nodegen => true do
64
42
  contents.should include('{handoff_port, 8082}')
65
43
  end
66
44
 
67
- it "should set the ring directory to point to the node directory" do
68
- contents.should include("{ring_state_dir, \"#{subject.root + 'ring'}\"}")
45
+ it "should set the ring directory to point to the node data directory" do
46
+ contents.should include("{ring_state_dir, \"#{subject.root + 'data/ring'}\"}")
69
47
  end
70
48
  end
71
49
 
@@ -86,8 +64,8 @@ describe Riak::Node, :test_server => false, :slow => true, :nodegen => true do
86
64
  end
87
65
  end
88
66
 
89
- describe "generating the start script" do
90
- let(:file) { subject.control_script }
67
+ describe "generating the environment script" do
68
+ let(:file) { (subject.version >= '1.4.0') ? subject.env_script : subject.control_script }
91
69
  let(:contents) { file.read }
92
70
 
93
71
  it "should create the script in the node directory" do
@@ -148,8 +126,8 @@ describe Riak::Node, :test_server => false, :slow => true, :nodegen => true do
148
126
  (subject.root + 'data').children.map {|dir| dir.children }.flatten.should be_empty
149
127
  end
150
128
 
151
- it "should not remove the ring" do
152
- (subject.root + 'ring').children.should_not be_empty
129
+ it "should remove the ring" do
130
+ (subject.root + 'data/ring').children.should be_empty
153
131
  end
154
132
  end
155
133