riak-client 1.0.0.beta → 1.0.0

data/spec/riak/excon_backend_spec.rb

@@ -24,13 +24,14 @@ else
 
     before :each do
       @client = Riak::Client.new(:http_port => $mock_server.port, :http_backend => :Excon) # Point to our mock
-      @backend = @client.http
+      @node = @client.node
+      @backend = @client.new_http_backend
       @_mock_set = false
     end
 
     after :each do
       if @_mock_set
-        $mock_server.satisfied.should be_true("Expected #{@_mock_set.inspect}, failed")
+        $mock_server.satisfied.should be_true #("Expected #{@_mock_set.inspect}, failed")
       end
     end
 

data/spec/riak/http_backend/configuration_spec.rb

@@ -2,7 +2,8 @@ require 'spec_helper'
 
 describe Riak::Client::HTTPBackend::Configuration do
   let(:client){ Riak::Client.new }
-  subject { Riak::Client::HTTPBackend.new(client) }
+  let(:node){ client.node }
+  subject { Riak::Client::HTTPBackend.new(client, node) }
   let(:uri){ URI.parse("http://127.0.0.1:8098/") }
 
   context "generating resource URIs" do
@@ -200,6 +201,37 @@ describe Riak::Client::HTTPBackend::Configuration do
     end
   end
 
+  context "when Luwak is enabled" do
+    before { subject.should_receive(:get).with(200, uri).and_return(:headers => {'link' => ['</riak>; rel="riak_kv_wm_raw", </ping>; rel="riak_kv_wm_ping", </stats>; rel="riak_kv_wm_stats", </mapred>; rel="riak_kv_wm_mapred", </luwak>; rel="luwak_wm_file"']}) }
+
+    it "should generate a path for a file" do
+      url = subject.luwak_path('foo')
+      url.should be_kind_of(URI)
+      url.path.should == '/luwak/foo'
+    end
+
+    it "should generate a path for the root" do
+      url = subject.luwak_path(nil)
+      url.should be_kind_of(URI)
+      url.path.should == '/luwak'
+    end
+
+    it "should escape a nested path" do
+      url = subject.luwak_path("foo/bar/baz")
+      url.should be_kind_of(URI)
+      url.path.should == '/luwak/foo%2Fbar%2Fbaz'
+    end
+  end
+
+  context "when Luwak is disabled" do
+    before { subject.should_receive(:get).with(200, uri).and_return(:headers => {'link' => ['</riak>; rel="riak_kv_wm_raw", </ping>; rel="riak_kv_wm_ping", </stats>; rel="riak_kv_wm_stats", </mapred>; rel="riak_kv_wm_mapred"']}) }
+
+    it "should raise an error when generating the path" do
+      expect { subject.luwak_path(nil) }.to raise_error
+      expect { subject.luwak_path('foo') }.to raise_error
+    end
+  end
+
   {
     :riak_kv_wm_raw => :prefix,
     :riak_kv_wm_link_walker => :prefix,
@@ -210,14 +242,14 @@ describe Riak::Client::HTTPBackend::Configuration do
       subject.send(resource).should == "/path"
     end
 
-    it "should fallback to client.http_paths[:#{alternate}] if the #{resource} resource is not found" do
+    it "should fallback to node.http_paths[:#{alternate}] if the #{resource} resource is not found" do
       subject.should_receive(:get).with(200, uri).and_return(:headers => {'link' => ['</>; rel="top"']})
-      subject.send(resource).should == client.http_paths[alternate]
+      subject.send(resource).should == node.http_paths[alternate]
     end
 
-    it "should fallback to client.http_paths[:#{alternate}] if request fails" do
+    it "should fallback to node.http_paths[:#{alternate}] if request fails" do
       subject.should_receive(:get).with(200, uri).and_raise(Riak::HTTPFailedRequest.new(:get, 200, 404, {}, ""))
-      subject.send(resource).should == client.http_paths[alternate]
+      subject.send(resource).should == node.http_paths[alternate]
     end
   end
 

data/spec/riak/http_backend/object_methods_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Riak::Client::HTTPBackend::ObjectMethods do
   before :each do
     @client = Riak::Client.new
-    @backend = Riak::Client::HTTPBackend.new(@client)
+    @backend = Riak::Client::HTTPBackend.new(@client, @client.node)
     @bucket = Riak::Bucket.new(@client, "bucket")
     @object = Riak::RObject.new(@bucket, "bar")
     @backend.stub!(:new_scheme?).and_return(false)

data/spec/riak/http_backend/transport_methods_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Riak::Client::HTTPBackend::TransportMethods do
   before :each do
     @client = Riak::Client.new
-    @backend = Riak::Client::HTTPBackend.new(@client)
+    @backend = Riak::Client::HTTPBackend.new(@client, @client.node)
     @backend.instance_variable_set(:@server_config, {})
   end
 
@@ -91,7 +91,7 @@ describe Riak::Client::HTTPBackend::TransportMethods do
 
   it "should allow using the https protocol" do
     @client  = Riak::Client.new(:protocol => 'https')
-    @backend = Riak::Client::HTTPBackend.new(@client)
+    @backend = Riak::Client::HTTPBackend.new(@client, @client.node)
     @backend.root_uri.to_s.should eq("https://127.0.0.1:8098")
   end
 end

data/spec/riak/http_backend_spec.rb

@@ -3,19 +3,25 @@ require 'spec_helper'
 describe Riak::Client::HTTPBackend do
   before :each do
     @client = Riak::Client.new
-    @backend = Riak::Client::HTTPBackend.new(@client)
+    @node = @client.nodes.first
+    @backend = Riak::Client::HTTPBackend.new(@client, @node)
     @backend.instance_variable_set(:@server_config, {})
   end
 
-  it "should take the Riak::Client when creating" do
+  it "should take the Riak::Client and Riak::Node when creating" do
     lambda { Riak::Client::HTTPBackend.new(nil) }.should raise_error(ArgumentError)
-    lambda { Riak::Client::HTTPBackend.new(@client) }.should_not raise_error
+    lambda { Riak::Client::HTTPBackend.new(@client) }.should raise_error(ArgumentError)
+    lambda { Riak::Client::HTTPBackend.new(@client, @node) }.should_not raise_error
   end
 
   it "should make the client accessible" do
     @backend.client.should == @client
   end
 
+  it 'should make the node accessible' do
+    @backend.node.should == @node
+  end
+
   context "pinging the server" do
     it "should succeed on 200" do
       @backend.should_receive(:get).with(200, @backend.ping_path).and_return({:code => 200, :body => "OK"})
@@ -285,4 +291,48 @@ describe Riak::Client::HTTPBackend do
       @backend.update_search_index(nil, 'postbody')
     end
   end
+  context "Luwak" do
+    before { @backend.send(:server_config)[:luwak_wm_file] = '/luwak' }
+    context "fetching a file" do
+      before do
+
+        @backend.should_receive(:get).with(200, @backend.luwak_path("greeting.txt")).and_yield("Hello,").and_yield(" world!").and_return({:code => 200, :headers => {"content-type" => ["text/plain"]}})
+      end
+
+      it "should return a tempfile when no block is given" do
+        file = @backend.get_file("greeting.txt")
+        file.open {|f| f.read.should == "Hello, world!" }
+      end
+
+      it "should expose the original key and content-type on the temporary file" do
+        file = @backend.get_file("greeting.txt")
+        file.original_filename.should == 'greeting.txt'
+        file.content_type.should == 'text/plain'
+      end
+
+      it "should yield chunks of the file to the block and return nil" do
+        string = ""
+        result = @backend.get_file("greeting.txt"){|chunk| string << chunk }
+        result.should be_nil
+        string.should == "Hello, world!"
+      end
+    end
+
+    context "storing a file" do
+      it "should store a file with the given filename" do
+        @backend.should_receive(:put).with(204, @backend.luwak_path("greeting.txt"), anything, {"Content-Type" => "text/plain"}).and_return({:code => 204, :headers => {}})
+        @backend.store_file("greeting.txt", "text/plain", "Hello, world").should == 'greeting.txt'
+      end
+
+      it "should store a file and return the key/filename when none is given" do
+        @backend.should_receive(:post).with(201, @backend.luwak_path(nil), anything, {"Content-Type" => "text/plain"}).and_return({:code => 201, :headers => {'location' => ["/luwak/123456789"]}})
+        @backend.store_file("text/plain", "Hello, world").should == '123456789'
+      end
+    end
+
+    it "should detect whether a file exists" do
+      @backend.should_receive(:head).with([200,404], @backend.luwak_path("foo")).and_return({:code => 200})
+      @backend.file_exists?("foo").should be_true
+    end
+  end
 end

data/spec/riak/map_reduce_spec.rb

@@ -4,7 +4,7 @@ describe Riak::MapReduce do
   before :each do
     @client = Riak::Client.new
     @backend = mock("Backend")
-    @client.stub!(:backend).and_return(@backend)
+    @client.stub!(:backend).and_yield(@backend)
     @mr = Riak::MapReduce.new(@client)
   end
 

data/spec/riak/net_http_backend_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Riak::Client::NetHTTPBackend do
   before :each do
     @client = Riak::Client.new(:http_backend => :NetHTTP)
-    @backend = @client.http
+    @backend = @client.new_http_backend
     FakeWeb.allow_net_connect = false
   end
 
@@ -12,5 +12,4 @@ describe Riak::Client::NetHTTPBackend do
   end
 
   it_should_behave_like "HTTP backend"
-
 end

data/spec/riak/node_spec.rb

@@ -0,0 +1,173 @@
+require 'spec_helper'
+
+describe Riak::Node do
+  before :each do
+    @client = Riak::Client.new
+    @node = Riak::Client::Node.new @client
+  end
+
+  describe 'when initializing' do
+    it 'should default to the local interface on port 8098/8087' do
+      node = Riak::Client::Node.new @client
+      node.host.should == '127.0.0.1'
+      node.http_port.should == 8098
+      node.pb_port.should == 8087
+    end
+
+    it 'should accept a host' do
+      node = Riak::Client::Node.new(@client, :host => 'riak.basho.com')
+      node.host.should == "riak.basho.com"
+    end
+
+    it 'should accept an HTTP port' do
+      node = Riak::Client::Node.new(@client, :http_port => 9000)
+      node.http_port.should == 9000
+    end
+
+    it 'should accept a Protobuffs port' do
+      node = Riak::Client::Node.new @client, :pb_port => 9000
+      node.pb_port.should == 9000
+    end
+
+    it 'should accept basic_auth' do
+      node = Riak::Client::Node.new @client, :basic_auth => 'user:pass'
+      node.basic_auth.should == "user:pass"
+    end
+
+    it 'should accept a path prefix' do
+      node = Riak::Client::Node.new @client, :prefix => "/jiak/"
+      node.http_paths[:prefix].should == "/jiak/"
+    end
+
+    it 'default prefix should be /riak/' do
+      @node.http_paths[:prefix].should == "/riak/"
+    end
+
+    it 'should accept a mapreduce path' do
+      node = Riak::Client::Node.new @client, :mapred => "/mr"
+      node.http_paths[:mapred].should == "/mr"
+    end
+
+    it "default mapreduce path should be /mapred" do
+      @node.http_paths[:mapred].should == "/mapred"
+    end
+
+    it 'should accept a solr path' do
+      node = Riak::Client::Node.new @client, :solr => "/mr"
+      node.http_paths[:solr].should == "/mr"
+    end
+
+    it "default solr path should be /solr" do
+      @node.http_paths[:solr].should == "/solr"
+    end
+
+    it 'should accept a luwak path' do
+      node = Riak::Client::Node.new @client, :luwak => "/mr"
+      node.http_paths[:luwak].should == "/mr"
+    end
+
+    it "default luwak path should be /luwak" do
+      @node.http_paths[:luwak].should == "/luwak"
+    end
+  end
+
+  describe "setting http auth" do
+    it "should allow setting basic_auth" do
+      @node.should respond_to(:basic_auth=)
+      @node.basic_auth = "user:pass"
+      @node.basic_auth.should == "user:pass"
+      @node.basic_auth = nil
+      @node.basic_auth.should == nil
+    end
+
+    it "should require that basic auth splits into two even parts" do
+      lambda { @node.basic_auth ="user" }.should raise_error(ArgumentError, "basic auth must be set using 'user:pass'")
+    end
+  end
+
+  describe 'ssl' do
+    before :each do
+      @client = Riak::Client.new
+      @node = Riak::Client::Node.new @client
+    end
+
+    it 'should not allow reading ssl options via ssl' do
+      @node.should_not respond_to(:ssl)
+    end
+
+    it 'should allow setting ssl to true' do
+      @node.ssl = true
+      @node.ssl_options[:verify_mode].should eq('none')
+    end
+
+    it "should should clear ssl options when setting ssl to false" do
+      @node.ssl = true
+      @node.ssl_options.should_not be_nil
+      @node.ssl = false
+      @node.ssl_options.should be_nil
+    end
+
+    it "should allow setting the verify_mode to none" do
+      @node.ssl = {:verify_mode => "none"}
+      @node.ssl_options[:verify_mode].should eq("none")
+    end
+
+    it "should allow setting the verify_mode to peer" do
+      @node.ssl = {:verify_mode => "peer"}
+      @node.ssl_options[:verify_mode].should eq("peer")
+    end
+
+    it "should not allow setting the verify_mode to anything else" do
+      lambda {@node.ssl = {:verify_mode => :your_mom}}.should raise_error(ArgumentError)
+    end
+
+    it "should default verify_mode to none" do
+      @node.ssl = true
+      @node.ssl_options[:verify_mode].should eq("none")
+    end
+
+    it "should let the backend know if ssl is enabled" do
+      @node.should_not be_ssl_enabled
+      @node.ssl = true
+      @node.should be_ssl_enabled
+    end
+
+    it "should allow setting the pem" do
+      @node.ssl = {:pem => 'i-am-a-pem'}
+      @node.ssl_options[:pem].should eq('i-am-a-pem')
+    end
+
+    it "should set them pem from the contents of pem_file" do
+      filepath = File.expand_path(File.join(File.dirname(__FILE__), '../fixtures/test.pem'))
+      @node.ssl = {:pem_file => filepath}
+      @node.ssl_options[:pem].should eq("i-am-a-pem\n")
+    end
+
+    it "should allow setting the pem_password" do
+      @node.ssl = {:pem_password => 'pem-pass'}
+      @node.ssl_options[:pem_password].should eq('pem-pass')
+    end
+
+    it "should allow setting the ca_file" do
+      @node.ssl = {:ca_file => '/path/to/ca.crt'}
+      @node.ssl_options[:ca_file].should eq('/path/to/ca.crt')
+    end
+
+    it "should allow setting the ca_path" do
+      @node.ssl = {:ca_path => '/path/to/certs/'}
+      @node.ssl_options[:ca_path].should eq('/path/to/certs/')
+    end
+
+    %w[pem ca_file ca_path].each do |option|
+      it "should default the verify_mode to peer when setting the #{option}" do
+        @node.ssl = {option.to_sym => 'test-data'}
+        @node.ssl_options[:verify_mode].should eq("peer")
+      end
+
+      it "should allow setting the verify mode when setting the #{option}" do
+        @node.ssl = {option.to_sym => 'test-data', :verify_mode => "none"}
+        @node.ssl_options[:verify_mode].should eq("none")
+      end
+    end
+  end
+end

data/spec/riak/pool_spec.rb

@@ -0,0 +1,306 @@
+require 'spec_helper'
+require 'thread'
+
+describe Riak::Client::Pool do
+  describe 'basics' do
+    subject {
+      described_class.new(
+        lambda { [0] },
+        lambda { |x| }
+      )
+    }
+
+    it 'yields a new object' do
+      subject.take do |x|
+        x.should == [0]
+      end
+    end
+
+    it 'retains a single object for serial access' do
+      n = 100
+      n.times do |i|
+        subject.take do |x|
+          x.should == [i]
+          x[0] += 1
+        end
+      end
+      subject.size.should == 1
+    end
+
+    it 'should be re-entrant' do
+      n = 10
+      n.times do |i|
+        subject.take do |x|
+          x.replace [1]
+          subject.take do |y|
+            y.replace [2]
+            subject.take do |z|
+              z.replace [3]
+              subject.take do |t|
+                t.replace [4]
+              end
+            end
+          end
+        end
+      end
+      subject.pool.map { |e| e.object.first }.sort.should == [1,2,3,4]
+    end
+
+
+    it 'should unlock when exceptions are raised' do
+      begin
+        subject.take do |x|
+          x << 1
+          subject.take do |y|
+            x << 2
+            y << 3
+            raise
+          end
+        end
+      rescue
+      end
+      subject.pool.all? { |e| not e.owner }.should == true
+      subject.pool.map { |e| e.object }.to_set.should == [
+        [0,1,2],
+        [0,3]
+      ].to_set
+    end
+
+    it 'should delete when BadResource is raised' do
+      subject.open = lambda do
+        m = mock('resource')
+        m.should_receive(:close)
+        m
+      end
+      subject.close = lambda do |res|
+        res.close
+      end
+
+      lambda do
+        subject.take do |x|
+          raise Riak::Client::Pool::BadResource
+        end
+      end.should raise_error(Riak::Client::Pool::BadResource)
+      subject.size.should == 0
+    end
+  end
+
+  describe 'threads' do
+    subject {
+      described_class.new(
+        lambda { [] },
+        lambda { |x| }
+      )
+    }
+
+    it 'should allocate n objects for n concurrent operations' do
+      # n threads concurrently allocate and sign objects from the pool
+      n = 10
+      readyq = Queue.new
+      finishq = Queue.new
+      threads = (0...n).map do
+        Thread.new do
+          subject.take do |x|
+            readyq << 1
+            x << Thread.current
+            finishq.pop
+          end
+        end
+      end
+
+      n.times { readyq.pop }
+      n.times { finishq << 1 }
+
+      # Wait for completion
+      threads.each do |t|
+        t.join
+      end
+
+      # Should have taken exactly n objects to do this
+      subject.size.should == n
+      # And each one should be signed exactly once
+      subject.pool.map do |e|
+        e.object.size.should == 1
+        e.object.first
+      end.to_set.should == threads.to_set
+    end
+
+    it 'take with filter and default' do
+      n = 10
+      subject = described_class.new(
+        lambda { [] },
+        lambda { |x| }
+      )
+
+      # Allocate several elements of the pool
+      q = Queue.new
+      threads = (0...n).map do |i|
+        Thread.new do
+          subject.take do |a|
+            a << i
+            q << 1
+            sleep 0.02
+          end
+        end
+      end
+
+      # Wait for all threads to have acquired an element
+      n.times { q.pop }
+
+      threads.each do |t|
+        t.join
+      end
+
+      # Get and delete existing even elements
+      got = Set.new
+      (n / 2).times do
+        begin
+          subject.take(
+            :filter => lambda { |x| x.first.even? },
+            :default => [:default]
+          ) do |x|
+            got << x.first
+            raise Riak::Client::Pool::BadResource
+          end
+        rescue Riak::Client::Pool::BadResource
+        end
+      end
+      got.should == (0...n).select(&:even?).to_set
+
+      # This time, no even elements exist, so we should get the default.
+      subject.take(
+        :filter => lambda { |x| x.first.even? },
+        :default => :default
+      ) do |x|
+        x.should == :default
+      end
+    end
+
+    it 'iterates over a snapshot of all connections, even ones in use' do
+      started = Queue.new
+      n = 30
+      threads = (0..n).map do
+        Thread.new do
+          psleep = 0.75 * rand # up to 50ms sleep
+          subject.take do |a|
+            started << 1
+            a << rand
+            sleep psleep
+          end
+        end
+      end
+
+      n.times { started.pop }
+      touched = []
+
+      subject.each do |e|
+        touched << e
+      end
+
+      threads.each do |t|
+        t.join
+      end
+
+      touched.should be_all {|item| subject.pool.find {|e| e.object == item } }
+    end
+
+    it 'should clear' do
+      n = 10
+      subject = described_class.new(
+        lambda { mock('connection').tap {|m| m.should_receive(:teardown) }},
+        lambda { |b| b.teardown }
+      )
+
+      # Allocate several elements of the pool
+      q = Queue.new
+      threads = (0...n).map do |i|
+        Thread.new do
+          subject.take do |a|
+            q << 1
+            sleep 0.1
+          end
+        end
+      end
+
+      # Wait for all threads to have acquired an element
+      n.times { q.pop }
+
+      # Clear the pool while threads still have elements checked out
+      subject.clear
+      subject.pool.should be_empty
+
+      # Wait for threads to complete
+      threads.each do |t|
+        t.join
+      end
+    end
+
+    it 'should delete_if' do
+      n = 10
+      subject = described_class.new(
+        lambda { [] },
+        lambda { |x| }
+      )
+
+      # Allocate several elements of the pool
+      q = Queue.new
+      threads = (0...n).map do |i|
+        Thread.new do
+          subject.take do |a|
+            a << i
+            q << 1
+            sleep 0.02
+          end
+        end
+      end
+
+      # Wait for all threads to have acquired an element
+      n.times { q.pop }
+
+      # Delete odd elements
+      subject.delete_if do |x|
+        x.first.odd?
+      end
+
+      # Verify odds are gone.
+      subject.pool.all? do |x|
+        x.object.first.even?
+      end.should == true
+
+      # Wait for threads
+      threads.each do |t|
+        t.join
+      end
+    end
+
+    it 'stress test', :slow => true do
+      n = 100
+      psleep = 0.8
+      tsleep = 0.01
+      rounds = 100
+
+      threads = (0...n).map do
+        Thread.new do
+          rounds.times do |i|
+            subject.take do |a|
+              a.should == []
+              a << Thread.current
+              a.should == [Thread.current]
+
+              # Sleep and check
+              while rand < psleep
+                sleep tsleep
+                a.should == [Thread.current]
+              end
+
+              a.delete Thread.current
+            end
+          end
+        end
+      end
+      threads.each do |t|
+        t.join
+      end
+    end
+  end
+end