riak-client-noenc 1.0.0

data/spec/support/crdt_search_fixtures.rb

@@ -0,0 +1,42 @@
+module CrdtSearchFixtures
+  def map_result_score
+    43.21
+  end
+
+  def maps_type_name
+    'maps'
+  end
+
+  def maps_bucket_type
+    return @maps_bucket_type if defined? @maps_bucket_type
+
+    @maps_bucket_type = instance_double('Riak::BucketType').tap do |bt|
+      allow(bt).to receive(:bucket).
+                    with(bucket_name).
+                    and_return(map_bucket)
+      allow(bt).to receive(:data_type_class).
+                    and_return(Riak::Crdt::Map)
+    end
+  end
+
+  def map_bucket
+    @map_bucket ||= instance_double('Riak::BucketTyped::Bucket')
+  end
+
+  def map_raw
+    @map_raw ||= {
+      'score'=>map_result_score,
+      '_yz_rb'=>bucket_name,
+      '_yz_rt'=>maps_type_name,
+      '_yz_rk'=>'map-key'
+    }
+  end
+
+  def map_results
+    @map_results ||= Riak::Search::ResultDocument.new client, map_raw
+  end
+end
+
+RSpec.configure do |config|
+  config.include CrdtSearchFixtures, crdt_search_fixtures: true
+end

data/spec/support/integration_setup.rb

@@ -0,0 +1,10 @@
+# auto-tag all integration specs with :integration => true
+module IntegrationSpecs
+  def self.included(klass)
+    klass.metadata[:integration] = true
+  end
+end
+
+RSpec.configure do |config|
+  config.include IntegrationSpecs, file_path: %r{spec/integration}
+end

data/spec/support/search_config.rb

@@ -0,0 +1,83 @@
+module SearchConfig
+  include TestClient
+
+  def search_bucket
+    return @search_bucket if defined? @search_bucket
+    type = test_client.bucket_type 'yokozuna'
+    @search_bucket = type.bucket "search_config-#{random_key}"
+  end
+
+  def index_name
+    @index_name ||= search_bucket.name
+  end
+
+  def index
+    return @index if defined? @index
+    create_index
+    @index = Riak::Search::Index.new test_client, index_name
+  end
+
+  def create_index
+    return if defined? @index_exists
+
+    test_client.create_search_index index_name
+
+    @index_exists = true
+  end
+
+  def configure_bucket
+    return if defined? @bucket_configured
+
+    create_index
+
+    test_client.set_bucket_props(search_bucket,
+                                 { search_index: index_name },
+                                 'yokozuna')
+
+    wait_until do
+      props = test_client.get_bucket_props search_bucket, type: 'yokozuna'
+      props['search_index'] == index_name
+    end
+
+    @bucket_configred = true
+  end
+
+  def load_corpus
+    return if defined? @corpus_loaded
+
+    configure_bucket
+
+    old_encoding = Encoding.default_external
+    Encoding.default_external = Encoding::UTF_8
+
+    IO.foreach('spec/fixtures/bitcask.txt').with_index do |para, idx|
+      next if para =~ /^\s*$|introduction|chapter/ui
+
+      Riak::RObject.new(search_bucket, "bitcask-#{idx}") do |obj|
+        obj.content_type = 'text/plain'
+        obj.raw_data = para
+        obj.store
+      end
+    end
+
+    Encoding.default_external = old_encoding
+
+    wait_until do
+      results = @client.search(index_name,
+                               'contain your entire keyspace',
+                               df: 'text')
+
+      results['docs'].length > 0
+    end
+
+    @corpus_loaded = true
+  end
+
+  def schema_xml(schema_name)
+    File.read('spec/fixtures/yz_schema_template.xml').sub('SCHEMA_NAME', schema_name)
+  end
+end
+
+RSpec.configure do |config|
+  config.include SearchConfig, search_config: true
+end

data/spec/support/search_corpus_setup.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+shared_context "search corpus setup" do
+  before do
+    @search_bucket = random_bucket 'search_test'
+    @backend.create_search_index @search_bucket.name
+
+    wait_until{ !@backend.get_search_index(@search_bucket.name).nil? }
+
+    @client.set_bucket_props(@search_bucket,
+                             {search_index: @search_bucket.name},
+                             'yokozuna')
+
+    wait_until do
+      p = @client.get_bucket_props(@search_bucket, type: 'yokozuna')
+      p['search_index'] == @search_bucket.name
+    end
+
+    idx = 0
+    old_encoding = Encoding.default_external
+    Encoding.default_external = Encoding::UTF_8
+    IO.foreach("spec/fixtures/bitcask.txt") do |para|
+      next if para =~ /^\s*$|introduction|chapter/ui
+      idx += 1
+      Riak::RObject.new(@search_bucket, "bitcask-#{idx}") do |obj|
+        obj.content_type = 'text/plain'
+        obj.raw_data = para
+        @backend.store_object(obj, type: 'yokozuna')
+      end
+    end
+    Encoding.default_external = old_encoding
+
+    wait_until do
+      results = @backend.search(@search_bucket.name,
+                                'contain your entire keyspace',
+                                df: 'text')
+      results['docs'].length > 0
+    end
+  end
+end

data/spec/support/test_client.rb

@@ -0,0 +1,46 @@
+module TestClient
+  def test_client
+    if defined? $test_client and $test_client.ping
+      return $test_client
+    end
+
+    candidate_client = Riak::Client.new test_client_configuration
+
+    live = candidate_client.ping
+
+    return $test_client = candidate_client if live
+  end
+
+  def test_client_configuration
+    TestClient.test_client_configuration
+  end
+
+  def self.test_client_configuration
+    if defined? $test_client_configuration
+      return $test_client_configuration
+    end
+
+    config_path = File.expand_path '../test_client.yml', __FILE__
+    config = YAML.load_file(config_path).symbolize_keys
+
+    if config[:nodes]
+      new_nodes = config[:nodes].map(&:symbolize_keys)
+      config[:nodes] = new_nodes
+    end
+
+    $test_client_configuration = config
+  end
+
+  def random_bucket(name = 'test_client')
+    bucket_name = [name, Time.now.to_i, random_key].join('-')
+    test_client.bucket bucket_name
+  end
+
+  def random_key
+    rand(36**10).to_s(36)
+  end
+end
+
+RSpec.configure do |config|
+  config.include TestClient, test_client: true
+end

data/spec/support/test_client.yml.example

@@ -0,0 +1,10 @@
+# This file is used for configuring test_client used by
+# many of the integration tests.
+#
+# Simply set the arguments you'd normally pass to
+# Riak::Client.new in the file.
+#
+# Don't run multiple nodes, integration tests include
+# quorum checks that may result in race conditions.
+
+pb_port: 10017

data/spec/support/unified_backend_examples.rb

@@ -0,0 +1,380 @@
+shared_examples_for "Unified backend API" do
+  # ping
+  it "pings the server" do
+    expect(@backend.ping).to be_truthy
+  end
+
+  it "gets info about the server" do
+    expect{ @backend.server_info }.to_not raise_error
+
+    expect(@backend.server_info).to include(:node, :server_version)
+  end
+
+  it "gets client id" do
+    expect{ @backend.get_client_id }.to_not raise_error
+
+    expect(@backend.get_client_id).to be_a String
+  end
+
+  # fetch_object
+  context "fetching an object" do
+    before do
+      @robject = Riak::RObject.new(@bucket, "fetch")
+      @robject.content_type = "application/json"
+      @robject.data = { "test" => "pass" }
+      @robject.indexes['test_bin'] << 'pass'
+      @robject.links << Riak::Link.new('/riak/foo/bar', 'next')
+      @robject.links << Riak::Link.new('/riak/foo/baz', 'next')
+      @backend.store_object(@robject)
+    end
+
+    it "finds a stored object" do
+      robj = @backend.fetch_object(@bucket.name, "fetch")
+      expect(robj).to be_kind_of(Riak::RObject)
+      expect(robj.data).to eq({ "test" => "pass" })
+      expect(robj.links).to be_a Set
+    end
+
+    it "raises an error when the object is not found" do
+      begin
+        @backend.fetch_object(@bucket.name, "notfound")
+      rescue Riak::FailedRequest => exception
+        @exception = exception
+      end
+      expect(@exception).to be_kind_of(Riak::FailedRequest)
+      expect(@exception).to be_not_found
+    end
+
+    [1, 2, 3, :one, :quorum, :all, :default].each do |q|
+      it "accepts a R value of #{q.inspect} for the request" do
+        robj = @backend.fetch_object(@bucket.name, "fetch", :r => q)
+        expect(robj).to be_kind_of(Riak::RObject)
+        expect(robj.data).to eq({ "test" => "pass" })
+      end
+
+      it "accepts a PR value of #{q.inspect} for the request" do
+        robj = @backend.fetch_object(@bucket.name, "fetch", :pr => q)
+        expect(robj).to be_kind_of(Riak::RObject)
+        expect(robj.data).to eq({ "test" => "pass" })
+      end
+    end
+
+    it "marshals indexes properly", :retries => 5 do
+      robj = @backend.fetch_object(@bucket.name, 'fetch')
+      expect(robj.indexes['test_bin']).to be
+      expect(robj.indexes['test_bin']).to include('pass')
+    end
+  end
+
+  # reload_object
+  context "reloading an existing object" do
+    before do
+      @robject = Riak::RObject.new(@bucket, 'reload')
+      @robject.content_type = "application/json"
+      @robject.data = {"test" => "pass"}
+      @backend.store_object(@robject)
+      @robject2 = @backend.fetch_object(@bucket.name, "reload")
+      @robject2.data["test"] = "second"
+      @backend.store_object(@robject2, :returnbody => true)
+    end
+
+    it "modifies the object with the reloaded data" do
+      @backend.reload_object(@robject)
+    end
+
+    [1, 2, 3, :one, :quorum, :all, :default].each do |q|
+      it "accepts a valid R value of #{q.inspect} for the request" do
+        @backend.reload_object(@robject, :r => q)
+      end
+
+      it "accepts a valid PR value of #{q.inspect} for the request" do
+        @backend.reload_object(@robject, :pr => q)
+      end
+    end
+
+    after do |example|
+      unless example.pending?
+        expect(@robject.vclock).to eq(@robject2.vclock)
+        expect(@robject.data['test']).to eq("second")
+      end
+    end
+  end
+
+  # store_object
+  context "storing an object" do
+    before do
+      @robject = Riak::RObject.new(@bucket, random_key)
+      @robject.content_type = "application/json"
+      @robject.data = {"test" => "pass"}
+    end
+
+    it "saves the object" do
+      @backend.store_object(@robject)
+    end
+
+    it "modifies the object with the returned data if returnbody" do
+      @backend.store_object(@robject, :returnbody => true)
+      expect(@robject.vclock).to be_present
+    end
+
+    [1, 2, 3, :one, :quorum, :all, :default].each do |q|
+      it "accepts a W value of #{q.inspect} for the request" do
+        @backend.store_object(@robject, :returnbody => false, :w => q)
+        expect(@bucket.exists?(@robject.key)).to be_truthy
+      end
+
+      it "accepts a DW value of #{q.inspect} for the request" do
+        @backend.store_object(@robject, :returnbody => false, :w => :all, :dw => q)
+      end
+
+      it "accepts a PW value of #{q.inspect} for the request" do
+        @backend.store_object(@robject, :returnbody => false, :pw => q)
+      end
+    end
+
+    it "stores an object with indexes" do
+      @robject.indexes['foo_bin'] << 'bar'
+      @backend.store_object(@robject, :returnbody => true)
+      expect(@robject.indexes).to include('foo_bin')
+      expect(@robject.indexes['foo_bin']).to include('bar')
+    end
+
+    after do
+      expect { @backend.fetch_object(@bucket.name, @robject.key) }.not_to raise_error
+    end
+  end
+
+  # delete_object
+  context "deleting an object" do
+    before do
+      @obj = Riak::RObject.new(@client.bucket("test"), "delete")
+      @obj.content_type = "application/json"
+      @obj.data = [1]
+      @backend.store_object(@obj)
+    end
+
+    it "removes the object" do
+      @backend.delete_object("test", "delete")
+      expect(@obj.bucket.exists?("delete")).to be_falsey
+    end
+
+    [1, 2, 3, :one, :quorum, :all, :default].each do |q|
+      it "accepts an RW value of #{q.inspect} for the request" do
+        @backend.delete_object("test", "delete", :rw => q)
+      end
+    end
+
+    it "accepts a vclock value for the request" do
+      @backend.delete_object("test", "delete", :vclock => @obj.vclock)
+    end
+
+    after do
+      expect(@obj.bucket.exists?("delete")).to be_falsey
+    end
+  end
+
+  # get_bucket_props
+  context "fetching bucket properties" do
+    it "fetches a hash of bucket properties" do
+      props = @backend.get_bucket_props("test")
+      expect(props).to be_kind_of(Hash)
+      expect(props).to include("n_val")
+    end
+  end
+
+  # set_bucket_props
+  context "setting bucket properties" do
+    it "stores properties for the bucket" do
+      @backend.set_bucket_props("test", {"n_val" => 3})
+      expect(@backend.get_bucket_props("test")["n_val"]).to eq(3)
+    end
+  end
+
+  # list_keys
+  context "listing keys in a bucket" do
+    before do
+      @list_bucket = random_bucket 'unified_backend_list_keys'
+      obj = Riak::RObject.new(@list_bucket, "keys")
+      obj.content_type = "application/json"
+      obj.data = [1]
+      @backend.store_object(obj)
+    end
+
+    it "fetches an array of string keys" do
+      expect(@backend.list_keys(@list_bucket)).to eq(["keys"])
+    end
+
+    context "streaming through a block" do
+      it "handles a large number of keys" do
+        obj = Riak::RObject.new(@list_bucket)
+        obj.content_type = "application/json"
+        obj.data = [1]
+        750.times do |i|
+          obj.key = i.to_s
+          obj.store(:w => 1, :dw => 0, :returnbody => false)
+        end
+        @backend.list_keys(@list_bucket) do |keys|
+          expect(keys).to be_all {|k| k == 'keys' || (0..749).include?(k.to_i) }
+        end
+      end
+
+      it "passes an array of keys to the block" do
+        @backend.list_keys(@list_bucket) do |keys|
+          expect(keys).to eq(["keys"]) unless keys.empty?
+        end
+      end
+
+      it "allows requests issued inside the block to execute" do
+        errors = []
+        @backend.list_keys(@list_bucket) do |keys|
+          keys.each do |key|
+            begin
+              @client.get_object(@list_bucket, key)
+            rescue => e
+              errors << e
+            end
+          end
+        end
+        expect(errors).to be_empty
+      end
+    end
+  end
+
+  # list_buckets
+  context "listing buckets" do
+    before do
+      obj = Riak::RObject.new(@client.bucket("test"), "buckets")
+      obj.content_type = "application/json"
+      obj.data = [1]
+      @backend.store_object(obj)
+    end
+
+    it "fetches a list of string bucket names" do
+      list = @backend.list_buckets
+      expect(list).to be_kind_of(Array)
+      expect(list).to include("test")
+    end
+  end
+
+  # get_index
+  context "querying secondary indexes" do
+    before do
+      50.times do |i|
+        @client.bucket('test').new(i.to_s).tap do |obj|
+          obj.indexes["index_int"] << i
+          obj.data = [i]
+          @backend.store_object(obj)
+        end
+      end
+    end
+
+    it "finds keys for an equality query" do
+      expect(@backend.get_index('test', 'index_int', 20)).to eq(["20"])
+    end
+
+    it "finds keys for a range query" do
+      expect(@backend.get_index('test', 'index_int', 19..21)).to match_array(%w(19 20 21))
+    end
+
+    it "returns an empty array for a query that does not match any keys" do
+      expect(@backend.get_index('test', 'index_int', 10000)).to eq([])
+    end
+  end
+
+  # mapred
+  context "performing MapReduce" do
+    before do
+      @mapred_bucket = random_bucket("mapred_test")
+      obj = Riak::RObject.new(@mapred_bucket, "1")
+      obj.content_type = "application/json"
+      obj.data = {"value" => "1" }
+      @backend.store_object(obj)
+      @mapred = Riak::MapReduce.new(@client).
+        add(@mapred_bucket.name).
+        map("Riak.mapValuesJson", :keep => true)
+    end
+
+    it "doesn't raise an error without phases" do
+      @mapred.query.clear
+      @backend.mapred(@mapred)
+    end
+
+    it "performs a simple MapReduce request" do
+      expect(@backend.mapred(@mapred)).to eq([{"value" => "1"}])
+    end
+
+    it "returns an ordered array of results when multiple phases are kept" do
+      @mapred.reduce("function(objects){ return objects; }", :keep => true)
+      expect(@backend.mapred(@mapred)).to eq([[{"value" => "1"}], [{"value" => "1"}]])
+    end
+
+    it "doesn't remove empty phase results when multiple phases are kept" do
+      @mapred.reduce("function(){ return []; }", :keep => true)
+      expect(@backend.mapred(@mapred)).to eq([[{"value" => "1"}], []])
+    end
+
+    context "streaming results through a block" do
+      it "passes phase number and result to the block" do
+        @backend.mapred(@mapred) do |phase, result|
+          unless result.empty?
+            expect(phase).to eq(0)
+            expect(result).to eq([{"value" => "1"}])
+          end
+        end
+      end
+
+      it "allows requests issued inside the block to execute" do
+        errors = []
+        @backend.mapred(@mapred) do |phase, result|
+          unless result.empty?
+            result.each do |v|
+              begin
+                @client.get_object(@mapred_bucket, v['value'])
+              rescue => e
+                errors << e
+              end
+            end
+          end
+        end
+        expect(errors).to be_empty
+      end
+    end
+  end
+
+  # search
+  context "searching fulltext indexes" do
+    # Search functionality existed since Riak 0.13, but PBC only
+    # entered into the picture in 1.2. PBC can support searches
+    # against 1.1 and earlier nodes using MapReduce emulation, but has
+    # limited functionality. We'll enter separate tests for the
+    # pre-1.2 functionality.
+    include_context "search corpus setup"
+
+    it 'finds indexed documents, returning ids' do
+      results = @backend.search @search_bucket.name, 'predictable operations behavior', fl: '_yz_rk', df: 'text'
+      expect(results).to have_key 'docs'
+      expect(results).to have_key 'max_score'
+      expect(results).to have_key 'num_found'
+
+      found = results['docs'].any? do |e|
+        e['_yz_rk'] == 'bitcask-10'
+      end
+
+      expect(found).to be_truthy
+    end
+
+    it 'finds indexed documents, returning documents' do
+      # For now use '*' until #122 is merged into riak_search
+      results = @backend.search @search_bucket.name, 'predictable operations behavior', fl: '_yz_rk', df: 'text'
+      expect(results).to have_key 'docs'
+      expect(results).to have_key 'max_score'
+      expect(results).to have_key 'num_found'
+
+      found = results['docs'].any? do |e|
+        e['_yz_rk'] == 'bitcask-10'
+      end
+
+      expect(found).to be_truthy
+    end
+  end
+end