elastictastic 0.5.0 → 0.10.2

data/spec/examples/properties_spec.rb

@@ -34,6 +34,59 @@ describe Elastictastic::Properties do
     it 'should map embedded object fields' do
       properties['author']['properties']['id']['type'].should == 'integer'
     end
+
+    it 'should set boost field' do
+      mapping['post']['_boost'].should == { 'name' => 'score', 'null_value' => 1.0 }
+    end
+
+    it 'should set routing param if given' do
+      Photo.mapping['photo']['_routing'].should == {
+        'required' => true,
+        'path' => 'post_id'
+      }
+    end
+  end
+
+  describe ':preset' do
+    before do
+      Elastictastic.config.presets[:silly] = { :type => 'integer', :store => 'yes', :index => 'no' }
+    end
+
+    let :clazz do
+      Class.new do
+        include Elastictastic::Document
+
+        def self.name
+          'Clazz'
+        end
+
+        field :title, :type => 'string', :preset => 'silly'
+        field :created, :preset => :silly
+        field :multi, :type => 'string' do
+          field :searchable, :preset => 'silly', :index => 'yes'
+        end
+      end
+    end
+
+    let(:properties) { clazz.mapping.values.first['properties'] }
+
+    it 'should apply preset values' do
+      properties['created'].should == {
+        'type' => 'integer', 'store' => 'yes', 'index' => 'no'
+      }
+    end
+
+    it 'should override preset values with given values' do
+      properties['title'].should == {
+        'type' => 'string', 'store' => 'yes', 'index' => 'no'
+      }
+    end
+
+    it 'should apply presets to field alternates' do
+      properties['multi']['fields']['searchable'].should == {
+        'store' => 'yes', 'index' => 'yes', 'type' => 'integer'
+      }
+    end
   end
 
   describe '#elasticsearch_doc' do

data/spec/examples/rotor_spec.rb

@@ -0,0 +1,132 @@
+require File.expand_path('../spec_helper', __FILE__)
+
+describe Elastictastic::Rotor do
+  let(:client) { Elastictastic::Client.new(config) }
+  let(:last_request) { FakeWeb.last_request }
+
+  context 'without backoff' do
+    let(:config) do
+      Elastictastic::Configuration.new.tap do |config|
+        config.hosts = ['http://es1.local', 'http://es2.local']
+      end
+    end
+
+    it 'should alternate requests between hosts' do
+      expect do
+        2.times do
+          1.upto 2 do |i|
+            host_status(i => true)
+            client.get('default', 'post', '1')
+          end
+        end
+      end.not_to raise_error # We can't check the hostname of last_request in Fakeweb
+    end
+
+    context 'if one host fails' do
+      let!(:now) { Time.now.tap { |now| Time.stub(:now).and_return(now) }}
+
+      before do
+        host_status(1 => false, 2 => true)
+      end
+
+      it 'should try the next host' do
+        client.get('default', 'post', '1').should == { 'success' => true }
+      end
+
+      it 'should pass PUT body to retry' do
+        client.update('default', 'post', '1', { 'title' => 'pizza' })
+        FakeWeb.last_request.body.should == { 'title' => 'pizza' }.to_json
+      end
+    end
+
+    context 'if all hosts fail' do
+      let!(:now) { Time.now.tap { |now| Time.stub(:now).and_return(now) }}
+
+      before do
+        host_status(1 => false, 2 => false)
+      end
+
+      it 'should raise error if no hosts respond' do
+        expect { client.get('default', 'post', '1') }.to(raise_error Elastictastic::NoServerAvailable)
+      end
+    end
+  end
+
+  context 'with backoff' do
+    let(:config) do
+      Elastictastic::Configuration.new.tap do |config|
+        config.hosts = ['http://es1.local', 'http://es2.local']
+        config.backoff_threshold = 2
+        config.backoff_start = 1
+        config.backoff_max = 4
+      end
+    end
+    let!(:time) { Time.now.tap { |time| Time.stub(:now).and_return(time) }}
+
+    before do
+      host_status(1 => false, 2 => true)
+    end
+
+    it 'should retry immediately before reaching initial failure count' do
+      client.get('default', 'post', '1')
+      expect { client.get('default', 'post', '1') }.to change(FakeWeb.requests, :length).by(2)
+    end
+
+    it 'should back off after initial failure count reached' do
+      2.times { client.get('default', 'post', '1') }
+      expect { client.get('default', 'post', '1') }.to change(FakeWeb.requests, :length).by(1)
+    end
+
+    it 'should retry after initial backoff period elapses' do
+      3.times { client.get('default', 'post', '1') }
+      Time.stub(:now).and_return(time + 1)
+      expect { client.get('default', 'post', '1') }.to change(FakeWeb.requests, :length).by(2)
+    end
+
+    it 'should double backoff after another failure' do
+      3.times { client.get('default', 'post', '1') }
+      Time.stub(:now).and_return(time + 1)
+      client.get('default', 'post', '1')
+      Time.stub(:now).and_return(time + 2)
+      expect { client.get('default', 'post', '1') }.to change(FakeWeb.requests, :length).by(1)
+    end
+
+    it 'should cap backoff interval at backoff_max' do
+      2.times { client.get('default', 'post', '1') } # first backoff - 1 second
+      Time.stub(:now).and_return(time + 1)
+      client.get('default', 'post', '1') # second backoff - 2 seconds
+      Time.stub(:now).and_return(time + 3)
+      client.get('default', 'post', '1') # third backoff - 4 seconds
+      Time.stub(:now).and_return(time + 7)
+      client.get('default', 'post', '1') # fourth backoff - 4 seconds again
+      Time.stub(:now).and_return(time + 11)
+      expect { client.get('default', 'post', '1') }.to change(FakeWeb.requests, :length).by(2)
+    end
+
+    it 'should reset backoff after a successful request' do
+      2.times { client.get('default', 'post', '1') } # initial backoff - 1 second
+      host_status(1 => true, 2 => true)
+      Time.stub(:now).and_return(time + 1)
+      2.times { client.get('default', 'post', '1') } # first one will go to es2 because of rotation. second one has success so es1 should reset
+      host_status(1 => false, 2 => true)
+      client.get('default', 'post', '1') # should be willing to immediately retry
+      host_status(1 => true, 2 => false)
+      expect { client.get('default', 'post', '1') }.to_not raise_error # only will succeed if it retries #1
+    end
+  end
+
+  private
+
+  def host_status(statuses)
+    FakeWeb.clean_registry
+    statuses.each_pair do |i, healthy|
+      url = %r(^http://es#{i}.local/)
+      if healthy
+        options = { :body => '{"success":true}' }
+      else
+        options = { :exception => Errno::ECONNREFUSED }
+      end
+      FakeWeb.register_uri(:any, url, options)
+    end
+  end
+end

data/spec/examples/scope_spec.rb

@@ -7,7 +7,7 @@ describe Elastictastic::Scope do
   include Elastictastic::TestHelpers
 
   let(:last_request) { FakeWeb.last_request }
-  let(:last_request_body) { JSON.parse(last_request.body) }
+  let(:last_request_body) { Elastictastic.json_decode(last_request.body) }
   let(:last_request_path) { last_request.path.split('?', 2)[0] }
   let(:last_request_params) { last_request.path.split('?', 2)[1].try(:split, '&') }
 
@@ -20,7 +20,7 @@ describe Elastictastic::Scope do
       let(:scroll_requests) { FakeWeb.requests[1..-1] }
 
       before do
-        @scroll_ids = stub_elasticsearch_scan(
+        @scroll_ids = stub_es_scan(
           'default', 'post', 2, *make_hits(3)
         )
       end
@@ -41,7 +41,7 @@ describe Elastictastic::Scope do
         end
 
         it 'should send query in data for initial search' do
-          scan_request.body.should == scope.params.to_json
+          scan_request.body.should == Elastictastic.json_encode(scope.params)
         end
 
         it 'should send POST request initially' do
@@ -81,7 +81,7 @@ describe Elastictastic::Scope do
       let(:scope) { Post.from(10).size(10) }
 
       before do
-        stub_elasticsearch_search(
+        stub_es_search(
           'default', 'post', 'hits' => {
             'total' => 2,
             'hits' => make_hits(2)
@@ -109,18 +109,35 @@ describe Elastictastic::Scope do
       end
     end # context 'with from/size'
 
+    describe 'with page out of range' do
+      let(:scope) { Post.from(10).size(10) }
+
+      before do
+        stub_es_search(
+          'default', 'post', 'hits' => {
+            'total' => 2,
+            'hits' => []
+          }
+        )
+      end
+
+      it 'should return empty array of results' do
+        scope.to_a.should == []
+      end
+    end
+
     context 'with sort but no from/size' do
       let(:scope) { Post.sort(:title => 'asc') }
       let(:requests) { FakeWeb.requests }
       let(:request_bodies) do
         requests.map do |request|
-          JSON.parse(request.body)
+          Elastictastic.json_decode(request.body)
         end
       end
 
       before do
         Elastictastic.config.default_batch_size = 2
-        stub_elasticsearch_search(
+        stub_es_search(
           'default', 'post',
           make_hits(3).each_slice(2).map { |batch| { 'hits' => { 'hits' => batch, 'total' => 3 }} }
         )
@@ -169,6 +186,38 @@ describe Elastictastic::Scope do
         scope.each { |post| post.should be_persisted }
       end
     end # context 'with sort but no from/size'
+
+    describe 'with routing' do
+      it 'should send routing param in single-search query' do
+        stub_es_search(
+          'default', 'post',
+          'hits' => {'total' => 2, 'hits' => make_hits(2)}
+        )
+        Post.routing('7').size(10).to_a
+        last_request_uri.query.split('&').should include('routing=7')
+      end
+
+      it 'should send routing param in scan query' do
+        stub_es_scan(
+          'default', 'post', 2, *make_hits(3)
+        )
+        Post.routing('7').to_a
+        URI.parse(FakeWeb.requests.first.path).query.split('&').
+          should include('routing=7')
+      end
+
+      it 'should send routing param in batch-search queries' do
+        Elastictastic.config.default_batch_size = 2
+        stub_es_search(
+          'default', 'post',
+          make_hits(3).each_slice(2).map { |batch| { 'hits' => { 'hits' => batch, 'total' => 3 }} }
+        )
+        Post.routing('7').sort(:score).to_a
+        FakeWeb.requests.each do |request|
+          URI.parse(request.path).query.split('&').should include('routing=7')
+        end
+      end
+    end
   end # describe '#each'
 
   describe 'hit metadata' do
@@ -203,7 +252,7 @@ describe Elastictastic::Scope do
       let(:scope) { Post }
 
       before do
-        stub_elasticsearch_scan('default', 'post', 2, *hits)
+        stub_es_scan('default', 'post', 2, *hits)
       end
 
       it_should_behave_like 'enumerator with hit metadata'
@@ -214,7 +263,7 @@ describe Elastictastic::Scope do
 
       before do
         batches = hits.each_slice(2).map { |batch| { 'hits' => { 'hits' => batch, 'total' => 3 }} }
-        stub_elasticsearch_search('default', 'post', batches)
+        stub_es_search('default', 'post', batches)
       end
 
       it_should_behave_like 'enumerator with hit metadata'
@@ -224,7 +273,7 @@ describe Elastictastic::Scope do
       let(:scope) { Post.size(3) }
 
       before do
-        stub_elasticsearch_search('default', 'post', 'hits' => { 'hits' => hits, 'total' => 3 })
+        stub_es_search('default', 'post', 'hits' => { 'hits' => hits, 'total' => 3 })
       end
 
       it_should_behave_like 'enumerator with hit metadata'
@@ -234,7 +283,7 @@ describe Elastictastic::Scope do
   describe '#count' do
     context 'with no operations performed yet' do
       let!(:count) do
-        stub_elasticsearch_search('default', 'post', 'hits' => { 'total' => 3 })
+        stub_es_search('default', 'post', 'hits' => { 'total' => 3 })
         Post.all.count
       end
 
@@ -249,7 +298,7 @@ describe Elastictastic::Scope do
 
     context 'with scan search performed' do
       let!(:count) do
-        stub_elasticsearch_scan(
+        stub_es_scan(
           'default', 'post', 2, *make_hits(3)
         )
         scope = Post.all
@@ -268,7 +317,7 @@ describe Elastictastic::Scope do
 
     context 'with paginated search performed' do
       let!(:count) do
-        stub_elasticsearch_search(
+        stub_es_search(
           'default', 'post', 'hits' => {
             'hits' => make_hits(3),
             'total' => 3
@@ -290,7 +339,7 @@ describe Elastictastic::Scope do
 
     context 'with paginated scan performed' do
       let!(:count) do
-        stub_elasticsearch_search(
+        stub_es_search(
           'default', 'post', 'hits' => {
             'hits' => make_hits(2),
             'total' => 2
@@ -309,6 +358,17 @@ describe Elastictastic::Scope do
         FakeWeb.should have(1).request
       end
     end # context 'with paginated scan performed'
+
+    context 'with routing specified' do
+      let!(:count) do
+        stub_es_search('default', 'post', 'hits' => { 'total' => 3 })
+        Post.routing(7).count
+      end
+
+      it 'should send routing parameter' do
+        last_request_uri.query.split('&').should include('routing=7')
+      end
+    end
   end # describe '#count'
 
   describe '#all_facets' do
@@ -327,7 +387,7 @@ describe Elastictastic::Scope do
 
     context 'with no requests performed' do
       let!(:facets) do
-        stub_elasticsearch_search(
+        stub_es_search(
           'default', 'post',
           'hits' => { 'hits' => [], 'total' => 2 },
           'facets' => facet_response
@@ -346,7 +406,7 @@ describe Elastictastic::Scope do
 
     context 'with count request performed' do
       let!(:facets) do
-        stub_elasticsearch_search(
+        stub_es_search(
           'default', 'post',
           'hits' => { 'hits' => [], 'total' => 2 },
           'facets' => facet_response
@@ -366,7 +426,7 @@ describe Elastictastic::Scope do
 
     context 'with single-page search performed' do
       let!(:facets) do
-        stub_elasticsearch_search(
+        stub_es_search(
           'default', 'post',
           'hits' => { 'hits' => make_hits(2), 'total' => 2 },
           'facets' => facet_response
@@ -387,7 +447,7 @@ describe Elastictastic::Scope do
 
     context 'with multi-page search performed' do
       let!(:facets) do
-        stub_elasticsearch_search(
+        stub_es_search(
           'default', 'post',
           'hits' => { 'hits' => make_hits(2), 'total' => 2 },
           'facets' => facet_response
@@ -410,7 +470,7 @@ describe Elastictastic::Scope do
   describe '#first' do
     shared_examples_for 'first method' do
       before do
-        stub_elasticsearch_search(
+        stub_es_search(
           index, 'post', 'hits' => {
             'total' => 12,
             'hits' => make_hits(1)

data/spec/examples/search_spec.rb

@@ -6,6 +6,8 @@ require File.expand_path('../spec_helper', __FILE__)
 # search and dealing with the results
 #
 describe Elastictastic::Search do
+  include Elastictastic::TestHelpers
+
   describe '#to_params' do
     {
       'query' => { 'match_all' => {} },
@@ -47,6 +49,30 @@ describe Elastictastic::Search do
     end
   end
 
+  describe '#[]' do
+    it 'should run #first query with an integer argument' do
+      stub_es_search('default', 'post', 'hits' => {
+        'total' => '2',
+        'hits' => [generate_es_hit('post', :id => '1')]
+      })
+      Post.all[4].id.should == '1'
+      last_request_json['from'].should == 4
+      last_request_json['size'].should == 1
+    end
+
+    it 'should add from/size to scope with a range argument' do
+      params = Post.all[2..4].params
+      params['from'].should == 2
+      params['size'].should == 3
+    end
+
+    it 'should add from/size to scope with an end-excluded range argument' do
+      params = Post.all[2...4].params
+      params['from'].should == 2
+      params['size'].should == 2
+    end
+  end
+
   describe 'merging' do
     let(:scope) { Post }
 

data/spec/examples/validation_spec.rb

@@ -21,12 +21,18 @@ describe Elastictastic::Validations do
     it 'should raise Elastictastic::RecordInvalid for save!' do
       expect { post.save! }.to raise_error(Elastictastic::RecordInvalid)
     end
+
+    it 'should save successfully if validations disabled' do
+      stub_es_create('default', 'post')
+      post.save(:validate => false)
+      FakeWeb.last_request.path.should == '/default/post'
+    end
   end
 
   describe 'with valid data' do
     let(:post) { Post.new }
 
-    before { stub_elasticsearch_create('default', 'post') }
+    before { stub_es_create('default', 'post') }
 
     it 'should be valid' do
       post.should be_valid

data/spec/models/author.rb

@@ -1,5 +1,5 @@
 class Author
-  include Elastictastic::NestedDocument
+  include Elastictastic::EmbeddedDocument
 
   field :id, :type => 'integer'
   field :name

data/spec/models/blog.rb

@@ -1,5 +1,7 @@
 class Blog
   include Elastictastic::Document
 
+  field :name
+
   has_many :posts
 end

data/spec/models/comment.rb

@@ -1,5 +1,5 @@
 class Comment
-  include Elastictastic::NestedDocument
+  include Elastictastic::EmbeddedDocument
 
   field :body
 end

data/spec/models/photo.rb

@@ -0,0 +1,9 @@
+class Photo
+  include Elastictastic::Document
+
+  field :post_id, :type => 'integer'
+  field :path
+  field :caption
+
+  route_with :post_id, :required => true
+end

data/spec/models/post.rb

@@ -3,12 +3,15 @@ class Post
 
   field :title
   field :comments_count, :type => 'integer'
+  field :score, :type => 'integer'
   field :tags, :index => 'analyzed' do
     field :non_analyzed, :index => 'not_analyzed'
   end
   field :created_at, :type => 'date'
   field :published_at, :type => 'date'
 
+  boost :score
+
   embed :author
   embed :comments