elastictastic 0.5.0 → 0.10.2

data/spec/examples/optimistic_locking_spec.rb

@@ -0,0 +1,417 @@
+require File.expand_path('../spec_helper', __FILE__)
+require 'ruby-debug'
+
+describe Elastictastic::OptimisticLocking do
+  include Elastictastic::TestHelpers
+
+  shared_examples_for 'updatable scope' do
+    let :post do
+      scope.new.tap do |post|
+        post.id = '123abc'
+        post.version = 1
+        post.persisted!
+      end
+    end
+
+    context 'when save is cancelled' do
+      before do
+        stub_request_json(
+          :get,
+          match_es_resource(index, 'post', '123abc'),
+          generate_es_hit('post', :id => '123abc', :index => index, :version => 1).merge('exists' => true),
+          generate_es_hit('post', :id => '123abc', :index => index, :version => 2, :source => { :title => 'Hey' }).merge('exists' => true)
+        )
+      end
+
+      describe '::update' do
+        before do
+          scope.update('123abc') do
+            raise Elastictastic::CancelSave
+          end
+        end
+
+        it 'should have 1 request for fetch' do
+          FakeWeb.should have(1).requests # 1 for fetch
+        end
+      end
+
+      describe '::create_or_update' do
+        before do
+          scope.create_or_update('123abc') do
+            raise Elastictastic::CancelSave
+          end
+        end
+
+        it 'should make no requests' do
+          FakeWeb.should have(0).requests
+        end
+      end
+    end
+
+    context 'when version conflict raised from discrete persistence' do
+      describe '#save' do
+        before do
+          stub_request_json(
+            :put,
+            match_es_resource(index, 'post', '123abc'),
+            version_conflict
+          )
+        end
+
+        it 'should raise VersionConflict' do
+          expect { post.save }.to raise_error(Elastictastic::ServerError::VersionConflictEngineException)
+        end
+
+        it 'should yield VersionConflict when called with block' do
+          ex = nil
+          post.save { |e| ex = e }
+          ex.should be_a(Elastictastic::ServerError::VersionConflictEngineException)
+        end
+      end # describe '#save'
+
+      describe '::update' do
+        before do
+          stub_request_json(
+            :get,
+            match_es_resource(index, 'post', '123abc'),
+            generate_es_hit('post', :id => '123abc', :index => index, :version => 1).merge('exists' => true),
+            generate_es_hit('post', :id => '123abc', :index => index, :version => 2, :source => { :title => 'Hey' }).merge('exists' => true)
+          )
+          stub_request_json(
+            :put,
+            match_es_resource(index, 'post', '123abc'),
+            version_conflict,
+            generate_es_hit('post', :id => '123abc', :version => 3, :index => index)
+          )
+          scope.update('123abc') do |post|
+            post.comments_count = 3
+          end
+        end
+
+        it 'should make four requests' do
+          FakeWeb.should have(4).requests
+        end
+
+        it 'should add preference=_primary to get request' do
+          URI.parse(FakeWeb.requests.first.path).query.split('&').
+            should include('preference=_primary_first')
+        end
+
+        it 'should make final update with modification in update block' do
+          last_request_json['comments_count'].should == 3
+        end
+
+        it 'should make final update with data from latest version of doc' do
+          last_request_json['title'].should == 'Hey'
+        end
+
+        it 'should make final update with correct version' do
+          last_request_uri.query.split('&').should include('version=2')
+        end
+
+        it 'should send final update to correct index' do
+          last_request_uri.path.split('/')[1].should == index
+        end
+      end # describe '::update'
+
+      describe '::update_each' do
+        let(:last_update_request) do
+          FakeWeb.requests.reverse.find { |req| req.method == 'PUT' }
+        end
+
+        before do
+          stub_es_scan(
+            index, 'post', 100,
+            generate_es_hit('post', :index => index, :id => '1'),
+            generate_es_hit('post', :index => index, :id => '2')
+          )
+          stub_es_get(index, 'post', '2', { :title => 'Hey' }, 2)
+          stub_es_update(index, 'post', '1')
+          stub_request_json(
+            :put,
+            match_es_resource(index, 'post', '2'),
+            version_conflict,
+            generate_es_hit('post', :id => '2', :index => index, :version => 3)
+          )
+          scope.update_each do |post|
+            post.comments_count = 2
+          end
+        end
+
+        it 'should retry unsuccessful updates' do
+          FakeWeb.should have(7).requests # initiate scan, 2 cursors, update '1', update '2' (fail), get '2', update '2'
+        end
+
+        it 'should re-perform update on failed document' do
+          URI.parse(last_update_request.path).path.should == "/#{index}/post/2"
+        end
+
+        it 'should send data from latest version in persistence' do
+          Elastictastic.json_decode(last_update_request.body)['title'].should == 'Hey'
+        end
+
+        it 'should send data from update block' do
+          Elastictastic.json_decode(last_update_request.body)['comments_count'].should == 2
+        end
+
+        it 'should update with latest version' do
+          URI.parse(last_update_request.path).query.split('&').should include('version=2')
+        end
+      end # describe '::update_each'
+    end # context 'when version conflict raised from discrete persistence'
+
+    context 'when version conflict raised from bulk persistence' do
+      describe '#save' do
+        before do
+          stub_es_bulk(
+            'index' => {
+              '_index' => index, '_type' => 'post', '_id' => '123abc',
+              'error' => version_conflict['error']
+            }
+          )
+        end
+
+        it 'should raise error' do
+          expect { Elastictastic.bulk { post.save }}.to raise_error(Elastictastic::ServerError::VersionConflictEngineException)
+        end
+
+        it 'should yield an error when called with block' do
+          ex = nil
+          Elastictastic.bulk { post.save { |e| ex = e }}
+          ex.should be_a(Elastictastic::ServerError::VersionConflictEngineException)
+        end
+      end
+
+      describe '::update' do
+        before do
+          stub_request_json(
+            :get,
+            match_es_resource(index, 'post', 'abc123'),
+            generate_es_hit('post', :id => 'abc123', :index => index, :version => 1),
+            generate_es_hit('post', :id => 'abc123', :index => index, :version => 2, :source => { :title => 'Hey' })
+          )
+          stub_es_bulk(
+            'index' => {
+              '_index' => index, '_type' => 'post', '_id' => '123abc',
+              'error' => version_conflict['error']
+            }
+          )
+          stub_es_update(index, 'post', 'abc123')
+
+          Elastictastic.bulk do
+            scope.update('abc123') do |post|
+              post.comments_count = 2
+            end
+          end
+        end
+
+        it 'should make 4 requests' do
+          FakeWeb.should have(4).requests
+        end
+
+        it 'should send data from block in last request' do
+          last_request_json['comments_count'].should == 2
+        end
+
+        it 'should send data from most recent version in last request' do
+          last_request_json['title'].should == 'Hey'
+        end
+
+        it 'should send correct version in last request' do
+          last_request_uri.query.split('&').should include('version=2')
+        end
+
+        it 'should send last request to correct index' do
+          last_request_uri.path.split('/')[1].should == index
+        end
+      end # describe '::update'
+
+      describe '::update_each' do
+        before do
+          stub_es_scan(
+            index, 'post', 100,
+            generate_es_hit('post', :index => index, :id => '1'),
+            generate_es_hit('post', :index => index, :id => '2')
+          )
+          stub_request_json(
+            :post,
+            match_es_path('/_bulk'),
+            'items' => [
+              { 'index' => generate_es_hit('post', :index => index, :id => '1').except('_source').merge('ok' => true) },
+              { 'index' => generate_es_hit('post', :index => index, :id => '2').except('_source').merge(version_conflict.slice('error')) }
+            ]
+          )
+          stub_es_get(index, 'post', '2', { 'title' => 'Hey' }, 2)
+          stub_es_update(index, 'post', '2', 3)
+          Elastictastic.bulk do 
+            scope.update_each { |post| post.comments_count = 2 }
+          end
+        end
+
+        it 'should retry failed update' do
+          FakeWeb.should have(6).requests # start scan, 2 cursor reads, bulk update, reload '2', update '2'
+        end
+
+        it 'should update failed document' do
+          last_request_uri.path.should == "/#{index}/post/2"
+        end
+
+        it 'should update conflicted document with reloaded data' do
+          last_request_json['title'].should == 'Hey'
+        end
+
+        it 'should update conflicted document with data from block' do
+          last_request_json['comments_count'].should == 2
+        end
+
+        it 'should update conflicted document with proper version' do
+          last_request_uri.query.split('&').should include('version=2')
+        end
+      end
+    end # context 'when version conflict raised from bulk persistence'
+  end # shared_examples_for 'updatable scope'
+
+  describe 'default scope' do
+    let(:scope) { Post }
+    let(:index) { 'default' }
+    let :version_conflict do
+      {
+        'error' => "VersionConflictEngineException: [[#{index}][3] [post][abc123]: version conflict, current[2], required[1]]",
+        'status' => 409
+      }
+    end
+    it_should_behave_like 'updatable scope'
+  end
+
+  describe 'scoped in index' do
+    let(:scope) { Post.in_index('my_index') }
+    let(:index) { 'my_index' }
+    let :version_conflict do
+      {
+        'error' => "VersionConflictEngineException: [[#{index}][3] [post][abc123]: version conflict, current[2], required[1]]",
+        'status' => 409
+      }
+    end
+    it_should_behave_like 'updatable scope'
+  end
+
+  describe 'default scope with nested exception' do
+    let(:scope) { Post }
+    let(:index) { 'default' }
+    let(:version_conflict) do
+      {
+        'error' => "RemoteTransportException: [[server][inet[/ip]][/index]]; nested: VersionConflictEngineException[[#{index}][0] [[post][abc123]: version conflict, current [2], required [1]]",
+        'status' => 409
+      }
+    end
+    it_should_behave_like 'updatable scope'
+  end
+
+  describe 'scoped in index with nested exception' do
+    let(:scope) { Post.in_index('my_index') }
+    let(:index) { 'my_index' }
+    let(:version_conflict) do
+      {
+        'error' => "RemoteTransportException: [[server][inet[/ip]][/index]]; nested: VersionConflictEngineException[[#{index}][0] [[post][abc123]: version conflict, current [2], required [1]]",
+        'status' => 409
+      }
+    end
+    it_should_behave_like 'updatable scope'
+  end
+
+  context '::update called on nonexistent document' do
+    before do
+      stub_es_get('default', 'post', '1', nil)
+    end
+
+    it 'should not do anything' do
+      expect { Post.update('1') { |post| post.title = 'bogus' }}.to_not raise_error
+    end
+  end
+
+  describe '::create_or_update' do
+    let :already_exists do
+      {
+        'error' => 'DocumentAlreadyExistsEngineException: [[my_index][2] [post][1]: document already exists]',
+        'status' => 409
+      }
+    end
+
+    let :version_conflict do
+      {
+        'error' => "VersionConflictEngineException: [[my_index][3] [post][1]: version conflict, current[2], required[1]]",
+        'status' => 409
+      }
+    end
+
+    context 'with discrete persistence' do
+      context "when document doesn't already exist" do
+        before do
+          stub_es_create('my_index', 'post', '1')
+          Post.in_index('my_index').create_or_update('1') { |post| post.title = 'hey' }
+        end
+
+        it 'should post to create endpoint' do
+          last_request.path.should == '/my_index/post/1/_create'
+        end
+
+        it 'should yield before saving' do
+          last_request_json['title'].should == 'hey'
+        end
+      end
+
+      context "when document already exists" do
+        before do
+          stub_request_json(
+            :put,
+            match_es_path('/my_index/post/1/_create'),
+            already_exists
+          )
+          stub_es_get('my_index', 'post', '1', :comments_count => 2)
+          stub_es_update('my_index', 'post', '1')
+          Post.in_index('my_index').create_or_update('1') { |post| post.title = 'hey' }
+        end
+
+        it 'should re-update data with correct version' do
+          last_request.path.should == '/my_index/post/1?version=1'
+        end
+
+        it 'should include data from storage' do
+          last_request_json['comments_count'].should == 2
+        end
+
+        it 'should include updated data from block' do
+          last_request_json['title'].should == 'hey'
+        end
+      end
+    end
+
+    context 'with bulk persistence' do
+      context 'when document already exists' do
+        before do
+          stub_es_bulk(
+            'create' => {
+              '_index' => 'my_index', '_type' => 'post', '_id' => '1',
+              'error' => already_exists['error']
+            }
+          )
+          stub_es_get('my_index', 'post', '1', 'comments_count' => 2)
+          stub_es_update('my_index', 'post', '1')
+          Elastictastic.bulk { Post.in_index('my_index').create_or_update('1') { |post| post.title = 'hey' } }
+        end
+
+        it 'should send update' do
+          last_request.path.should == '/my_index/post/1?version=1'
+        end
+
+        it 'should send data from existing document' do
+          last_request_json['comments_count'].should == 2
+        end
+
+        it 'should send data from block' do
+          last_request_json['title'].should == 'hey'
+        end
+      end
+    end
+  end
+end

data/spec/examples/parent_child_spec.rb

@@ -12,7 +12,7 @@ describe Elastictastic::ParentChild do
   describe 'child instance' do
     let(:post) { blog.posts.new }
     let(:blog) do
-      stub_elasticsearch_create('default', 'blog')
+      stub_es_create('default', 'blog')
       Blog.new.tap { |blog| blog.save }
     end
 
@@ -24,40 +24,74 @@ describe Elastictastic::ParentChild do
       blog.posts.from_hash('title' => 'hey').blog.should == blog
     end
 
+    describe 'when retrieved directly' do
+      let(:post) { Post.in_index('my_index').fields('_source', '_parent').first }
+
+      before do
+        stub_es_search(
+          'my_index', 'post',
+          'hits' => {
+            'total' => 1,
+            'hits' => [{
+              '_id' => '2',
+              '_type' => 'post',
+              '_index' => 'my_index',
+              '_source' => {},
+              'fields' => { '_parent' => '1' }
+            }]
+          }
+        )
+
+        stub_es_get(
+          'my_index', 'blog', '1',
+          { 'name' => 'Awesome Blog' }
+        )
+      end
+
+      it 'should retrieve parent via GET' do
+        post.blog.name.should == 'Awesome Blog'
+      end
+
+      it 'should lookup in correct index' do
+        post.blog
+        FakeWeb.last_request.path.split('?').first.should == '/my_index/blog/1'
+      end
+    end
+
     describe 'discrete persistence' do
       it 'should pass parent param on create' do
-        stub_elasticsearch_create('default', 'post')
+        stub_es_create('default', 'post')
         post.save
         URI.parse(FakeWeb.last_request.path).query.should == "parent=#{blog.id}"
       end
 
       it 'should pass parent param on update' do
-        stub_elasticsearch_create('default', 'post')
+        stub_es_create('default', 'post')
         post.save
-        stub_elasticsearch_update('default', 'post', post.id)
+        stub_es_update('default', 'post', post.id)
         post.save
-        URI.parse(FakeWeb.last_request.path).query.should == "parent=#{blog.id}"
+        URI.parse(FakeWeb.last_request.path).query.split('&').should include("parent=#{blog.id}")
       end
 
       it 'should pass parent on delete' do
-        stub_elasticsearch_create('default', 'post')
+        stub_es_create('default', 'post')
         post = blog.posts.new
         post.save
-        stub_elasticsearch_destroy('default', 'post', post.id)
+        stub_es_destroy('default', 'post', post.id)
         post.destroy
-        URI.parse(FakeWeb.last_request.path).query.should == "parent=#{blog.id}"
+        URI.parse(FakeWeb.last_request.path).query.split('&').should include("parent=#{blog.id}")
       end
     end
 
     describe 'bulk persistence' do
       let(:bulk_requests) do
         FakeWeb.last_request.body.split("\n").map do |line|
-          JSON.parse(line)
+          Elastictastic.json_decode(line)
         end
       end
 
       before do
-        stub_elasticsearch_bulk
+        stub_es_bulk
       end
 
       it 'should pass parent param on create' do
@@ -102,7 +136,7 @@ describe Elastictastic::ParentChild do
     end
 
     it 'should set index' do
-      stub_elasticsearch_create('my_index', 'blog')
+      stub_es_create('my_index', 'blog')
       blog = Blog.in_index('my_index').new
       blog.save
       post = blog.posts.new
@@ -112,7 +146,7 @@ describe Elastictastic::ParentChild do
 
   describe 'collection proxies' do
     let(:blog) do
-      stub_elasticsearch_create('my_index', 'blog')
+      stub_es_create('my_index', 'blog')
       Blog.in_index('my_index').new.tap { |blog| blog.save }
     end
     let(:posts) { blog.posts }
@@ -137,20 +171,20 @@ describe Elastictastic::ParentChild do
     end
 
     it 'should search correct index' do
-      stub_elasticsearch_scan('my_index', 'post', 100, { '_id' => '1' })
+      stub_es_scan('my_index', 'post', 100, { '_id' => '1' })
       posts.to_a.first.id.should == '1'
     end
 
     it 'should set routing to parent ID on get' do
-      stub_elasticsearch_get('my_index', 'post', 1)
+      stub_es_get('my_index', 'post', 1)
       blog.posts.find(1)
       URI.parse(FakeWeb.last_request.path).query.should == "routing=#{blog.id}"
     end
 
     it 'should set routing to parent ID on multiget' do
-      stub_elasticsearch_mget('my_index', 'post')
+      stub_es_mget('my_index', 'post')
       blog.posts.find(1, 2)
-      JSON.parse(FakeWeb.last_request.body).should == {
+      Elastictastic.json_decode(FakeWeb.last_request.body).should == {
         'docs' => [
           { '_id' => 1, 'routing' => blog.id },
           { '_id' => 2, 'routing' => blog.id }
@@ -160,14 +194,14 @@ describe Elastictastic::ParentChild do
 
     it 'should save transient instances when parent is saved' do
       post = posts.new
-      stub_elasticsearch_update('my_index', 'blog', blog.id)
-      stub_elasticsearch_create('my_index', 'post')
+      stub_es_update('my_index', 'blog', blog.id)
+      stub_es_create('my_index', 'post')
       blog.save
       post.should be_persisted
     end
 
     it 'should not attempt to save transient instances that are pending for save in a bulk block' do
-      stub_elasticsearch_bulk(
+      stub_es_bulk(
         { 'create' => { '_index' => 'my_index', '_type' => 'post', '_id' => '123', '_version' => 1, 'ok' => true }},
         { 'index' => { '_index' => 'my_index', '_type' => 'blog', '_id' => blog.id, '_version' => 2, 'ok' => true }}
       )
@@ -177,34 +211,34 @@ describe Elastictastic::ParentChild do
         blog.save
       end
       request_statements =
-        FakeWeb.last_request.body.each_line.map { |line| JSON.parse(line) }
+        FakeWeb.last_request.body.each_line.map { |line| Elastictastic.json_decode(line) }
       request_statements.length.should == 4
     end
 
     it 'should not save transient instances again' do
       post = posts.new
-      stub_elasticsearch_update('my_index', 'blog', blog.id)
-      stub_elasticsearch_create('my_index', 'post')
+      stub_es_update('my_index', 'blog', blog.id)
+      stub_es_create('my_index', 'post')
       blog.save
       FakeWeb.clean_registry
-      stub_elasticsearch_update('my_index', 'blog', blog.id)
+      stub_es_update('my_index', 'blog', blog.id)
       expect { blog.save }.to_not raise_error
     end
 
     it 'should populate parent when finding one' do
-      stub_elasticsearch_get('my_index', 'post', '1')
+      stub_es_get('my_index', 'post', '1')
       blog.posts.find('1').blog.should == blog
     end
 
     it 'should populate parent when finding many' do
-      stub_elasticsearch_mget('my_index', 'post', '1', '2')
+      stub_es_mget('my_index', 'post', '1', '2')
       blog.posts.find('1', '2').each do |post|
         post.blog.should == blog
       end
     end
 
     it 'should populate parent when retrieving first' do
-      stub_elasticsearch_search(
+      stub_es_search(
         'my_index', 'post',
         'total' => 1,
         'hits' => { 'hits' => [{ '_id' => '2', 'index' => 'my_index', '_type' => 'post' }]}
@@ -213,7 +247,7 @@ describe Elastictastic::ParentChild do
     end
 
     it 'should populate parent when iterating over cursor' do
-      stub_elasticsearch_scan(
+      stub_es_scan(
         'my_index', 'post', 100,
         '_id' => '1'
       )
@@ -221,7 +255,7 @@ describe Elastictastic::ParentChild do
     end
 
     it 'should populate parent when paginating' do
-      stub_elasticsearch_search(
+      stub_es_search(
         'my_index', 'post',
         'hits' => {
           'total' => 1,
@@ -232,15 +266,21 @@ describe Elastictastic::ParentChild do
     end
 
     it 'should iterate over transient instances along with retrieved results' do
-      stub_elasticsearch_scan('my_index', 'post', 100, '_id' => '1')
+      stub_es_scan('my_index', 'post', 100, '_id' => '1')
       post = blog.posts.new
       posts = blog.posts.to_a
       posts[0].id.should == '1'
       posts[1].should == post
     end
 
+    it 'should iterate over only transient instances if parent is transient' do
+      blog = Blog.new
+      post = blog.posts.new
+      blog.posts.to_a.should == [post]
+    end
+
     it 'should return transient instance as #first if no persisted results' do
-      stub_elasticsearch_search(
+      stub_es_search(
         'my_index', 'post', 'hits' => { 'total' => 0, 'hits' => [] })
       post = blog.posts.new
       blog.posts.first.should == post
@@ -274,7 +314,7 @@ describe Elastictastic::ParentChild do
 
   describe 'searching children directly' do
     before do
-      stub_elasticsearch_search(
+      stub_es_search(
         'default', 'post',
         'hits' => {
           'hits' => [
@@ -291,7 +331,7 @@ describe Elastictastic::ParentChild do
     let(:post) { Post.first }
 
     it 'should provide access to parent' do
-      stub_elasticsearch_get('default', 'blog', '3')
+      stub_es_get('default', 'blog', '3')
       post.blog.id.should == '3'
     end
 
@@ -300,7 +340,7 @@ describe Elastictastic::ParentChild do
     end
 
     it 'should save post without dereferencing parent' do
-      stub_elasticsearch_update('default', 'post', post.id)
+      stub_es_update('default', 'post', post.id)
       post.save
       URI.parse(FakeWeb.last_request.path).query.should == 'parent=3'
     end