elastictastic 0.5.0 → 0.10.2

data/lib/elastictastic.rb

@@ -1,80 +1,201 @@
 require 'active_support/core_ext'
 require 'active_model'
+
+require 'elastictastic/adapter'
+require 'elastictastic/basic_document'
 require 'elastictastic/errors'
+require 'elastictastic/client'
+require 'elastictastic/configuration'
+require 'elastictastic/discrete_persistence_strategy'
+require 'elastictastic/embedded_document'
+require 'elastictastic/field'
+require 'elastictastic/index'
+require 'elastictastic/middleware'
+require 'elastictastic/multi_get'
+require 'elastictastic/multi_search'
+require 'elastictastic/optimistic_locking'
+require 'elastictastic/persistence'
+require 'elastictastic/properties'
+require 'elastictastic/scope'
+require 'elastictastic/scope_builder'
+require 'elastictastic/scoped'
+require 'elastictastic/search'
+require 'elastictastic/server_error'
+require 'elastictastic/util'
 
 module Elastictastic
   autoload :Association, 'elastictastic/association'
   autoload :BulkPersistenceStrategy, 'elastictastic/bulk_persistence_strategy'
   autoload :Callbacks, 'elastictastic/callbacks'
   autoload :ChildCollectionProxy, 'elastictastic/child_collection_proxy'
-  autoload :Client, 'elastictastic/client'
-  autoload :Configuration, 'elastictastic/configuration'
   autoload :Dirty, 'elastictastic/dirty'
-  autoload :DiscretePersistenceStrategy, 'elastictastic/discrete_persistence_strategy'
   autoload :Document, 'elastictastic/document'
-  autoload :Field, 'elastictastic/field'
-  autoload :Index, 'elastictastic/index'
   autoload :MassAssignmentSecurity, 'elastictastic/mass_assignment_security'
-  autoload :Middleware, 'elastictastic/middleware'
   autoload :NestedCollectionProxy, 'elastictastic/nested_collection_proxy'
-  autoload :NestedDocument, 'elastictastic/nested_document'
   autoload :Observer, 'elastictastic/observer'
   autoload :Observing, 'elastictastic/observing'
   autoload :ParentChild, 'elastictastic/parent_child'
-  autoload :Persistence, 'elastictastic/persistence'
-  autoload :Properties, 'elastictastic/properties'
-  autoload :Resource, 'elastictastic/resource'
-  autoload :Scope, 'elastictastic/scope'
-  autoload :ScopeBuilder, 'elastictastic/scope_builder'
-  autoload :Scoped, 'elastictastic/scoped'
-  autoload :Search, 'elastictastic/search'
-  autoload :ServerError, 'elastictastic/server_error'
+  autoload :Rotor, 'elastictastic/rotor'
   autoload :TestHelpers, 'elastictastic/test_helpers'
-  autoload :Util, 'elastictastic/util'
+  autoload :ThriftAdapter, 'elastictastic/thrift_adapter'
   autoload :Validations, 'elastictastic/validations'
 
+  autoload :NestedDocument, 'elastictastic/nested_document' # Deprecated
+
   class <<self
     attr_writer :config
 
+    # 
+    # Elastictastic global configuration. In a Rails environment, you can
+    # configure Elastictastic by creating a `config/elastictastic.yml` file,
+    # whose keys will be passed into the Configuration object when your
+    # application boots. In non-Rails environment, you can configure
+    # Elastictastic directly using the object returned by this method.
+    #
+    # @return [Configuration] global configuration object
+    #
     def config
       @config ||= Configuration.new
     end
 
+    def multi_get(&block)
+      MultiGet.new.tap(&block).to_a
+    end
+
+    #
+    # Perform multiple searches in a single request to ElasticSearch. Each
+    # scope will be eagerly populated with results.
+    #
+    # @param [Scope, Array] collection of scopes to execute multisearch on
+    #
+    def multi_search(*scopes)
+    end
+
+    #
+    # Return a lower-level ElasticSearch client. This is likely to be extracted
+    # into a separate gem in the future.
+    #
+    # @return [Client] client
+    # @api private
+    #
     def client
       Thread.current['Elastictastic::client'] ||= Client.new(config)
     end
 
+    # 
+    # Set the current persistence strategy
+    #
+    # @param [DiscretePersistenceStrategy,BulkPersistenceStrategy] persistence strategy
+    # @api private
+    # @see ::persister
+    #
     def persister=(persister)
       Thread.current['Elastictastic::persister'] = persister
     end
 
+    # 
+    # The current persistence strategy for ElasticSearch. Usually this will
+    # be the DiscretePersistenceStrategy singleton; inside a ::bulk block, it
+    # will be an instance of BulkPersistenceStrategy
+    #
+    # @return [DiscretePersistenceStrategy,BulkPersistenceStrategy] current persistence strategy
+    # @api private
+    # @see ::bulk
+    #
     def persister
       Thread.current['Elastictastic::persister'] ||=
         Elastictastic::DiscretePersistenceStrategy.instance
     end
 
-    def bulk
+    # 
+    # Perform write operations in a single request to ElasticSearch. Highly
+    # recommended for any operation which writes a large quantity of data to
+    # ElasticSearch. Write operations (e.g. save/destroy documents) are buffered
+    # in the client and sent to ElasticSearch when the bulk operation exits
+    # (or when an auto-flush threshold is reached; see below).
+    #
+    # @example Create posts in bulk
+    #   Elastictastic.bulk do
+    #     params[:posts].each do |post_params|
+    #       Post.new(post_params).save!
+    #     end
+    #   end # posts are actually persisted here
+    #
+    # Since write operations inside a bulk block are not performed
+    # synchronously, server-side errors will only be raised once the bulk block
+    # completes; you may pass a block into Document#save and Document#destroy
+    # that will be called once the operation completes. The block is passed an
+    # error param if the operation was not successful.
+    #
+    # @example Custom handling for conflicting IDs in a bulk block
+    #   errors = []
+    #   Elastictastic.bulk do
+    #     params[:posts].each do |post_params|
+    #       Post.new(post_params).save! do |e|
+    #         case e
+    #         when nil # success!
+    #         when Elastictastic::ServerError::DocumentAlreadyExistsEngineException
+    #           conflicting_ids << post_params[:id]
+    #         else
+    #           raise e
+    #         end
+    #       end
+    #     end
+    #   end
+    #
+    # @option options [Fixnum] :auto_flush Flush to ElasticSearch after this many operations performed.
+    # @yield Block during which all write operations are buffered for bulk
+    #
+    def bulk(options = {})
       original_persister = self.persister
+      bulk_persister = self.persister =
+        Elastictastic::BulkPersistenceStrategy.new(options)
       begin
-        self.persister = Elastictastic::BulkPersistenceStrategy.new
         yield
-        self.persister.flush
-      rescue Elastictastic::CancelBulkOperation
-        # Nothing to see here...
       ensure
         self.persister = original_persister
       end
+      bulk_persister.flush
     end
 
-    def Index(name_or_index)
-      Index === name_or_index ?  name_or_index : Index.new(name_or_index)
+    #
+    # Use Elastictastic's configured JSON encoder to encode a JSON message.
+    #
+    # @param [Object] Object to encode to JSON
+    # @return JSON representation of object
+    # @api private
+    #
+    def json_encode(object)
+      if config.json_engine.respond_to?(:encode)
+        config.json_engine.encode(object)
+      else
+        config.json_engine.dump(object)
+      end
     end
 
-    private
+    #
+    # Use Elastictastic's configured JSON decoder to decode a JSON message
+    #
+    # @param [String] JSON message to decode
+    # @return Ruby object represented by json param
+    # @api private
+    #
+    def json_decode(json)
+      if config.json_engine.respond_to?(:decode)
+        config.json_engine.decode(json)
+      else
+        config.json_engine.load(json)
+      end
+    end
 
-    def new_transport
-      transport_class = const_get("#{config.transport.camelize}Transport")
-      transport_class.new(config)
+    #
+    # Coerce the argument to an Elastictastic index.
+    # 
+    # @param [String,Elastictastic::Index] name_or_index Index name or object
+    # @api private
+    #
+    def Index(name_or_index)
+      Index === name_or_index ?  name_or_index : Index.new(name_or_index)
     end
   end
 end

data/spec/environment.rb

@@ -1,6 +1,6 @@
 require 'bundler'
 Bundler.require(:default, :test, :development)
 
-%w(author blog comment post post_observer).each do |model|
+%w(author blog comment photo post post_observer).each do |model|
   require File.expand_path("../models/#{model}", __FILE__)
 end

data/spec/examples/bulk_persistence_strategy_spec.rb

@@ -6,7 +6,7 @@ describe Elastictastic::BulkPersistenceStrategy do
   let(:last_request) { FakeWeb.last_request }
   let(:bulk_requests) do
     last_request.body.split("\n").map do |request|
-      JSON.parse(request)
+      Elastictastic.json_decode(request)
     end
   end
 
@@ -14,7 +14,7 @@ describe Elastictastic::BulkPersistenceStrategy do
     let(:post) { Post.new }
 
     before do
-      stub_elasticsearch_bulk(
+      stub_es_bulk(
         'create' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 1, 'ok' => true }
       )
       Elastictastic.bulk do
@@ -38,6 +38,10 @@ describe Elastictastic::BulkPersistenceStrategy do
       post.id.should == '123'
     end
 
+    it 'should set version' do
+      post.version.should == 1
+    end
+
     it 'should set persisted' do
       post.should be_persisted
     end
@@ -55,7 +59,7 @@ describe Elastictastic::BulkPersistenceStrategy do
         example.run
         # have to do this here because the before/after hooks run inside the
         # around hook
-        stub_elasticsearch_bulk(
+        stub_es_bulk(
           'create' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 1, 'ok' => true }
         )
       end
@@ -80,7 +84,7 @@ describe Elastictastic::BulkPersistenceStrategy do
     let(:posts) { Array.new(2) { Post.new }}
 
     before do
-      stub_elasticsearch_bulk(
+      stub_es_bulk(
         { 'create' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 1, 'ok' => true }},
         { 'create' => { '_index' => 'default', '_type' => 'post', '_id' => '124', '_version' => 1, 'ok' => true }}
       )
@@ -99,6 +103,10 @@ describe Elastictastic::BulkPersistenceStrategy do
     it 'should set IDs' do
       posts.map { |post| post.id }.should == %w(123 124)
     end
+
+    it 'should set versions' do
+      posts.each { |post| post.version.should == 1 }
+    end
   end
 
   describe 'create with ID set' do
@@ -110,7 +118,7 @@ describe Elastictastic::BulkPersistenceStrategy do
     end
 
     before do
-      stub_elasticsearch_bulk(
+      stub_es_bulk(
         'create' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 1, 'ok' => true }
       )
       Elastictastic.bulk { post.save }
@@ -130,30 +138,38 @@ describe Elastictastic::BulkPersistenceStrategy do
     it 'should retain ID' do
       post.id.should == '123'
     end
+
+    it 'should set version' do
+      post.version.should == 1
+    end
   end
 
   describe '#update' do
     let(:post) do
       Post.new.tap do |post|
         post.id = '123'
+        post.version = 1
         post.persisted!
       end
     end
 
     before do
-      stub_elasticsearch_bulk(
-        'index' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 1, 'ok' => true }
+      stub_es_bulk(
+        'index' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 2, 'ok' => true }
       )
+      Elastictastic.bulk { post.save }
     end
 
     it 'should send update' do
-      Elastictastic.bulk { post.save }
-
       bulk_requests.should == [
-        { 'index' => { '_index' => 'default', '_type' => 'post', '_id' => '123' }},
+        { 'index' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 1 }},
         post.elasticsearch_doc
       ]
     end
+
+    it 'should set version' do
+      post.version.should == 2
+    end
   end
 
   describe 'destroy' do
@@ -161,20 +177,21 @@ describe Elastictastic::BulkPersistenceStrategy do
       Post.new.tap do |post|
         post.id = '123'
         post.title = 'bulky'
+        post.version = 1
         post.persisted!
       end
     end
 
     before do
-      stub_elasticsearch_bulk(
-        'destroy' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 1, 'ok' => true }
+      stub_es_bulk(
+        'delete' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 2, 'ok' => true }
       )
       Elastictastic.bulk { post.destroy }
     end
 
     it 'should send destroy' do
       bulk_requests.should == [
-        { 'delete' => { '_index' => 'default', '_type' => 'post', '_id' => '123' }}
+        { 'delete' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 1 }}
       ]
     end
 
@@ -191,7 +208,7 @@ describe Elastictastic::BulkPersistenceStrategy do
 
     it 'should return to individual persistence strategy' do
       error_proc.call rescue nil
-      stub_elasticsearch_create('default', 'post')
+      stub_es_create('default', 'post')
       Post.new.save
       last_request.path.should == '/default/post'
     end
@@ -214,20 +231,131 @@ describe Elastictastic::BulkPersistenceStrategy do
     it_should_behave_like 'block with error'
   end
 
-  describe 'raising CancelBulkOperation' do
-    let :error_proc do
-      lambda do
-        Elastictastic.bulk do
-          Post.new.save
-          raise Elastictastic::CancelBulkOperation
+  describe 'with :auto_flush specified' do
+    before do
+      responses = Array.new(3) do
+        { 'create' => generate_es_hit('post').except('_source').merge('ok' => true) }
+      end.each_slice(2).map { |slice| { 'items' => slice } }
+      stub_request_json(
+        :post,
+        match_es_path('/_bulk'),
+        *responses
+      )
+      Elastictastic.bulk(:auto_flush => 2) { 3.times { Post.new.save }}
+    end
+
+    it 'should perform multiple requests when auto-flush triggered' do
+      FakeWeb.should have(2).requests
+    end
+
+    it 'should flush after specified number of operations' do
+      FakeWeb.requests.first.body.split("\n").should have(4).items
+      FakeWeb.requests.last.body.split("\n").should have(2).items
+    end
+  end
+
+  describe 'multiple operations on the same document' do
+    let(:post) do
+      Post.new.tap do |post|
+        post.id = '1'
+        post.version = 1
+        post.persisted!
+      end
+    end
+
+    before do
+      stub_es_bulk(
+        'delete' => generate_es_hit('post', :version => 2, :id => '1').merge('ok' => true)
+      )
+      Elastictastic.bulk do
+        post.save
+        post.destroy
+      end
+    end
+
+    it 'should only send one operation per document' do
+      bulk_requests.length.should == 1
+    end
+
+    it 'should send last operation for each document' do
+      bulk_requests.last.should ==  { 'delete' => generate_es_hit('post', :id => '1').except('_source') }
+    end
+  end
+
+  describe 'multiple creates' do
+    before do
+      stub_es_bulk(
+        { 'create' => generate_es_hit('post', :id => '1').merge('ok' => true) },
+        { 'create' => generate_es_hit('post', :id => '2').merge('ok' => true) }
+      )
+      Elastictastic.bulk { 2.times { |i| Post.new(:title => "post #{i}").save }}
+    end
+
+    it 'should create all documents' do
+      bulk_requests.length.should == 4
+    end
+
+    it 'should send correct info for each document' do
+      bulk_requests.each_slice(2).map do |commands|
+        commands.last['title']
+      end.should == ['post 0', 'post 1']
+    end
+  end
+
+  describe 'updating documents with same ID but different index' do
+    let(:posts) do
+      %w(default my_index).map do |index|
+        Post.in_index(index).new.tap do |post|
+          post.id = '1'
+          post.persisted!
         end
       end
     end
 
-    it 'should not propagate error' do
-      expect(&error_proc).not_to raise_error
+    before do
+      stub_es_bulk(
+        { 'index' => generate_es_hit('post', :id => '1').merge('ok' => true) },
+        { 'index' => generate_es_hit('post', :index => 'my_index', :id => '1').merge('ok' => true) }
+      )
+      Elastictastic.bulk { posts.map { |post| post.save }}
     end
 
-    it_should_behave_like 'block with error'
+    it 'should send updates for both documents' do
+      bulk_requests.length.should == 4
+    end
+  end
+
+  describe 'with routing' do
+    let(:photo) { Photo.new(:post_id => 1) }
+
+    it 'should include routing on create' do
+      stub_es_bulk(
+        'create' => { '_index' => 'default', '_type' => 'photo', '_id' => '123', '_version' => 1, 'ok' => true }
+      )
+      Elastictastic.bulk { photo.save }
+      bulk_requests.first['create']['_routing'].should == '1'
+    end
+
+    it 'should include routing on update' do
+      photo.id = '123'
+      photo.version = 1
+      photo.persisted!
+      stub_es_bulk(
+        'index' => { '_index' => 'default', '_type' => 'photo', '_id' => '123', '_version' => 2, 'ok' => true }
+      )
+      Elastictastic.bulk { photo.save }
+      bulk_requests.first['index']['_routing'].should == '1'
+    end
+
+    it 'should include routing on destroy' do
+      photo.id = '123'
+      photo.version = 1
+      photo.persisted!
+      stub_es_bulk(
+        'delete' => { '_index' => 'default', '_type' => 'post', '_id' => '123', '_version' => 2, 'ok' => true }
+      )
+      Elastictastic.bulk { photo.destroy }
+      bulk_requests.first['delete']['_routing'].should == '1'
+    end
   end
 end

data/spec/examples/callbacks_spec.rb

@@ -15,6 +15,10 @@ describe Elastictastic::Callbacks do
       before_create :before_create_ran!
       before_update :before_update_ran!
       before_destroy :before_destroy_ran!
+      after_save :after_save_ran!
+      after_create :after_create_ran!
+      after_update :after_update_ran!
+      after_destroy :after_destroy_ran!
       
       def hooks_that_ran
         @hooks_that_ran ||= Set[]
@@ -45,52 +49,79 @@ describe Elastictastic::Callbacks do
   end
 
   before do
-    stub_elasticsearch_create('default', 'my_model')
-    stub_elasticsearch_update('default', 'my_model', id)
-    stub_elasticsearch_destroy('default', 'my_model', id)
+    stub_es_create('default', 'my_model')
+    stub_es_update('default', 'my_model', id)
+    stub_es_destroy('default', 'my_model', id)
   end
 
-  describe '#before_save' do
+  %w(before after).each do |position|
+    describe "##{position}_save" do
 
-    it 'should run before create' do
-      instance.save
-      instance.should have_run_hook(:before_save)
-    end
+      it "should run #{position} create" do
+        instance.save
+        instance.should have_run_hook(:"#{position}_save")
+      end
 
-    it 'should run before update' do
-      persisted_instance.save
-      persisted_instance.should have_run_hook(:before_save)
-    end
-  end
+      it 'should run before update' do
+        persisted_instance.save
+        persisted_instance.should have_run_hook(:"#{position}_save")
+      end
 
-  describe '#before_create' do
-    it 'should run before create' do
-      instance.save
-      instance.should have_run_hook(:before_create)
-    end
+      it 'should not run before create when callbacks disabled' do
+        instance.save(:callbacks => false)
+        instance.should_not have_run_hook(:"#{position}_save")
+      end
 
-    it 'should run before update' do
-      persisted_instance.save
-      persisted_instance.should_not have_run_hook(:before_create)
+      it 'should not run before update when hooks disabled' do
+        persisted_instance.save(:callbacks => false)
+        instance.should_not have_run_hook(:"#{position}_save")
+      end
     end
-  end
 
-  describe '#before_update' do
-    it 'should not run before create' do
-      instance.save
-      instance.should_not have_run_hook :before_update
+    describe "##{position}_create" do
+      it "should run #{position} create" do
+        instance.save
+        instance.should have_run_hook(:"#{position}_create")
+      end
+
+      it "should not run #{position} update" do
+        persisted_instance.save
+        persisted_instance.should_not have_run_hook(:"#{position}_create")
+      end
+
+      it "should not run #{position} create when callbacks disabled" do
+        instance.save(:callbacks => false)
+        instance.should_not have_run_hook(:"#{position}_create")
+      end
     end
 
-    it 'should run before update' do
-      persisted_instance.save
-      persisted_instance.should have_run_hook(:before_update)
+    describe "##{position}_update" do
+      it "should not run #{position} create" do
+        instance.save
+        instance.should_not have_run_hook :"#{position}_update"
+      end
+
+      it "should run #{position} update" do
+        persisted_instance.save
+        persisted_instance.should have_run_hook(:"#{position}_update")
+      end
+
+      it "should not run #{position} update if callbacks disabled" do
+        persisted_instance.save(:callbacks => false)
+        persisted_instance.should_not have_run_hook(:"#{position}_update")
+      end
     end
-  end
 
-  describe '#before_destroy' do
-    it 'should run before destroy' do
-      persisted_instance.destroy
-      persisted_instance.should have_run_hook(:before_destroy)
+    describe "##{position}_destroy" do
+      it "should run #{position} destroy" do
+        persisted_instance.destroy
+        persisted_instance.should have_run_hook(:"#{position}_destroy")
+      end
+
+      it "should not run #{position} destroy if callbacks disabled" do
+        persisted_instance.destroy(:callbacks => false)
+        persisted_instance.should_not have_run_hook(:"#{position}_destroy")
+      end
     end
   end
 end