es-elasticity 0.11.1 → 0.11.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 11dddda9376573d12d8d6c0db18342c7a242199a
-  data.tar.gz: 2613b868ae0ca372149fa84724790ade9bbca2a2
+  metadata.gz: b90fa843196b6ec0a4ed99f4605d089f1127696a
+  data.tar.gz: 16e810b1114f1829e21c8e655f2fd7ad40fa1022
 SHA512:
-  metadata.gz: 13a78c3cbe8eb55a911d7b39d1a31e07329e4e3f809fde432ece6a12511db5e40422af47f3508938ec597579aef697a05622dc449670537b45b4ba64ee4b22e3
-  data.tar.gz: bcb01eb45bf67cf5dc2d6c0cfd7bb6ad00fc40167e5068da1e067d6a77f8f7d811b812b7de923fca134eea8ea28510114e9e1e235c9ff61b148a391e30a5f035
+  metadata.gz: 9d54bd8d914de323f3ce7d430e7477eda5b74d4b1dbd3bb9127a2a5b460ec7facbbef5fcbc5e00fe2f9f3df674834cca6641724191acb5c2b81922874c6d32a6
+  data.tar.gz: cf91125f554221f3ea89364fc8aab76395cfa9f0376fd03369bcfb02bf6235cd7b35b668ea952d7669123883c3c93c83e20cdb8024087b887fe9d95f481865e7

data/CHANGELOG

@@ -1,3 +1,14 @@
+v0.11.5
+ - Give the option of retrying the deletion for certain exceptions during remap
+v0.11.4
+ - Fully clean up if error occurs during remap (assign all aliases back to original index)
+v0.11.3
+ - Adds support for preserving the order or normalized names of `highlight` through `highlighted_attrs`
+
+v0.11.2
+ - Adds support for passing arguments to Search definition through `search(query, search_args)` in index searching and msearch
+ - adds _explanation to hold value of returned explanations in the base document
+
 v0.11.1
   - support `action.destructive_requires_name` setting by being explict about which indices to delete
 

data/README.md

@@ -15,7 +15,7 @@ Elasticity maps those documents into objects, providing a rich object representa
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'es-elasticity', require "elasticity"
+gem 'es-elasticity', require: "elasticity"
 ```
 
 And then execute:
@@ -191,6 +191,26 @@ cursor.each { |doc| ... }
 adults = adults.active_records(User)
 ```
 
+#### Search Args
+
+##### explain: true
+For `search` definitions we support passing  `{ explain: true }` to the search as a second argument in order to surface the reason a search result was returned. 
+
+```ruby
+# example in single search
+search_results_with_explanation = SearchDoc::A.search(query_body, { explain: true }).search_results
+
+# In multisearch
+search_a = SearchDoc::A.search(query_body, { explain: true })
+search_b = SearchDoc::B.search(query_body, { explain: true })
+search_c = SearchDoc::C.search(query_body, { explain: true })
+
+multi = Elasticity::MultiSearch.new do |m|
+  m.add(:a, search_a, documents: ::SearchDoc::A)
+  m.add(:b, search_b, documents: ::SearchDoc::B)
+  m.add(:c, search_c, documents: ::SearchDoc::C)
+end
+```
 For more information about the `active_records` method, read [ActiveRecord integration](#activerecord-integration).
 
 ### Segmented Documents

data/lib/elasticity/base_document.rb

@@ -12,7 +12,7 @@ module Elasticity
     end
 
     # Define common attributes for all documents
-    attr_accessor :_id, :highlighted, :_score, :sort
+    attr_accessor :_id, :highlighted, :_score, :sort, :_explanation, :highlighted_attrs
 
     def attributes=(attributes)
       attributes.each do |attr, value|

data/lib/elasticity/index_mapper.rb

@@ -65,8 +65,9 @@ module Elasticity
     end
 
     # Remap
-    def remap!
-      @strategy.remap(@index_config.definition)
+    # retry_delay & max_delay are in seconds
+    def remap!(retry_delete_on_recoverable_errors: true, retry_delay: 30, max_delay: 600)
+      @strategy.remap(@index_config.definition, retry_delete_on_recoverable_errors: retry_delete_on_recoverable_errors, retry_delay: retry_delay, max_delay: max_delay)
     end
 
     # Flushes the index, forcing any writes
@@ -82,8 +83,10 @@ module Elasticity
     # Searches the index using the parameters provided in the body hash, following the same
     # structure Elasticsearch expects.
     # Returns a DocumentSearch object.
-    def search(body)
-      search_obj = Search.build(@index_config.client, @strategy.search_index, document_types, body)
+    # search_args allows for
+    #   explain: boolean to specify we should request _explanation of the query
+    def search(body, search_args = {})
+      search_obj = Search.build(@index_config.client, @strategy.search_index, document_types, body, search_args)
       Search::DocumentProxy.new(search_obj, self.method(:map_hit))
     end
 
@@ -141,23 +144,28 @@ module Elasticity
       attrs.merge!(sort: hit["sort"])
       attrs.merge!(hit["_source"]) if hit["_source"]
 
-      if hit["highlight"]
-        highlighted_attrs = attrs.dup
-        attrs_set = Set.new
+      highlighted = nil
 
-        hit["highlight"].each do |name, v|
+      if hit["highlight"]
+        highlighted_attrs = hit["highlight"].each_with_object({}) do |(name, v), attrs|
           name = name.gsub(/\..*\z/, '')
-          next if attrs_set.include?(name)
-          highlighted_attrs[name] = v
-          attrs_set << name
+
+          attrs[name] ||= v
         end
 
-        highlighted = @document_klass.new(highlighted_attrs)
+        highlighted = @document_klass.new(attrs.merge(highlighted_attrs))
       end
+
+      injected_attrs = attrs.merge({
+        highlighted: highlighted,
+        highlighted_attrs: highlighted_attrs.try(:keys),
+        _explanation: hit["_explanation"]
+      })
+
       if @document_klass.config.subclasses.present?
-        @document_klass.config.subclasses[hit["_type"].to_sym].constantize.new(attrs.merge(highlighted: highlighted))
+        @document_klass.config.subclasses[hit["_type"].to_sym].constantize.new(injected_attrs)
       else
-        @document_klass.new(attrs.merge(highlighted: highlighted))
+        @document_klass.new(injected_attrs)
       end
     end
   end

data/lib/elasticity/search.rb

@@ -1,7 +1,7 @@
 module Elasticity
   module Search
-    def self.build(client, index_name, document_types, body)
-      search_def = Search::Definition.new(index_name, document_types, body)
+    def self.build(client, index_name, document_types, body, search_args = {})
+      search_def = Search::Definition.new(index_name, document_types, body, search_args)
       Search::Facade.new(client, search_def)
     end
 
@@ -10,10 +10,11 @@ module Elasticity
     class Definition
       attr_accessor :index_name, :document_types, :body
 
-      def initialize(index_name, document_types, body)
+      def initialize(index_name, document_types, body, search_args = {})
         @index_name     = index_name
         @document_types = document_types
         @body           = body.deep_symbolize_keys!
+        @search_args    = search_args
       end
 
       def update(body_changes)
@@ -28,11 +29,13 @@ module Elasticity
       end
 
       def to_search_args
-        { index: @index_name, type: @document_types, body: @body }
+        @search_args.merge({ index: @index_name, type: @document_types, body: @body })
       end
 
       def to_msearch_args
-        { index: @index_name, type: @document_types, search: @body }
+        search_body = @search_args.merge(@body)
+
+        { index: @index_name, type: @document_types, search: search_body }
       end
     end
 

data/lib/elasticity/strategies/alias_index.rb

@@ -4,6 +4,11 @@ module Elasticity
     # runtime changes by simply atomically updating the aliases. For example, look at the remap method
     # implementation.
     class AliasIndex
+      SNAPSHOT_ERROR_SNIPPET =  "Cannot delete indices that are being snapshotted".freeze
+      RETRYABLE_ERROR_SNIPPETS = [
+        SNAPSHOT_ERROR_SNIPPET
+      ].freeze
+
       STATUSES = [:missing, :ok]
 
       def initialize(client, index_base_name, document_type)
@@ -29,7 +34,7 @@ module Elasticity
       #
       # It does a little bit more to ensure consistency and to handle race-conditions. For more details
       # look at the implementation.
-      def remap(index_def)
+      def remap(index_def, retry_delete_on_recoverable_errors: false, retry_delay: 0, max_delay: 0)
         main_indexes   = self.main_indexes
         update_indexes = self.update_indexes
 
@@ -99,11 +104,23 @@ module Elasticity
               { remove: { index: original_index, alias: @main_alias } },
             ]
           })
-          @client.index_delete(index: original_index)
 
+          waiting_duration = 0
+          begin
+            @client.index_delete(index: original_index)
+          rescue Elasticsearch::Transport::Transport::ServerError => e
+            if retryable_error?(e)  && retry_delete_on_recoverable_errors && waiting_duration < max_delay
+              waiting_duration += retry_delay
+              sleep(retry_delay)
+              retry
+            else
+              raise e
+            end
+          end
         rescue
           @client.index_update_aliases(body: {
             actions: [
+              { add:    { index: original_index, alias: @main_alias } },
               { add:    { index: original_index, alias: @update_alias } },
               { remove: { index: new_index, alias: @update_alias } },
             ]
@@ -268,6 +285,12 @@ module Elasticity
         @client.index_create(index: index_name, body: index_def)
         index_name
       end
+
+      def retryable_error?(e)
+        RETRYABLE_ERROR_SNIPPETS.any? do |s|
+          e.message.match(s)
+        end
+      end
     end
   end
 end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = "0.11.1"
+  VERSION = "0.11.5"
 end

data/spec/functional/persistence_spec.rb

@@ -85,8 +85,9 @@ RSpec.describe "Persistence", elasticsearch: true do
         c.index_base_name = "cats_and_dogs"
         c.strategy = Elasticity::Strategies::SingleIndex
         c.document_type  = "cat"
+
         c.mapping = { "properties" => {
-          name: { type: "string", index: "not_analyzed" },
+          name: { type: "text", index: true },
           age: { type: "integer" }
         } }
       end
@@ -104,7 +105,7 @@ RSpec.describe "Persistence", elasticsearch: true do
         c.strategy = Elasticity::Strategies::SingleIndex
         c.document_type = "dog"
         c.mapping = { "properties" => {
-          name: { type: "string", index: "not_analyzed" },
+          name: { type: "text", index: true },
           age: { type: "integer" },
           hungry: { type: "boolean" }
         } }
@@ -167,7 +168,7 @@ RSpec.describe "Persistence", elasticsearch: true do
           c.mapping = {
             "properties" => {
               id: { type: "integer" },
-              name: { type: "string", index: "not_analyzed" },
+              name: { type: "text", index: true },
               birthdate: { type: "date" },
             },
           }
@@ -268,7 +269,7 @@ RSpec.describe "Persistence", elasticsearch: true do
           c.mapping = {
             "properties" => {
               id: { type: "integer" },
-              name: { type: "string", index: "not_analyzed" },
+              name: { type: "text", index: true },
             },
           }
         end
@@ -313,6 +314,88 @@ RSpec.describe "Persistence", elasticsearch: true do
       expect(results.total).to eq(2010)
     end
 
+    it "fully cleans up if error occurs deleting the old index during remap" do
+      expected_aliases = %w[elasticity_test_users elasticity_test_users_update]
+      original_aliases = all_aliases(subject)
+      expect(original_aliases).to match_array(expected_aliases)
+      number_of_docs = 20
+      number_of_docs.times.map do |i|
+        subject.new(id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
+      end
+
+      allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).and_raise("KAPOW")
+      expect do
+        subject.remap!
+      end.to raise_error("KAPOW")
+      cleaned_up_aliases = all_aliases(subject)
+      expect(cleaned_up_aliases).to match_array(expected_aliases)
+      allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).and_call_original
+    end
+
+    context "recovering from remap errors" do
+      let(:recoverable_message) do
+        '[400] {"error":{"root_cause":[{"type":"remote_transport_exception","reason":"[your_cluster][cluster_id][indices:admin/delete]"}],"type":"illegal_argument_exception","reason":"Cannot delete indices that are being snapshotted: [[full_index_name_and_id]]. Try again after snapshot finishes or cancel the currently running snapshot."},"status":400}'
+      end
+      after do
+        allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).and_call_original
+      end
+
+      it "waits for recoverable errors before deleting old index during remap" do
+        original_index_name = subject.config.client.index_get_alias(index: "#{subject.ref_index_name}-*", name: subject.ref_index_name).keys.first
+
+        call_count = 0
+        allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete) do
+          call_count +=1
+          if call_count < 3
+            raise Elasticsearch::Transport::Transport::Errors::BadRequest.new(recoverable_message)
+          else
+            []
+          end
+        end
+        build_some_docs(subject)
+
+        subject.remap!(retry_delete_on_recoverable_errors: true, retry_delay: 0.5, max_delay: 1)
+        subject.flush_index
+        results = subject.search({})
+        expect(results.total).to eq(20)
+        remapped_index_name = subject.config.client.index_get_alias(index: "#{subject.ref_index_name}-*", name: subject.ref_index_name).keys.first
+        expect(remapped_index_name).to_not eq(original_index_name)
+      end
+
+      it "will only retry for a set amount of time" do
+        allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).and_raise(
+          Elasticsearch::Transport::Transport::Errors::BadRequest.new(recoverable_message)
+        )
+        build_some_docs(subject)
+        expect {
+          subject.remap!(retry_delete_on_recoverable_errors: true, retry_delay: 0.5, max_delay: 1)
+        }.to raise_error(Elasticsearch::Transport::Transport::ServerError)
+      end
+
+      it "will not only retry if the arguments say not to" do
+        original_index_name = subject.config.client.index_get_alias(index: "#{subject.ref_index_name}-*", name: subject.ref_index_name).keys.first
+        allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).with(any_args).and_call_original
+        allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).with(index: original_index_name).and_raise(
+          Elasticsearch::Transport::Transport::Errors::BadRequest.new(recoverable_message)
+        )
+        build_some_docs(subject)
+        expect {
+          subject.remap!(retry_delete_on_recoverable_errors: false)
+        }.to raise_error(Elasticsearch::Transport::Transport::ServerError)
+      end
+
+      it "will not retry for 'non-recoverable' errors" do
+        exception_message = '[404] {"error":"alias [some_index_name] missing","status":404}'
+        allow_any_instance_of(Elasticity::InstrumentedClient).to receive(:index_delete).and_raise(
+          Elasticsearch::Transport::Transport::Errors::BadRequest.new(exception_message)
+        )
+        build_some_docs(subject)
+        expect {
+          subject.remap!(retry_delete_on_recoverable_errors: true, retry_delay: 0.5, max_delay: 1)
+        }.to raise_error(Elasticsearch::Transport::Transport::ServerError)
+      end
+    end
+
     it "bulk indexes, updates and delete" do
       docs = 2000.times.map do |i|
         subject.new(_id: i, id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
@@ -342,4 +425,15 @@ RSpec.describe "Persistence", elasticsearch: true do
       expect(results.total).to eq 0
     end
   end
+
+  def all_aliases(subj)
+    base_name = subj.ref_index_name
+    subj.config.client.index_get_alias(index: "#{base_name}-*", name: "#{base_name}*").values.first["aliases"].keys
+  end
+
+  def build_some_docs(subj, doc_count = 20)
+    doc_count.times.map do |i|
+      subj.new(id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
+    end
+  end
 end

data/spec/functional/search_spec.rb

@@ -0,0 +1,117 @@
+RSpec.describe "Search", elasticsearch: true do
+  class CatDoc < Elasticity::Document
+    configure do |c|
+      c.strategy = Elasticity::Strategies::SingleIndex
+      c.document_type  = "cat"
+      c.mapping = { "properties" => {
+        name: { type: "text" },
+        description: { type: "text" },
+        age: { type: "integer" }
+      } }
+    end
+
+    attr_accessor :name, :age, :description
+
+    def to_document
+      { name: name, age: age, description: description }
+    end
+  end
+
+  class DogDoc < Elasticity::Document
+    configure do |c|
+      c.strategy = Elasticity::Strategies::SingleIndex
+      c.document_type = "dog"
+      c.mapping = { "properties" => {
+        name: { type: "keyword" },
+        description: { type: "text" },
+        age: { type: "integer" },
+        hungry: { type: "boolean" }
+      } }
+    end
+    attr_accessor :name, :age, :description, :hungry
+
+    def to_document
+      { name: name, age: age, description: description, hungry: hungry }
+    end
+  end
+
+  describe "search_args" do
+    before do
+      CatDoc.recreate_index
+      DogDoc.recreate_index
+
+      @elastic_search_client.cluster.health wait_for_status: 'yellow'
+
+      cat = CatDoc.new(name: "felix the cat", age: 10, description: "I am an old cat")
+      dog = DogDoc.new(name: "fido", age: 4, hungry: true, description: "I am a hungry dog")
+
+      cat.update
+      dog.update
+
+      CatDoc.flush_index
+    end
+
+    describe "explain: true" do
+      def get_explanations(results)
+        results.map(&:_explanation)
+      end
+
+      it "supports on single index search results" do
+        results = CatDoc.search({}, { explain: true }).search_results
+
+        expect(get_explanations(results)).to all( be_truthy )
+      end
+
+      it "supports for multisearch" do
+        cat = CatDoc.search({}, { explain: true })
+        dog = DogDoc.search({})
+
+        subject = Elasticity::MultiSearch.new do |m|
+          m.add(:cats, cat, documents: CatDoc)
+          m.add(:dogs, dog, documents: DogDoc)
+        end
+
+        expect(get_explanations(subject[:cats])).to all( be_truthy )
+        expect(get_explanations(subject[:dogs])).to all( be_nil )
+      end
+    end
+
+    describe "highlight" do
+      it "is nil when the highlight does not return" do
+        results =  CatDoc.search({}).search_results
+
+        expect(results.first.highlighted_attrs).to be_nil
+        expect(results.first.highlighted).to be_nil
+      end
+
+      describe "when specifying highlight" do
+        let(:cat_search_result) {
+          highlight_search = {
+              query: {
+                  multi_match: {
+                      query: "cat",
+                      fields: ["name^1000", "description"]
+                  }
+              },
+              highlight: {
+                  fields: {
+                      "*": {}
+                  }
+              }
+          }
+
+          CatDoc.search(highlight_search).search_results.first
+        }
+
+        it "highlighted_attrs returns the highlighted" do
+          expect(cat_search_result.highlighted_attrs).to eq(["name", "description"])
+        end
+
+        it "highlighted returns a new object with the name transformed" do
+          expect(cat_search_result.highlighted.name.first).to include("felix")
+          expect(cat_search_result.highlighted.description.first).to include("old")
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: es-elasticity
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.11.5
 platform: ruby
 authors:
 - Rodrigo Kochenburger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-30 00:00:00.000000000 Z
+date: 2018-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -231,6 +231,7 @@ files:
 - lib/elasticity/strategies/single_index.rb
 - lib/elasticity/version.rb
 - spec/functional/persistence_spec.rb
+- spec/functional/search_spec.rb
 - spec/functional/segmented_spec.rb
 - spec/rspec_config.rb
 - spec/units/document_spec.rb
@@ -260,12 +261,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: ActiveModel-based library for working with Elasticsearch
 test_files:
 - spec/functional/persistence_spec.rb
+- spec/functional/search_spec.rb
 - spec/functional/segmented_spec.rb
 - spec/rspec_config.rb
 - spec/units/document_spec.rb