elastic_record 4.1.8 → 5.1.0

data/lib/elastic_record/connection.rb

@@ -5,7 +5,7 @@ module ElasticRecord
     attr_accessor :servers, :options
     attr_accessor :request_count, :current_server
     attr_accessor :max_request_count
-    attr_accessor :bulk_stack
+    attr_accessor :bulk_actions
     def initialize(servers, options = {})
       self.servers = Array(servers)
 
@@ -14,7 +14,7 @@ module ElasticRecord
       self.request_count      = 0
       self.max_request_count  = 100
       self.options            = options.symbolize_keys
-      self.bulk_stack         = []
+      self.bulk_actions       = nil
     end
 
     def head(path)
@@ -38,10 +38,10 @@ module ElasticRecord
     end
 
     def json_request(method, path, json)
-      body = json.is_a?(Hash) ? ElasticRecord::JSON.encode(json) : json
+      body = json.is_a?(Hash) ? JSON.generate(json) : json
       response = http_request_with_retry(method, path, body)
 
-      json = ElasticRecord::JSON.decode(response.body)
+      json = JSON.parse(response.body)
       raise ConnectionError.new(response.code, json['error']) if json['error']
 
       json

data/lib/elastic_record/errors.rb

@@ -12,4 +12,10 @@ module ElasticRecord
 
   class BulkError < Error
   end
-end
+
+  class ExpiredScrollError < Error
+  end
+
+  class InvalidScrollError < Error
+  end
+end

data/lib/elastic_record/index.rb

@@ -4,6 +4,7 @@ require 'elastic_record/index/documents'
 require 'elastic_record/index/manage'
 require 'elastic_record/index/mapping'
 require 'elastic_record/index/settings'
+require 'elastic_record/index/mapping_type'
 
 require 'active_support/core_ext/object/deep_dup'
 
@@ -31,18 +32,15 @@ module ElasticRecord
     include Mapping, Settings
     include Analyze
     include Deferred
-
-    attr_accessor :doctypes
+    include MappingType
 
     attr_accessor :disabled
     attr_accessor :model
     attr_accessor :partial_updates
     attr_accessor :load_from_source
 
-    def initialize(models)
-      models = Array.wrap(models)
-      @model = models.first
-      @doctypes = models.map(&:doctype)
+    def initialize(model)
+      @model = model
       @disabled = false
       @load_from_source = false
     end
@@ -78,9 +76,9 @@ module ElasticRecord
       model.elastic_connection
     end
 
-    def get(end_path, doctype = nil, json = nil)
+    def get(end_path, json = nil)
       path = "/#{alias_name}"
-      path += "/#{doctype.name}" if doctype
+      path += "/#{mapping_type}"
       path += "/#{end_path}"
 
       connection.json_get path, json

data/lib/elastic_record/index/analyze.rb

@@ -1,6 +1,16 @@
 module ElasticRecord
   class Index
     module Analyze
+      attr_accessor :analysis
+
+      def analysis
+        @analysis ||= {}
+      end
+
+      def analysis=(custom_analysis)
+        analysis.deep_merge!(custom_analysis)
+      end
+
       def analyze(params)
         json = connection.json_get "/#{alias_name}/_analyze", params
         json['tokens'].map { |token_hash| token_hash['token'] }

data/lib/elastic_record/index/deferred.rb

@@ -11,11 +11,11 @@ module ElasticRecord
         attr_accessor :index
         attr_accessor :deferred_actions
         attr_accessor :writes_made
-        attr_accessor :bulk_stack
+        attr_accessor :bulk_actions
 
         def initialize(index)
           @index = index
-          @bulk_stack = []
+          @bulk_actions = nil
           reset!
         end
 

data/lib/elastic_record/index/documents.rb

@@ -13,8 +13,8 @@ module ElasticRecord
       end
 
       def each_slice(&block)
-        while (hit_ids = request_more_ids).any?
-          hit_ids.each_slice(batch_size, &block)
+        while (hits = request_more_hits).any?
+          hits.each_slice(batch_size, &block)
         end
       end
 
@@ -45,6 +45,7 @@ module ElasticRecord
         @initial_search_response ||= begin
           search_options = {size: batch_size, scroll: keep_alive}
           elastic_query = @search.reverse_merge('sort' => '_doc')
+
           @elastic_index.search(elastic_query, search_options)
         end
       end
@@ -56,7 +57,6 @@ module ElasticRecord
           index_document(
             record.try(:id),
             record.as_search_document,
-            doctype: record.doctype,
             index_name: index_name
           )
         end
@@ -67,21 +67,20 @@ module ElasticRecord
           update_document(
             record.id,
             record.as_partial_update_document,
-            doctype: record.doctype,
             index_name: index_name
           )
         end
       end
 
-      def index_document(id, document, doctype: model.doctype, parent: nil, index_name: alias_name)
+      def index_document(id, document, parent: nil, index_name: alias_name)
         if batch = current_bulk_batch
-          instructions = { _index: index_name, _type: doctype.name, _id: id }
+          instructions = { _index: index_name, _type: mapping_type, _id: id }
           instructions[:parent] = parent if parent
 
           batch << { index: instructions }
           batch << document
         else
-          path = "/#{index_name}/#{doctype.name}/#{id}"
+          path = "/#{index_name}/#{mapping_type}/#{id}"
           path << "?parent=#{parent}" if parent
 
           if id
@@ -92,34 +91,34 @@ module ElasticRecord
         end
       end
 
-      def update_document(id, document, doctype: model.doctype, parent: nil, index_name: alias_name)
+      def update_document(id, document, parent: nil, index_name: alias_name)
         raise "Cannot update a document with empty id" if id.blank?
         params = {doc: document, doc_as_upsert: true}
 
         if batch = current_bulk_batch
-          instructions = { _index: index_name, _type: doctype.name, _id: id, _retry_on_conflict: 3 }
+          instructions = { _index: index_name, _type: mapping_type, _id: id, retry_on_conflict: 3 }
           instructions[:parent] = parent if parent
 
           batch << { update: instructions }
           batch << params
         else
-          path = "/#{index_name}/#{doctype.name}/#{id}/_update?retry_on_conflict=3"
+          path = "/#{index_name}/#{mapping_type}/#{id}/_update?retry_on_conflict=3"
           path << "&parent=#{parent}" if parent
 
           connection.json_post path, params
         end
       end
 
-      def delete_document(id, doctype: model.doctype, parent: nil, index_name: alias_name)
+      def delete_document(id, parent: nil, index_name: alias_name)
         raise "Cannot delete document with empty id" if id.blank?
         index_name ||= alias_name
 
         if batch = current_bulk_batch
-          instructions = { _index: index_name, _type: doctype.name, _id: id, _retry_on_conflict: 3 }
+          instructions = { _index: index_name, _type: mapping_type, _id: id, retry_on_conflict: 3 }
           instructions[:parent] = parent if parent
           batch << { delete: instructions }
         else
-          path = "/#{index_name}/#{doctype.name}/#{id}"
+          path = "/#{index_name}/#{mapping_type}/#{id}"
           path << "&parent=#{parent}" if parent
 
           connection.json_delete path
@@ -129,15 +128,15 @@ module ElasticRecord
       def delete_by_query(query)
         scroll_enumerator = build_scroll_enumerator search: query
 
-        scroll_enumerator.each_slice do |ids|
+        scroll_enumerator.each_slice do |hits|
           bulk do
-            ids.each { |id| delete_document(id) }
+            hits.each { |hit| delete_document hit['_id'] }
           end
         end
       end
 
       def record_exists?(id)
-        get(id, model.doctype)['found']
+        get(id)['found']
       end
 
       def search(elastic_query, options = {})
@@ -146,7 +145,7 @@ module ElasticRecord
           url += "?#{options.to_query}"
         end
 
-        get url, model.doctype, elastic_query
+        get url, elastic_query.update('_source' => load_from_source)
       end
 
       def explain(id, elastic_query)
@@ -160,20 +159,20 @@ module ElasticRecord
       def scroll(scroll_id, scroll_keep_alive)
         options = {scroll_id: scroll_id, scroll: scroll_keep_alive}
         connection.json_get("/_search/scroll?#{options.to_query}")
+      rescue ElasticRecord::ConnectionError => e
+        case e.status_code
+        when '400' then raise ElasticRecord::InvalidScrollError, e.message
+        when '404' then raise ElasticRecord::ExpiredScrollError, e.message
+        else raise e
+        end
       end
 
-      def bulk(options = {})
-        connection.bulk_stack.push []
-
-        yield
-
-        if current_bulk_batch.any?
-          body = current_bulk_batch.map { |action| "#{ElasticRecord::JSON.encode(action)}\n" }.join
-          results = connection.json_post("/_bulk?#{options.to_query}", body)
-          verify_bulk_results(results)
+      def bulk(options = {}, &block)
+        if current_bulk_batch
+          yield
+        else
+          start_new_bulk_batch(options, &block)
         end
-      ensure
-        connection.bulk_stack.pop
       end
 
       def bulk_add(batch, index_name: alias_name)
@@ -185,11 +184,25 @@ module ElasticRecord
       end
 
       def current_bulk_batch
-        connection.bulk_stack.last
+        connection.bulk_actions
       end
 
       private
 
+        def start_new_bulk_batch(options, &block)
+          connection.bulk_actions = []
+
+          yield
+
+          if current_bulk_batch.any?
+            body = current_bulk_batch.map { |action| "#{JSON.generate(action)}\n" }.join
+            results = connection.json_post("/_bulk?#{options.to_query}", body)
+            verify_bulk_results(results)
+          end
+        ensure
+          connection.bulk_actions = nil
+        end
+
         def verify_bulk_results(results)
           return unless results.is_a?(Hash)
 

data/lib/elastic_record/index/manage.rb

@@ -7,10 +7,12 @@ module ElasticRecord
         index_name
       end
 
-      def create(index_name = new_index_name)
+      def create(index_name = new_index_name, setting_overrides: {})
         connection.json_put "/#{index_name}", {
-          "mappings" => mapping_body,
-          "settings" => settings
+          "mappings" => {
+            mapping_type => mapping
+          },
+          "settings" => setting_overrides.merge(settings)
         }
         index_name
       end
@@ -29,10 +31,6 @@ module ElasticRecord
         connection.head("/#{index_name}") == '200'
       end
 
-      def type_exists?(index_name = alias_name, type = model.doctype.name)
-        connection.head("/#{index_name}/_mapping/#{type}") == '200'
-      end
-
       def deploy(index_name)
         actions = [
           {
@@ -76,7 +74,7 @@ module ElasticRecord
       end
 
       def all_names
-        connection.json_get("/#{alias_name}/_mapping/#{model.doctype.name}/").keys
+        connection.json_get("/#{alias_name}/_mapping").keys
       rescue
         # TODO: In ES 1.4, this returns empty rather than a 404
         []

data/lib/elastic_record/index/mapping.rb

@@ -1,23 +1,31 @@
 module ElasticRecord
   class Index
     module Mapping
+      attr_accessor :mapping
+
+      DEFAULT_MAPPING = {
+        properties: {
+        }
+      }
+      def mapping
+        @mapping ||= DEFAULT_MAPPING.deep_dup
+      end
+
+      def mapping=(custom_mapping)
+        mapping.deep_merge!(custom_mapping.deep_dup)
+      end
+
       def update_mapping(index_name = alias_name)
-        connection.json_put "/#{index_name}/_mapping", mapping_body
+        connection.json_put "/#{index_name}/_mapping/#{mapping_type}", mapping
       end
 
       def get_mapping(index_name = alias_name)
-        json = connection.json_get "/#{index_name}/_mapping"
+        json = connection.json_get "/#{index_name}/_mapping/#{mapping_type}"
 
         unless json.empty?
           json.values.first['mappings']
         end
       end
-
-      def mapping_body
-        doctypes.each_with_object({}) do |doctype, result|
-          result[doctype.name] = doctype.mapping
-        end
-      end
     end
   end
 end

data/lib/elastic_record/index/mapping_type.rb

@@ -0,0 +1,16 @@
+module ElasticRecord
+  class Index
+    # This module facilitates the removal of multiple mapping types from ElasticSearch.
+    # See https://www.elastic.co/guide/en/elasticsearch/reference/6.x/removal-of-types.html
+    #
+    #   * 6.x - Type defaults to _doc, but any type can be specified
+    #   * 7.x - Only _doc will be supported, effectively removing the type concept.
+    module MappingType
+      attr_accessor :mapping_type
+
+      def mapping_type
+        @mapping_type || '_doc'
+      end
+    end
+  end
+end

data/lib/elastic_record/index/mapping_type_test.rb

@@ -0,0 +1,18 @@
+require 'helper'
+
+class ElasticRecord::Index::MappingTypeTest < MiniTest::Test
+  def test_default
+    assert_equal '_doc', index.mapping_type
+  end
+
+  def test_writer
+    index.mapping_type = 'widget'
+    assert_equal 'widget', index.mapping_type
+  end
+
+  private
+
+    def index
+      @index ||= ElasticRecord::Index.new(Widget)
+    end
+end

data/lib/elastic_record/index/settings.rb

@@ -6,27 +6,16 @@ module ElasticRecord
       end
 
       def settings
-        @settings ||=
-          begin
-            result = {}
-
-            if (analysis = analysis_body).any?
-              result['analysis'] = analysis
-            end
-
-            result
-          end
+        @settings ||= begin
+          result = ElasticRecord::Config.default_index_settings.deep_dup
+          result['analysis'] = analysis if analysis.any?
+          result
+        end
       end
 
-      def update_settings(index_name = alias_name)
+      def update_settings(index_name = alias_name, settings: self.settings)
         connection.json_put "/#{index_name}/_settings", settings
       end
-
-      def analysis_body
-        doctypes.each_with_object({}) do |doctype, result|
-          result.deep_merge!(doctype.analysis)
-        end
-      end
     end
   end
 end

data/lib/elastic_record/model.rb

@@ -6,7 +6,6 @@ module ElasticRecord
         extend ClassMethods
         include Callbacks
         include AsDocument
-        include NameCache
 
         class_attribute :elastic_connection
         self.elastic_connection = ElasticRecord::Connection.new(ElasticRecord::Config.servers, ElasticRecord::Config.connection_options)
@@ -19,7 +18,8 @@ module ElasticRecord
 
         if child < child.base_class
           child.elastic_index = elastic_index.dup
-          child.doctype = doctype.dup
+          child.elastic_index.model = child
+          child.elastic_index.mapping_type = elastic_index.mapping_type
         end
       end
 
@@ -27,14 +27,6 @@ module ElasticRecord
         Arelastic::Builders::Search
       end
 
-      def doctype
-        @doctype ||= Doctype.new(base_class.name.demodulize.underscore)
-      end
-
-      def doctype=(new_doctype)
-        @doctype = new_doctype
-      end
-
       def elastic_index
         @elastic_index ||= ElasticRecord::Index.new(self)
       end
@@ -51,9 +43,5 @@ module ElasticRecord
     def elastic_index
       self.class.elastic_index
     end
-
-    def doctype
-      self.class.doctype
-    end
   end
 end