searchkick 3.1.2 → 4.4.2

data/lib/searchkick.rb

@@ -1,8 +1,10 @@
-require "active_model"
+# dependencies
+require "active_support"
 require "active_support/core_ext/hash/deep_merge"
 require "elasticsearch"
 require "hashie"
 
+# modules
 require "searchkick/bulk_indexer"
 require "searchkick/index"
 require "searchkick/indexer"
@@ -17,29 +19,18 @@ require "searchkick/record_indexer"
 require "searchkick/results"
 require "searchkick/version"
 
+# integrations
+require "searchkick/railtie" if defined?(Rails)
 require "searchkick/logging" if defined?(ActiveSupport::Notifications)
 
-begin
-  require "rake"
-rescue LoadError
-  # do nothing
-end
-require "searchkick/tasks" if defined?(Rake)
-
-# background jobs
-begin
-  require "active_job"
-rescue LoadError
-  # do nothing
-end
-if defined?(ActiveJob)
-  require "searchkick/bulk_reindex_job"
-  require "searchkick/process_batch_job"
-  require "searchkick/process_queue_job"
-  require "searchkick/reindex_v2_job"
-end
-
 module Searchkick
+  # background jobs
+  autoload :BulkReindexJob,  "searchkick/bulk_reindex_job"
+  autoload :ProcessBatchJob, "searchkick/process_batch_job"
+  autoload :ProcessQueueJob, "searchkick/process_queue_job"
+  autoload :ReindexV2Job,    "searchkick/reindex_v2_job"
+
+  # errors
   class Error < StandardError; end
   class MissingIndexError < Error; end
   class UnsupportedVersionError < Error; end
@@ -62,7 +53,7 @@ module Searchkick
 
   def self.client
     @client ||= begin
-      require "typhoeus/adapters/faraday" if defined?(Typhoeus)
+      require "typhoeus/adapters/faraday" if defined?(Typhoeus) && Gem::Version.new(Faraday::VERSION) < Gem::Version.new("0.14.0")
 
       Elasticsearch::Client.new({
         url: ENV["ELASTICSEARCH_URL"],
@@ -80,7 +71,7 @@ module Searchkick
   end
 
   def self.search_timeout
-    @search_timeout || timeout
+    (defined?(@search_timeout) && @search_timeout) || timeout
   end
 
   def self.server_version
@@ -91,21 +82,41 @@ module Searchkick
     Gem::Version.new(server_version.split("-")[0]) < Gem::Version.new(version.split("-")[0])
   end
 
+  # memoize for performance
+  def self.server_below7?
+    unless defined?(@server_below7)
+      @server_below7 = server_below?("7.0.0")
+    end
+    @server_below7
+  end
+
   def self.search(term = "*", model: nil, **options, &block)
     options = options.dup
     klass = model
 
-    # make Searchkick.search(index_name: [Product]) and Product.search equivalent
+    # convert index_name into models if possible
+    # this should allow for easier upgrade
+    if options[:index_name] && !options[:models] && Array(options[:index_name]).all? { |v| v.respond_to?(:searchkick_index) }
+      options[:models] = options.delete(:index_name)
+    end
+
+    # make Searchkick.search(models: [Product]) and Product.search equivalent
     unless klass
-      index_name = Array(options[:index_name])
-      if index_name.size == 1 && index_name.first.respond_to?(:searchkick_index)
-        klass = index_name.first
-        options.delete(:index_name)
+      models = Array(options[:models])
+      if models.size == 1
+        klass = models.first
+        options.delete(:models)
+      end
+    end
+
+    if klass
+      if (options[:models] && Array(options[:models]) != [klass]) || Array(options[:index_name]).any? { |v| v.respond_to?(:searchkick_index) && v != klass }
+        raise ArgumentError, "Use Searchkick.search to search multiple models"
       end
     end
 
-    query = Searchkick::Query.new(klass, term, options)
-    block.call(query.body) if block
+    options = options.merge(block: block) if block
+    query = Searchkick::Query.new(klass, term, **options)
     if options[:execute] == false
       query
     else
@@ -135,7 +146,7 @@ module Searchkick
     end
   end
 
-  def self.callbacks(value)
+  def self.callbacks(value = nil)
     if block_given?
       previous_value = callbacks_value
       begin
@@ -153,6 +164,7 @@ module Searchkick
 
   def self.aws_credentials=(creds)
     begin
+      # TODO remove in Searchkick 5 (just use aws_sigv4)
       require "faraday_middleware/aws_signers_v4"
     rescue LoadError
       require "faraday_middleware/aws_sigv4"
@@ -162,17 +174,16 @@ module Searchkick
   end
 
   def self.reindex_status(index_name)
-    if redis
-      batches_left = Searchkick::Index.new(index_name).batches_left
-      {
-        completed: batches_left == 0,
-        batches_left: batches_left
-      }
-    else
-      raise Searchkick::Error, "Redis not configured"
-    end
+    raise Searchkick::Error, "Redis not configured" unless redis
+
+    batches_left = Searchkick::Index.new(index_name).batches_left
+    {
+      completed: batches_left == 0,
+      batches_left: batches_left
+    }
   end
 
+  # TODO use ConnectionPool::Wrapper when redis is set so this is no longer needed
   def self.with_redis
     if redis
       if redis.respond_to?(:with)
@@ -185,6 +196,10 @@ module Searchkick
     end
   end
 
+  def self.warn(message)
+    super("[searchkick] WARNING: #{message}")
+  end
+
   # private
   def self.load_records(records, ids)
     records =
@@ -238,10 +253,26 @@ module Searchkick
       }
     end
   end
+
+  # private
+  # methods are forwarded to base class
+  # this check to see if scope exists on that class
+  # it's a bit tricky, but this seems to work
+  def self.relation?(klass)
+    if klass.respond_to?(:current_scope)
+      !klass.current_scope.nil?
+    elsif defined?(Mongoid::Threaded)
+      !Mongoid::Threaded.current_scope(klass).nil?
+    end
+  end
 end
 
-# TODO find better ActiveModel hook
+require "active_model/callbacks"
 ActiveModel::Callbacks.include(Searchkick::Model)
+# TODO use
+# ActiveSupport.on_load(:mongoid) do
+#   Mongoid::Document::ClassMethods.include Searchkick::Model
+# end
 
 ActiveSupport.on_load(:active_record) do
   extend Searchkick::Model

data/lib/searchkick/bulk_indexer.rb

@@ -61,7 +61,7 @@ module Searchkick
       if records.any?
         if async
           Searchkick::BulkReindexJob.perform_later(
-            class_name: records.first.class.name,
+            class_name: records.first.class.searchkick_options[:class_name],
             record_ids: records.map(&:id),
             index_name: index.name,
             method_name: method_name ? method_name.to_s : nil
@@ -87,6 +87,8 @@ module Searchkick
         # TODO expire Redis key
         primary_key = scope.primary_key
 
+        scope = scope.select(primary_key).except(:includes, :preload)
+
         starting_id =
           begin
             scope.minimum(primary_key)
@@ -139,8 +141,8 @@ module Searchkick
 
     def bulk_reindex_job(scope, batch_id, options)
       Searchkick.with_redis { |r| r.sadd(batches_key, batch_id) }
-      Searchkick::BulkReindexJob.perform_later({
-        class_name: scope.model_name.name,
+      Searchkick::BulkReindexJob.perform_later(**{
+        class_name: scope.searchkick_options[:class_name],
         index_name: index.name,
         batch_id: batch_id
       }.merge(options))

data/lib/searchkick/index.rb

@@ -2,8 +2,6 @@ require "searchkick/index_options"
 
 module Searchkick
   class Index
-    include IndexOptions
-
     attr_reader :name, :options
 
     def initialize(name, options = {})
@@ -12,12 +10,16 @@ module Searchkick
       @klass_document_type = {} # cache
     end
 
+    def index_options
+      IndexOptions.new(self).index_options
+    end
+
     def create(body = {})
       client.indices.create index: name, body: body
     end
 
     def delete
-      if !Searchkick.server_below?("6.0.0") && alias_exists?
+      if alias_exists?
         # can't call delete directly on aliases in ES 6
         indices = client.indices.get_alias(name: name).keys
         client.indices.delete index: indices
@@ -47,7 +49,7 @@ module Searchkick
     end
 
     def refresh_interval
-      settings.values.first["settings"]["index"]["refresh_interval"]
+      index_settings["refresh_interval"]
     end
 
     def update_settings(settings)
@@ -68,7 +70,7 @@ module Searchkick
           }
         )
 
-      response["hits"]["total"]
+      Searchkick::Results.new(nil, response).total_count
     end
 
     def promote(new_name, update_refresh_interval: false)
@@ -91,17 +93,22 @@ module Searchkick
     alias_method :swap, :promote
 
     def retrieve(record)
-      client.get(
-        index: name,
-        type: document_type(record),
-        id: search_id(record)
-      )["_source"]
+      record_data = RecordData.new(self, record).record_data
+
+      # remove underscore
+      get_options = Hash[record_data.map { |k, v| [k.to_s.sub(/\A_/, "").to_sym, v] }]
+
+      client.get(get_options)["_source"]
     end
 
     def all_indices(unaliased: false)
       indices =
         begin
-          client.indices.get_aliases
+          if client.indices.respond_to?(:get_alias)
+            client.indices.get_alias
+          else
+            client.indices.get_aliases
+          end
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
           {}
         end
@@ -169,6 +176,17 @@ module Searchkick
       Searchkick.search(like_text, model: record.class, **options)
     end
 
+    def reload_synonyms
+      require "elasticsearch/xpack"
+      raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
+      raise Error, "Requires elasticsearch-xpack 7.8+" unless client.xpack.respond_to?(:indices)
+      begin
+        client.xpack.indices.reload_search_analyzers(index: name)
+      rescue Elasticsearch::Transport::Transport::Errors::MethodNotAllowed
+        raise Error, "Requires non-OSS version of Elasticsearch"
+      end
+    end
+
     # queue
 
     def reindex_queue
@@ -179,13 +197,20 @@ module Searchkick
 
     def reindex(relation, method_name, scoped:, full: false, scope: nil, **options)
       refresh = options.fetch(:refresh, !scoped)
+      options.delete(:refresh)
 
       if method_name
+        # TODO throw ArgumentError
+        Searchkick.warn("unsupported keywords: #{options.keys.map(&:inspect).join(", ")}") if options.any?
+
         # update
         import_scope(relation, method_name: method_name, scope: scope)
         self.refresh if refresh
         true
       elsif scoped && !full
+        # TODO throw ArgumentError
+        Searchkick.warn("unsupported keywords: #{options.keys.map(&:inspect).join(", ")}") if options.any?
+
         # reindex association
         import_scope(relation, scope: scope)
         self.refresh if refresh
@@ -244,6 +269,11 @@ module Searchkick
       end
     end
 
+    # private
+    def uuid
+      index_settings["uuid"]
+    end
+
     protected
 
     def client
@@ -254,6 +284,14 @@ module Searchkick
       @bulk_indexer ||= BulkIndexer.new(self)
     end
 
+    def index_settings
+      settings.values.first["settings"]["index"]
+    end
+
+    def import_before_promotion(index, relation, **import_options)
+      index.import_scope(relation, **import_options)
+    end
+
     # https://gist.github.com/jarosan/3124884
     # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
     def reindex_scope(relation, import: true, resume: false, retain: false, async: false, refresh_interval: nil, scope: nil)
@@ -276,14 +314,16 @@ module Searchkick
         scope: scope
       }
 
+      uuid = index.uuid
+
       # check if alias exists
       alias_exists = alias_exists?
       if alias_exists
-        # import before promotion
-        index.import_scope(relation, **import_options) if import
+        import_before_promotion(index, relation, **import_options) if import
 
         # get existing indices to remove
         unless async
+          check_uuid(uuid, index.uuid)
           promote(index.name, update_refresh_interval: !refresh_interval.nil?)
           clean_indices unless retain
         end
@@ -308,6 +348,7 @@ module Searchkick
           # already promoted if alias didn't exist
           if alias_exists
             puts "Jobs complete. Promoting..."
+            check_uuid(uuid, index.uuid)
             promote(index.name, update_refresh_interval: !refresh_interval.nil?)
           end
           clean_indices unless retain
@@ -326,5 +367,15 @@ module Searchkick
 
       raise e
     end
+
+    # safety check
+    # still a chance for race condition since its called before promotion
+    # ideal is for user to disable automatic index creation
+    # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-creation
+    def check_uuid(old_uuid, new_uuid)
+      if old_uuid != new_uuid
+        raise Searchkick::Error, "Safety check failed - only run one Model.reindex per model at a time"
+      end
+    end
   end
 end

data/lib/searchkick/index_options.rb

@@ -1,426 +1,558 @@
 module Searchkick
-  module IndexOptions
+  class IndexOptions
+    attr_reader :options
+
+    def initialize(index)
+      @options = index.options
+    end
+
     def index_options
-      options = @options
-      language = options[:language]
-      language = language.call if language.respond_to?(:call)
-      index_type = options[:_type]
-      index_type = index_type.call if index_type.respond_to?(:call)
+      custom_mapping = options[:mappings] || {}
+      if below70? && custom_mapping.keys.map(&:to_sym).include?(:properties)
+        # add type
+        custom_mapping = {index_type => custom_mapping}
+      end
 
       if options[:mappings] && !options[:merge_mappings]
         settings = options[:settings] || {}
-        mappings = options[:mappings]
+        mappings = custom_mapping
       else
-        below60 = Searchkick.server_below?("6.0.0")
-        below62 = Searchkick.server_below?("6.2.0")
-
-        default_type = "text"
-        default_analyzer = :searchkick_index
-        keyword_mapping = {type: "keyword"}
-
-        all = options.key?(:_all) ? options[:_all] : false
-        index_true_value = true
-        index_false_value = false
-
-        keyword_mapping[:ignore_above] = options[:ignore_above] || 30000
-
-        settings = {
-          analysis: {
-            analyzer: {
-              searchkick_keyword: {
-                type: "custom",
-                tokenizer: "keyword",
-                filter: ["lowercase"] + (options[:stem_conversions] ? ["searchkick_stemmer"] : [])
-              },
-              default_analyzer => {
-                type: "custom",
-                # character filters -> tokenizer -> token filters
-                # https://www.elastic.co/guide/en/elasticsearch/guide/current/analysis-intro.html
-                char_filter: ["ampersand"],
-                tokenizer: "standard",
-                # synonym should come last, after stemming and shingle
-                # shingle must come before searchkick_stemmer
-                filter: ["standard", "lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"]
-              },
-              searchkick_search: {
-                type: "custom",
-                char_filter: ["ampersand"],
-                tokenizer: "standard",
-                filter: ["standard", "lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"]
-              },
-              searchkick_search2: {
-                type: "custom",
-                char_filter: ["ampersand"],
-                tokenizer: "standard",
-                filter: ["standard", "lowercase", "asciifolding", "searchkick_stemmer"]
-              },
-              # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
-              searchkick_autocomplete_search: {
-                type: "custom",
-                tokenizer: "keyword",
-                filter: ["lowercase", "asciifolding"]
-              },
-              searchkick_word_search: {
-                type: "custom",
-                tokenizer: "standard",
-                filter: ["lowercase", "asciifolding"]
-              },
-              searchkick_suggest_index: {
-                type: "custom",
-                tokenizer: "standard",
-                filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
-              },
-              searchkick_text_start_index: {
-                type: "custom",
-                tokenizer: "keyword",
-                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
-              },
-              searchkick_text_middle_index: {
-                type: "custom",
-                tokenizer: "keyword",
-                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
-              },
-              searchkick_text_end_index: {
-                type: "custom",
-                tokenizer: "keyword",
-                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
-              },
-              searchkick_word_start_index: {
-                type: "custom",
-                tokenizer: "standard",
-                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
-              },
-              searchkick_word_middle_index: {
-                type: "custom",
-                tokenizer: "standard",
-                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
-              },
-              searchkick_word_end_index: {
-                type: "custom",
-                tokenizer: "standard",
-                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
-              }
-            },
-            filter: {
-              searchkick_index_shingle: {
-                type: "shingle",
-                token_separator: ""
-              },
-              # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7
-              searchkick_search_shingle: {
-                type: "shingle",
-                token_separator: "",
-                output_unigrams: false,
-                output_unigrams_if_no_shingles: true
-              },
-              searchkick_suggest_shingle: {
-                type: "shingle",
-                max_shingle_size: 5
-              },
-              searchkick_edge_ngram: {
-                type: "edgeNGram",
-                min_gram: 1,
-                max_gram: 50
-              },
-              searchkick_ngram: {
-                type: "nGram",
-                min_gram: 1,
-                max_gram: 50
-              },
-              searchkick_stemmer: {
-                # use stemmer if language is lowercase, snowball otherwise
-                type: language == language.to_s.downcase ? "stemmer" : "snowball",
-                language: language || "English"
-              }
-            },
-            char_filter: {
-              # https://www.elastic.co/guide/en/elasticsearch/guide/current/custom-analyzers.html
-              # &_to_and
-              ampersand: {
-                type: "mapping",
-                mappings: ["&=> and "]
-              }
-            }
-          }
-        }
+        settings = generate_settings
+        mappings = generate_mappings.symbolize_keys.deep_merge(custom_mapping.symbolize_keys)
+      end
+
+      set_deep_paging(settings) if options[:deep_paging]
+
+      {
+        settings: settings,
+        mappings: mappings
+      }
+    end
 
-        case language
-        when "chinese"
-          settings[:analysis][:analyzer].merge!(
+    def generate_settings
+      language = options[:language]
+      language = language.call if language.respond_to?(:call)
+
+      settings = {
+        analysis: {
+          analyzer: {
+            searchkick_keyword: {
+              type: "custom",
+              tokenizer: "keyword",
+              filter: ["lowercase"] + (options[:stem_conversions] ? ["searchkick_stemmer"] : [])
+            },
             default_analyzer => {
-              type: "ik_smart"
+              type: "custom",
+              # character filters -> tokenizer -> token filters
+              # https://www.elastic.co/guide/en/elasticsearch/guide/current/analysis-intro.html
+              char_filter: ["ampersand"],
+              tokenizer: "standard",
+              # synonym should come last, after stemming and shingle
+              # shingle must come before searchkick_stemmer
+              filter: ["lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"]
             },
             searchkick_search: {
-              type: "ik_smart"
+              type: "custom",
+              char_filter: ["ampersand"],
+              tokenizer: "standard",
+              filter: ["lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"]
             },
             searchkick_search2: {
-              type: "ik_max_word"
-            }
-          )
-
-          settings[:analysis][:filter].delete(:searchkick_stemmer)
-        when "japanese"
-          settings[:analysis][:analyzer].merge!(
-            default_analyzer => {
-              type: "kuromoji"
+              type: "custom",
+              char_filter: ["ampersand"],
+              tokenizer: "standard",
+              filter: ["lowercase", "asciifolding", "searchkick_stemmer"]
             },
-            searchkick_search: {
-              type: "kuromoji"
+            # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
+            searchkick_autocomplete_search: {
+              type: "custom",
+              tokenizer: "keyword",
+              filter: ["lowercase", "asciifolding"]
             },
-            searchkick_search2: {
-              type: "kuromoji"
-            }
-          )
-        when "korean"
-          settings[:analysis][:analyzer].merge!(
-            default_analyzer => {
-              type: "openkoreantext-analyzer"
+            searchkick_word_search: {
+              type: "custom",
+              tokenizer: "standard",
+              filter: ["lowercase", "asciifolding"]
             },
-            searchkick_search: {
-              type: "openkoreantext-analyzer"
+            searchkick_suggest_index: {
+              type: "custom",
+              tokenizer: "standard",
+              filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
             },
-            searchkick_search2: {
-              type: "openkoreantext-analyzer"
-            }
-          )
-        when "vietnamese"
-          settings[:analysis][:analyzer].merge!(
-            default_analyzer => {
-              type: "vi_analyzer"
+            searchkick_text_start_index: {
+              type: "custom",
+              tokenizer: "keyword",
+              filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
             },
-            searchkick_search: {
-              type: "vi_analyzer"
+            searchkick_text_middle_index: {
+              type: "custom",
+              tokenizer: "keyword",
+              filter: ["lowercase", "asciifolding", "searchkick_ngram"]
             },
-            searchkick_search2: {
-              type: "vi_analyzer"
+            searchkick_text_end_index: {
+              type: "custom",
+              tokenizer: "keyword",
+              filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
+            },
+            searchkick_word_start_index: {
+              type: "custom",
+              tokenizer: "standard",
+              filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+            },
+            searchkick_word_middle_index: {
+              type: "custom",
+              tokenizer: "standard",
+              filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+            },
+            searchkick_word_end_index: {
+              type: "custom",
+              tokenizer: "standard",
+              filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
             }
-          )
-        when "polish", "ukrainian", "smartcn"
-          settings[:analysis][:analyzer].merge!(
-            default_analyzer => {
-              type: language
+          },
+          filter: {
+            searchkick_index_shingle: {
+              type: "shingle",
+              token_separator: ""
             },
-            searchkick_search: {
-              type: language
+            # lucky find https://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7
+            searchkick_search_shingle: {
+              type: "shingle",
+              token_separator: "",
+              output_unigrams: false,
+              output_unigrams_if_no_shingles: true
             },
-            searchkick_search2: {
-              type: language
+            searchkick_suggest_shingle: {
+              type: "shingle",
+              max_shingle_size: 5
+            },
+            searchkick_edge_ngram: {
+              type: "edge_ngram",
+              min_gram: 1,
+              max_gram: 50
+            },
+            searchkick_ngram: {
+              type: "ngram",
+              min_gram: 1,
+              max_gram: 50
+            },
+            searchkick_stemmer: {
+              # use stemmer if language is lowercase, snowball otherwise
+              type: language == language.to_s.downcase ? "stemmer" : "snowball",
+              language: language || "English"
             }
-          )
-        end
+          },
+          char_filter: {
+            # https://www.elastic.co/guide/en/elasticsearch/guide/current/custom-analyzers.html
+            # &_to_and
+            ampersand: {
+              type: "mapping",
+              mappings: ["&=> and "]
+            }
+          }
+        }
+      }
 
-        if Searchkick.env == "test"
-          settings[:number_of_shards] = 1
-          settings[:number_of_replicas] = 0
-        end
+      update_language(settings, language)
+      update_stemming(settings)
 
-        if options[:similarity]
-          settings[:similarity] = {default: {type: options[:similarity]}}
-        end
+      if Searchkick.env == "test"
+        settings[:number_of_shards] = 1
+        settings[:number_of_replicas] = 0
+      end
 
-        unless below62
-          settings[:index] = {
-            max_ngram_diff: 49,
-            max_shingle_diff: 4
-          }
-        end
+      # TODO remove in Searchkick 5 (classic no longer supported)
+      if options[:similarity]
+        settings[:similarity] = {default: {type: options[:similarity]}}
+      end
 
-        if options[:case_sensitive]
-          settings[:analysis][:analyzer].each do |_, analyzer|
-            analyzer[:filter].delete("lowercase")
-          end
-        end
+      unless below62?
+        settings[:index] = {
+          max_ngram_diff: 49,
+          max_shingle_diff: 4
+        }
+      end
 
-        if options[:stem] == false
-          settings[:analysis][:analyzer].each do |_, analyzer|
-            analyzer[:filter].delete("searchkick_stemmer")
-          end
+      if options[:case_sensitive]
+        settings[:analysis][:analyzer].each do |_, analyzer|
+          analyzer[:filter].delete("lowercase")
         end
+      end
 
-        settings = settings.symbolize_keys.deep_merge((options[:settings] || {}).symbolize_keys)
-
-        # synonyms
-        synonyms = options[:synonyms] || []
+      # TODO do this last in Searchkick 5
+      settings = settings.symbolize_keys.deep_merge((options[:settings] || {}).symbolize_keys)
 
-        synonyms = synonyms.call if synonyms.respond_to?(:call)
+      add_synonyms(settings)
+      add_search_synonyms(settings)
+      # TODO remove in Searchkick 5
+      add_wordnet(settings) if options[:wordnet]
 
-        if synonyms.any?
-          settings[:analysis][:filter][:searchkick_synonym] = {
-            type: "synonym",
-            # only remove a single space from synonyms so three-word synonyms will fail noisily instead of silently
-            synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |s2| s2.sub(/\s+/, "") }.join(",") : s }.map(&:downcase)
-          }
-          # choosing a place for the synonym filter when stemming is not easy
-          # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
-          # TODO use a snowball stemmer on synonyms when creating the token filter
-
-          # http://elasticsearch-users.115913.n3.nabble.com/synonym-multi-words-search-td4030811.html
-          # I find the following approach effective if you are doing multi-word synonyms (synonym phrases):
-          # - Only apply the synonym expansion at index time
-          # - Don't have the synonym filter applied search
-          # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
-          settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_synonym") if below60
-          settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_synonym"
-
-          %w(word_start word_middle word_end).each do |type|
-            settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_synonym")
-          end
+      if options[:special_characters] == false
+        settings[:analysis][:analyzer].each_value do |analyzer_settings|
+          analyzer_settings[:filter].reject! { |f| f == "asciifolding" }
         end
+      end
 
-        if options[:wordnet]
-          settings[:analysis][:filter][:searchkick_wordnet] = {
-            type: "synonym",
-            format: "wordnet",
-            synonyms_path: Searchkick.wordnet_path
+      settings
+    end
+
+    def update_language(settings, language)
+      case language
+      when "chinese"
+        settings[:analysis][:analyzer].merge!(
+          default_analyzer => {
+            type: "ik_smart"
+          },
+          searchkick_search: {
+            type: "ik_smart"
+          },
+          searchkick_search2: {
+            type: "ik_max_word"
+          }
+        )
+      when "chinese2", "smartcn"
+        settings[:analysis][:analyzer].merge!(
+          default_analyzer => {
+            type: "smartcn"
+          },
+          searchkick_search: {
+            type: "smartcn"
+          },
+          searchkick_search2: {
+            type: "smartcn"
+          }
+        )
+      when "japanese"
+        settings[:analysis][:analyzer].merge!(
+          default_analyzer => {
+            type: "kuromoji"
+          },
+          searchkick_search: {
+            type: "kuromoji"
+          },
+          searchkick_search2: {
+            type: "kuromoji"
+          }
+        )
+      when "korean"
+        settings[:analysis][:analyzer].merge!(
+          default_analyzer => {
+            type: "openkoreantext-analyzer"
+          },
+          searchkick_search: {
+            type: "openkoreantext-analyzer"
+          },
+          searchkick_search2: {
+            type: "openkoreantext-analyzer"
           }
+        )
+      when "korean2"
+        settings[:analysis][:analyzer].merge!(
+          default_analyzer => {
+            type: "nori"
+          },
+          searchkick_search: {
+            type: "nori"
+          },
+          searchkick_search2: {
+            type: "nori"
+          }
+        )
+      when "vietnamese"
+        settings[:analysis][:analyzer].merge!(
+          default_analyzer => {
+            type: "vi_analyzer"
+          },
+          searchkick_search: {
+            type: "vi_analyzer"
+          },
+          searchkick_search2: {
+            type: "vi_analyzer"
+          }
+        )
+      when "polish", "ukrainian"
+        settings[:analysis][:analyzer].merge!(
+          default_analyzer => {
+            type: language
+          },
+          searchkick_search: {
+            type: language
+          },
+          searchkick_search2: {
+            type: language
+          }
+        )
+      end
+    end
 
-          settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_wordnet")
-          settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_wordnet"
+    def update_stemming(settings)
+      stem = options[:stem]
 
-          %w(word_start word_middle word_end).each do |type|
-            settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_wordnet")
-          end
-        end
+      # language analyzer used
+      stem = false if settings[:analysis][:analyzer][default_analyzer][:type] != "custom"
 
-        if options[:special_characters] == false
-          settings[:analysis][:analyzer].each_value do |analyzer_settings|
-            analyzer_settings[:filter].reject! { |f| f == "asciifolding" }
-          end
+      if stem == false
+        settings[:analysis][:filter].delete(:searchkick_stemmer)
+        settings[:analysis][:analyzer].each do |_, analyzer|
+          analyzer[:filter].delete("searchkick_stemmer") if analyzer[:filter]
         end
+      end
 
-        mapping = {}
+      if options[:stemmer_override]
+        stemmer_override = {
+          type: "stemmer_override"
+        }
+        if options[:stemmer_override].is_a?(String)
+          stemmer_override[:rules_path] = options[:stemmer_override]
+        else
+          stemmer_override[:rules] = options[:stemmer_override]
+        end
+        settings[:analysis][:filter][:searchkick_stemmer_override] = stemmer_override
 
-        # conversions
-        Array(options[:conversions]).each do |conversions_field|
-          mapping[conversions_field] = {
-            type: "nested",
-            properties: {
-              query: {type: default_type, analyzer: "searchkick_keyword"},
-              count: {type: "integer"}
-            }
-          }
+        settings[:analysis][:analyzer].each do |_, analyzer|
+          stemmer_index = analyzer[:filter].index("searchkick_stemmer") if analyzer[:filter]
+          analyzer[:filter].insert(stemmer_index, "searchkick_stemmer_override") if stemmer_index
         end
+      end
 
-        mapping_options = Hash[
-          [:suggest, :word, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight, :searchable, :filterable]
-            .map { |type| [type, (options[type] || []).map(&:to_s)] }
-        ]
+      if options[:stem_exclusion]
+        settings[:analysis][:filter][:searchkick_stem_exclusion] = {
+          type: "keyword_marker",
+          keywords: options[:stem_exclusion]
+        }
 
-        word = options[:word] != false && (!options[:match] || options[:match] == :word)
+        settings[:analysis][:analyzer].each do |_, analyzer|
+          stemmer_index = analyzer[:filter].index("searchkick_stemmer") if analyzer[:filter]
+          analyzer[:filter].insert(stemmer_index, "searchkick_stem_exclusion") if stemmer_index
+        end
+      end
+    end
 
-        mapping_options[:searchable].delete("_all")
+    def generate_mappings
+      mapping = {}
 
-        analyzed_field_options = {type: default_type, index: index_true_value, analyzer: default_analyzer}
+      keyword_mapping = {type: "keyword"}
+      keyword_mapping[:ignore_above] = options[:ignore_above] || 30000
 
-        mapping_options.values.flatten.uniq.each do |field|
-          fields = {}
+      # conversions
+      Array(options[:conversions]).each do |conversions_field|
+        mapping[conversions_field] = {
+          type: "nested",
+          properties: {
+            query: {type: default_type, analyzer: "searchkick_keyword"},
+            count: {type: "integer"}
+          }
+        }
+      end
 
-          if options.key?(:filterable) && !mapping_options[:filterable].include?(field)
-            fields[field] = {type: default_type, index: index_false_value}
-          else
-            fields[field] = keyword_mapping
-          end
+      mapping_options = Hash[
+        [:suggest, :word, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight, :searchable, :filterable]
+          .map { |type| [type, (options[type] || []).map(&:to_s)] }
+      ]
 
-          if !options[:searchable] || mapping_options[:searchable].include?(field)
-            if word
-              fields[:analyzed] = analyzed_field_options
+      word = options[:word] != false && (!options[:match] || options[:match] == :word)
 
-              if mapping_options[:highlight].include?(field)
-                fields[:analyzed][:term_vector] = "with_positions_offsets"
-              end
-            end
+      mapping_options[:searchable].delete("_all")
 
-            mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f|
-              if options[:match] == type || f.include?(field)
-                fields[type] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{type}_index"}
-              end
-            end
-          end
+      analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer}
 
-          mapping[field] = fields[field].merge(fields: fields.except(field))
-        end
+      mapping_options.values.flatten.uniq.each do |field|
+        fields = {}
 
-        (options[:locations] || []).map(&:to_s).each do |field|
-          mapping[field] = {
-            type: "geo_point"
-          }
+        if options.key?(:filterable) && !mapping_options[:filterable].include?(field)
+          fields[field] = {type: default_type, index: false}
+        else
+          fields[field] = keyword_mapping
         end
 
-        options[:geo_shape] = options[:geo_shape].product([{}]).to_h if options[:geo_shape].is_a?(Array)
-        (options[:geo_shape] || {}).each do |field, shape_options|
-          mapping[field] = shape_options.merge(type: "geo_shape")
-        end
+        if !options[:searchable] || mapping_options[:searchable].include?(field)
+          if word
+            fields[:analyzed] = analyzed_field_options
 
-        if options[:inheritance]
-          mapping[:type] = keyword_mapping
-        end
+            if mapping_options[:highlight].include?(field)
+              fields[:analyzed][:term_vector] = "with_positions_offsets"
+            end
+          end
 
-        routing = {}
-        if options[:routing]
-          routing = {required: true}
-          unless options[:routing] == true
-            routing[:path] = options[:routing].to_s
+          mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f|
+            if options[:match] == type || f.include?(field)
+              fields[type] = {type: default_type, index: true, analyzer: "searchkick_#{type}_index"}
+            end
           end
         end
 
-        dynamic_fields = {
-          # analyzed field must be the default field for include_in_all
-          # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
-          # however, we can include the not_analyzed field in _all
-          # and the _all index analyzer will take care of it
-          "{name}" => keyword_mapping
+        mapping[field] = fields[field].merge(fields: fields.except(field))
+      end
+
+      (options[:locations] || []).map(&:to_s).each do |field|
+        mapping[field] = {
+          type: "geo_point"
         }
+      end
 
-        if below60 && all
-          dynamic_fields["{name}"][:include_in_all] = !options[:searchable]
-        end
+      options[:geo_shape] = options[:geo_shape].product([{}]).to_h if options[:geo_shape].is_a?(Array)
+      (options[:geo_shape] || {}).each do |field, shape_options|
+        mapping[field] = shape_options.merge(type: "geo_shape")
+      end
+
+      if options[:inheritance]
+        mapping[:type] = keyword_mapping
+      end
 
-        if options.key?(:filterable)
-          dynamic_fields["{name}"] = {type: default_type, index: index_false_value}
+      routing = {}
+      if options[:routing]
+        routing = {required: true}
+        unless options[:routing] == true
+          routing[:path] = options[:routing].to_s
         end
+      end
 
-        unless options[:searchable]
-          if options[:match] && options[:match] != :word
-            dynamic_fields[options[:match]] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{options[:match]}_index"}
-          end
+      dynamic_fields = {
+        # analyzed field must be the default field for include_in_all
+        # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
+        # however, we can include the not_analyzed field in _all
+        # and the _all index analyzer will take care of it
+        "{name}" => keyword_mapping
+      }
 
-          if word
-            dynamic_fields[:analyzed] = analyzed_field_options
-          end
+      if options.key?(:filterable)
+        dynamic_fields["{name}"] = {type: default_type, index: false}
+      end
+
+      unless options[:searchable]
+        if options[:match] && options[:match] != :word
+          dynamic_fields[options[:match]] = {type: default_type, index: true, analyzer: "searchkick_#{options[:match]}_index"}
         end
 
-        # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
-        multi_field = dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}"))
-
-        mappings = {
-          index_type => {
-            properties: mapping,
-            _routing: routing,
-            # https://gist.github.com/kimchy/2898285
-            dynamic_templates: [
-              {
-                string_template: {
-                  match: "*",
-                  match_mapping_type: "string",
-                  mapping: multi_field
-                }
-              }
-            ]
+        if word
+          dynamic_fields[:analyzed] = analyzed_field_options
+        end
+      end
+
+      # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
+      multi_field = dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}"))
+
+      mappings = {
+        properties: mapping,
+        _routing: routing,
+        # https://gist.github.com/kimchy/2898285
+        dynamic_templates: [
+          {
+            string_template: {
+              match: "*",
+              match_mapping_type: "string",
+              mapping: multi_field
+            }
           }
+        ]
+      }
+
+      if below70?
+        mappings = {index_type => mappings}
+      end
+
+      mappings
+    end
+
+    def add_synonyms(settings)
+      synonyms = options[:synonyms] || []
+      synonyms = synonyms.call if synonyms.respond_to?(:call)
+      if synonyms.any?
+        settings[:analysis][:filter][:searchkick_synonym] = {
+          type: "synonym",
+          # only remove a single space from synonyms so three-word synonyms will fail noisily instead of silently
+          synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |s2| s2.sub(/\s+/, "") }.join(",") : s }.map(&:downcase)
         }
+        # choosing a place for the synonym filter when stemming is not easy
+        # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
+        # TODO use a snowball stemmer on synonyms when creating the token filter
+
+        # http://elasticsearch-users.115913.n3.nabble.com/synonym-multi-words-search-td4030811.html
+        # I find the following approach effective if you are doing multi-word synonyms (synonym phrases):
+        # - Only apply the synonym expansion at index time
+        # - Don't have the synonym filter applied search
+        # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
+        settings[:analysis][:analyzer][default_analyzer][:filter].insert(2, "searchkick_synonym")
+
+        %w(word_start word_middle word_end).each do |type|
+          settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_synonym")
+        end
+      end
+    end
 
-        if below60
-          all_enabled = all && (!options[:searchable] || options[:searchable].to_a.map(&:to_s).include?("_all"))
-          mappings[index_type][:_all] = all_enabled ? analyzed_field_options : {enabled: false}
+    def add_search_synonyms(settings)
+      search_synonyms = options[:search_synonyms] || []
+      search_synonyms = search_synonyms.call if search_synonyms.respond_to?(:call)
+      if search_synonyms.is_a?(String) || search_synonyms.any?
+        if search_synonyms.is_a?(String)
+          synonym_graph = {
+            type: "synonym_graph",
+            synonyms_path: search_synonyms
+          }
+          synonym_graph[:updateable] = true unless below73?
+        else
+          synonym_graph = {
+            type: "synonym_graph",
+            # TODO confirm this is correct
+            synonyms: search_synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.join(",") : s }.map(&:downcase)
+          }
         end
+        settings[:analysis][:filter][:searchkick_synonym_graph] = synonym_graph
 
-        mappings = mappings.symbolize_keys.deep_merge((options[:mappings] || {}).symbolize_keys)
+        [:searchkick_search2, :searchkick_word_search].each do |analyzer|
+          settings[:analysis][:analyzer][analyzer][:filter].insert(2, "searchkick_synonym_graph")
+        end
       end
+    end
 
-      {
-        settings: settings,
-        mappings: mappings
+    def add_wordnet(settings)
+      settings[:analysis][:filter][:searchkick_wordnet] = {
+        type: "synonym",
+        format: "wordnet",
+        synonyms_path: Searchkick.wordnet_path
       }
+
+      settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_wordnet")
+      settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_wordnet"
+
+      %w(word_start word_middle word_end).each do |type|
+        settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_wordnet")
+      end
+    end
+
+    def set_deep_paging(settings)
+      if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
+        settings[:index] ||= {}
+        settings[:index][:max_result_window] = 1_000_000_000
+      end
+    end
+
+    def index_type
+      @index_type ||= begin
+        index_type = options[:_type]
+        index_type = index_type.call if index_type.respond_to?(:call)
+        index_type
+      end
+    end
+
+    def default_type
+      "text"
+    end
+
+    def default_analyzer
+      :searchkick_index
+    end
+
+    def below62?
+      Searchkick.server_below?("6.2.0")
+    end
+
+    def below70?
+      Searchkick.server_below?("7.0.0")
+    end
+
+    def below73?
+      Searchkick.server_below?("7.3.0")
     end
   end
 end