searchkick-hooopo 2.3.0

data/lib/searchkick/index_options.rb

@@ -0,0 +1,333 @@
+module Searchkick
+  module IndexOptions
+    def index_options
+      options = @options
+      language = options[:language]
+      language = language.call if language.respond_to?(:call)
+
+      if options[:mappings] && !options[:merge_mappings]
+        settings = options[:settings] || {}
+        mappings = options[:mappings]
+      else
+        below22 = Searchkick.server_below?("2.2.0")
+        below50 = Searchkick.server_below?("5.0.0-alpha1")
+        default_type = below50 ? "string" : "text"
+        default_analyzer = :searchkick_index
+        keyword_mapping =
+          if below50
+            {
+              type: default_type,
+              index: "not_analyzed"
+            }
+          else
+            {
+              type: "keyword"
+            }
+          end
+
+        keyword_mapping[:ignore_above] = (options[:ignore_above] || 30000) unless below22
+
+        settings = {
+          analysis: {
+            analyzer: {
+              searchkick_keyword: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase"] + (options[:stem_conversions] == false ? [] : ["searchkick_stemmer"])
+              },
+              default_analyzer => {
+                type: "custom",
+                # character filters -> tokenizer -> token filters
+                # https://www.elastic.co/guide/en/elasticsearch/guide/current/analysis-intro.html
+                char_filter: ["ampersand"],
+                tokenizer: "standard",
+                # synonym should come last, after stemming and shingle
+                # shingle must come before searchkick_stemmer
+                filter: ["standard", "lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"]
+              },
+              searchkick_search: {
+                type: "custom",
+                char_filter: ["ampersand"],
+                tokenizer: "standard",
+                filter: ["standard", "lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"]
+              },
+              searchkick_search2: {
+                type: "custom",
+                char_filter: ["ampersand"],
+                tokenizer: "standard",
+                filter: ["standard", "lowercase", "asciifolding", "searchkick_stemmer"]
+              },
+              # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
+              searchkick_autocomplete_search: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding"]
+              },
+              searchkick_word_search: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding"]
+              },
+              searchkick_suggest_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
+              },
+              searchkick_text_start_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+              },
+              searchkick_text_middle_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+              },
+              searchkick_text_end_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
+              },
+              searchkick_word_start_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+              },
+              searchkick_word_middle_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+              },
+              searchkick_word_end_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
+              }
+            },
+            filter: {
+              searchkick_index_shingle: {
+                type: "shingle",
+                token_separator: ""
+              },
+              # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7
+              searchkick_search_shingle: {
+                type: "shingle",
+                token_separator: "",
+                output_unigrams: false,
+                output_unigrams_if_no_shingles: true
+              },
+              searchkick_suggest_shingle: {
+                type: "shingle",
+                max_shingle_size: 5
+              },
+              searchkick_edge_ngram: {
+                type: "edgeNGram",
+                min_gram: 1,
+                max_gram: 50
+              },
+              searchkick_ngram: {
+                type: "nGram",
+                min_gram: 1,
+                max_gram: 50
+              },
+              searchkick_stemmer: {
+                # use stemmer if language is lowercase, snowball otherwise
+                # TODO deprecate language option in favor of stemmer
+                type: language == language.to_s.downcase ? "stemmer" : "snowball",
+                language: language || "English"
+              }
+            },
+            char_filter: {
+              # https://www.elastic.co/guide/en/elasticsearch/guide/current/custom-analyzers.html
+              # &_to_and
+              ampersand: {
+                type: "mapping",
+                mappings: ["&=> and "]
+              }
+            }
+          }
+        }
+
+        if Searchkick.env == "test"
+          settings[:number_of_shards] = 1
+          settings[:number_of_replicas] = 0
+        end
+
+        if options[:similarity]
+          settings[:similarity] = {default: {type: options[:similarity]}}
+        end
+
+        settings.deep_merge!(options[:settings] || {})
+
+        # synonyms
+        synonyms = options[:synonyms] || []
+
+        synonyms = synonyms.call if synonyms.respond_to?(:call)
+
+        if synonyms.any?
+          settings[:analysis][:filter][:searchkick_synonym] = {
+            type: "synonym",
+            synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.join(",") : s }.map(&:downcase)
+          }
+          # choosing a place for the synonym filter when stemming is not easy
+          # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
+          # TODO use a snowball stemmer on synonyms when creating the token filter
+
+          # http://elasticsearch-users.115913.n3.nabble.com/synonym-multi-words-search-td4030811.html
+          # I find the following approach effective if you are doing multi-word synonyms (synonym phrases):
+          # - Only apply the synonym expansion at index time
+          # - Don't have the synonym filter applied search
+          # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
+          settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_synonym")
+          settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_synonym"
+
+          %w(word_start word_middle word_end).each do |type|
+            settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_synonym")
+          end
+        end
+
+        if options[:wordnet]
+          settings[:analysis][:filter][:searchkick_wordnet] = {
+            type: "synonym",
+            format: "wordnet",
+            synonyms_path: Searchkick.wordnet_path
+          }
+
+          settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_wordnet")
+          settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_wordnet"
+
+          %w(word_start word_middle word_end).each do |type|
+            settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_wordnet")
+          end
+        end
+
+        if options[:special_characters] == false
+          settings[:analysis][:analyzer].each do |_, analyzer_settings|
+            analyzer_settings[:filter].reject! { |f| f == "asciifolding" }
+          end
+        end
+
+        mapping = {}
+
+        # conversions
+        Array(options[:conversions]).each do |conversions_field|
+          mapping[conversions_field] = {
+            type: "nested",
+            properties: {
+              query: {type: default_type, analyzer: "searchkick_keyword"},
+              count: {type: "integer"}
+            }
+          }
+        end
+
+        mapping_options = Hash[
+          [:suggest, :word, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight, :searchable, :filterable]
+            .map { |type| [type, (options[type] || []).map(&:to_s)] }
+        ]
+
+        word = options[:word] != false && (!options[:match] || options[:match] == :word)
+
+        mapping_options[:searchable].delete("_all")
+
+        analyzed_field_options = {type: default_type, index: "analyzed", analyzer: default_analyzer}
+
+        mapping_options.values.flatten.uniq.each do |field|
+          fields = {}
+
+          if options.key?(:filterable) && !mapping_options[:filterable].include?(field)
+            fields[field] = {type: default_type, index: "no"}
+          else
+            fields[field] = keyword_mapping
+          end
+
+          if !options[:searchable] || mapping_options[:searchable].include?(field)
+            if word
+              fields["analyzed"] = analyzed_field_options
+
+              if mapping_options[:highlight].include?(field)
+                fields["analyzed"][:term_vector] = "with_positions_offsets"
+              end
+            end
+
+            mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f|
+              if options[:match] == type || f.include?(field)
+                fields[type] = {type: default_type, index: "analyzed", analyzer: "searchkick_#{type}_index"}
+              end
+            end
+          end
+
+          mapping[field] = fields[field].merge(fields: fields.except(field))
+        end
+
+        (options[:locations] || []).map(&:to_s).each do |field|
+          mapping[field] = {
+            type: "geo_point"
+          }
+        end
+
+        options[:geo_shape] = options[:geo_shape].product([{}]).to_h if options[:geo_shape].is_a?(Array)
+        (options[:geo_shape] || {}).each do |field, shape_options|
+          mapping[field] = shape_options.merge(type: "geo_shape")
+        end
+
+        routing = {}
+        if options[:routing]
+          routing = {required: true}
+          unless options[:routing] == true
+            routing[:path] = options[:routing].to_s
+          end
+        end
+
+        dynamic_fields = {
+          # analyzed field must be the default field for include_in_all
+          # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
+          # however, we can include the not_analyzed field in _all
+          # and the _all index analyzer will take care of it
+          "{name}" => keyword_mapping.merge(include_in_all: !options[:searchable])
+        }
+
+        if options.key?(:filterable)
+          dynamic_fields["{name}"] = {type: default_type, index: "no"}
+        end
+
+        unless options[:searchable]
+          if options[:match] && options[:match] != :word
+            dynamic_fields[options[:match]] = {type: default_type, index: "analyzed", analyzer: "searchkick_#{options[:match]}_index"}
+          end
+
+          if word
+            dynamic_fields["analyzed"] = analyzed_field_options
+          end
+        end
+
+        # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
+        multi_field = dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}"))
+
+        all_enabled = !options[:searchable] || options[:searchable].to_a.map(&:to_s).include?("_all")
+
+        mappings = {
+          _default_: {
+            _all: all_enabled ? analyzed_field_options : {enabled: false},
+            properties: mapping,
+            _routing: routing,
+            # https://gist.github.com/kimchy/2898285
+            dynamic_templates: [
+              {
+                string_template: {
+                  match: "*",
+                  match_mapping_type: "string",
+                  mapping: multi_field
+                }
+              }
+            ]
+          }
+        }.deep_merge(options[:mappings] || {})
+      end
+
+      {
+        settings: settings,
+        mappings: mappings
+      }
+    end
+  end
+end

data/lib/searchkick/indexer.rb

@@ -0,0 +1,28 @@
+module Searchkick
+  class Indexer
+    attr_reader :queued_items
+
+    def initialize
+      @queued_items = []
+    end
+
+    def queue(items)
+      @queued_items.concat(items)
+      perform unless Searchkick.callbacks_value == :bulk
+    end
+
+    def perform
+      items = @queued_items
+      @queued_items = []
+      if items.any?
+        response = Searchkick.client.bulk(body: items)
+        if response["errors"]
+          first_with_error = response["items"].map do |item|
+            (item["index"] || item["delete"] || item["update"])
+          end.find { |item| item["error"] }
+          raise Searchkick::ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
+        end
+      end
+    end
+  end
+end

data/lib/searchkick/logging.rb

@@ -0,0 +1,242 @@
+# based on https://gist.github.com/mnutt/566725
+require "active_support/core_ext/module/attr_internal"
+
+module Searchkick
+  module QueryWithInstrumentation
+    def execute_search
+      name = searchkick_klass ? "#{searchkick_klass.name} Search" : "Search"
+      event = {
+        name: name,
+        query: params
+      }
+      ActiveSupport::Notifications.instrument("search.searchkick", event) do
+        super
+      end
+    end
+  end
+
+  module IndexWithInstrumentation
+    def store(record)
+      event = {
+        name: "#{record.searchkick_klass.name} Store",
+        id: search_id(record)
+      }
+      if Searchkick.callbacks_value == :bulk
+        super
+      else
+        ActiveSupport::Notifications.instrument("request.searchkick", event) do
+          super
+        end
+      end
+    end
+
+    def remove(record)
+      name = record && record.searchkick_klass ? "#{record.searchkick_klass.name} Remove" : "Remove"
+      event = {
+        name: name,
+        id: search_id(record)
+      }
+      if Searchkick.callbacks_value == :bulk
+        super
+      else
+        ActiveSupport::Notifications.instrument("request.searchkick", event) do
+          super
+        end
+      end
+    end
+
+    def update_record(record, method_name)
+      event = {
+        name: "#{record.searchkick_klass.name} Update",
+        id: search_id(record)
+      }
+      if Searchkick.callbacks_value == :bulk
+        super
+      else
+        ActiveSupport::Notifications.instrument("request.searchkick", event) do
+          super
+        end
+      end
+    end
+
+    def bulk_index(records)
+      if records.any?
+        event = {
+          name: "#{records.first.searchkick_klass.name} Import",
+          count: records.size
+        }
+        event[:id] = search_id(records.first) if records.size == 1
+        if Searchkick.callbacks_value == :bulk
+          super
+        else
+          ActiveSupport::Notifications.instrument("request.searchkick", event) do
+            super
+          end
+        end
+      end
+    end
+    alias_method :import, :bulk_index
+
+    def bulk_update(records, *args)
+      if records.any?
+        event = {
+          name: "#{records.first.searchkick_klass.name} Update",
+          count: records.size
+        }
+        event[:id] = search_id(records.first) if records.size == 1
+        if Searchkick.callbacks_value == :bulk
+          super
+        else
+          ActiveSupport::Notifications.instrument("request.searchkick", event) do
+            super
+          end
+        end
+      end
+    end
+
+    def bulk_delete(records)
+      if records.any?
+        event = {
+          name: "#{records.first.searchkick_klass.name} Delete",
+          count: records.size
+        }
+        event[:id] = search_id(records.first) if records.size == 1
+        if Searchkick.callbacks_value == :bulk
+          super
+        else
+          ActiveSupport::Notifications.instrument("request.searchkick", event) do
+            super
+          end
+        end
+      end
+    end
+  end
+
+  module IndexerWithInstrumentation
+    def perform
+      if Searchkick.callbacks_value == :bulk
+        event = {
+          name: "Bulk",
+          count: queued_items.size
+        }
+        ActiveSupport::Notifications.instrument("request.searchkick", event) do
+          super
+        end
+      else
+        super
+      end
+    end
+  end
+
+  module SearchkickWithInstrumentation
+    def multi_search(searches)
+      event = {
+        name: "Multi Search",
+        body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join
+      }
+      ActiveSupport::Notifications.instrument("multi_search.searchkick", event) do
+        super
+      end
+    end
+  end
+
+  # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/log_subscriber.rb
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def self.runtime=(value)
+      Thread.current[:searchkick_runtime] = value
+    end
+
+    def self.runtime
+      Thread.current[:searchkick_runtime] ||= 0
+    end
+
+    def self.reset_runtime
+      rt = runtime
+      self.runtime = 0
+      rt
+    end
+
+    def search(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+      type = payload[:query][:type]
+      index = payload[:query][:index].is_a?(Array) ? payload[:query][:index].join(",") : payload[:query][:index]
+
+      # no easy way to tell which host the client will use
+      host = Searchkick.client.transport.hosts.first
+      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{payload[:query][:body].to_json}'"
+    end
+
+    def request(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      debug "  #{color(name, YELLOW, true)}  #{payload.except(:name).to_json}"
+    end
+
+    def multi_search(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      # no easy way to tell which host the client will use
+      host = Searchkick.client.transport.hosts.first
+      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -d '#{payload[:body]}'"
+    end
+  end
+
+  # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/railties/controller_runtime.rb
+  module ControllerRuntime
+    extend ActiveSupport::Concern
+
+    protected
+
+    attr_internal :searchkick_runtime
+
+    def process_action(action, *args)
+      # We also need to reset the runtime before each action
+      # because of queries in middleware or in cases we are streaming
+      # and it won't be cleaned up by the method below.
+      Searchkick::LogSubscriber.reset_runtime
+      super
+    end
+
+    def cleanup_view_runtime
+      searchkick_rt_before_render = Searchkick::LogSubscriber.reset_runtime
+      runtime = super
+      searchkick_rt_after_render = Searchkick::LogSubscriber.reset_runtime
+      self.searchkick_runtime = searchkick_rt_before_render + searchkick_rt_after_render
+      runtime - searchkick_rt_after_render
+    end
+
+    def append_info_to_payload(payload)
+      super
+      payload[:searchkick_runtime] = (searchkick_runtime || 0) + Searchkick::LogSubscriber.reset_runtime
+    end
+
+    module ClassMethods
+      def log_process_action(payload)
+        messages = super
+        runtime = payload[:searchkick_runtime]
+        messages << ("Searchkick: %.1fms" % runtime.to_f) if runtime.to_f > 0
+        messages
+      end
+    end
+  end
+end
+Searchkick::Query.send(:prepend, Searchkick::QueryWithInstrumentation)
+Searchkick::Index.send(:prepend, Searchkick::IndexWithInstrumentation)
+Searchkick::Indexer.send(:prepend, Searchkick::IndexerWithInstrumentation)
+Searchkick.singleton_class.send(:prepend, Searchkick::SearchkickWithInstrumentation)
+Searchkick::LogSubscriber.attach_to :searchkick
+ActiveSupport.on_load(:action_controller) do
+  include Searchkick::ControllerRuntime
+end

data/lib/searchkick/middleware.rb

@@ -0,0 +1,12 @@
+require "faraday/middleware"
+
+module Searchkick
+  class Middleware < Faraday::Middleware
+    def call(env)
+      if env[:method] == :get && env[:url].path.to_s.end_with?("/_search")
+        env[:request][:timeout] = Searchkick.search_timeout
+      end
+      @app.call(env)
+    end
+  end
+end