searchkick 5.0.2 → 5.4.0

data/lib/searchkick/index_options.rb

@@ -19,7 +19,7 @@ module Searchkick
         mappings = generate_mappings.deep_symbolize_keys.deep_merge(custom_mappings)
       end
 
-      set_deep_paging(settings) if options[:deep_paging]
+      set_deep_paging(settings) if options[:deep_paging] || options[:max_result_window]
 
       {
         settings: settings,
@@ -169,6 +169,21 @@ module Searchkick
         max_shingle_diff: 4
       }
 
+      if options[:knn]
+        unless Searchkick.knn_support?
+          if Searchkick.opensearch?
+            raise Error, "knn requires OpenSearch 2.4+"
+          else
+            raise Error, "knn requires Elasticsearch 8.6+"
+          end
+        end
+
+        if Searchkick.opensearch? && options[:knn].any? { |_, v| !v[:distance].nil? }
+          # only enable if doing approximate search
+          settings[:index][:knn] = true
+        end
+      end
+
       if options[:case_sensitive]
         settings[:analysis][:analyzer].each do |_, analyzer|
           analyzer[:filter].delete("lowercase")
@@ -406,6 +421,66 @@ module Searchkick
         mapping[field] = shape_options.merge(type: "geo_shape")
       end
 
+      (options[:knn] || []).each do |field, knn_options|
+        distance = knn_options[:distance]
+
+        if Searchkick.opensearch?
+          if distance.nil?
+            # avoid server crash if method not specified
+            raise ArgumentError, "Must specify a distance for OpenSearch"
+          end
+
+          vector_options = {
+            type: "knn_vector",
+            dimension: knn_options[:dimensions]
+          }
+
+          if !distance.nil?
+            space_type =
+              case distance
+              when "cosine"
+                "cosinesimil"
+              when "euclidean"
+                "l2"
+              when "inner_product"
+                "innerproduct"
+              else
+                raise ArgumentError, "Unknown distance: #{distance}"
+              end
+
+            vector_options[:method] = {
+              name: "hnsw",
+              space_type: space_type,
+              engine: "lucene"
+            }
+          end
+
+          mapping[field.to_s] = vector_options
+        else
+          vector_options = {
+            type: "dense_vector",
+            dims: knn_options[:dimensions],
+            index: !distance.nil?
+          }
+
+          if !distance.nil?
+            vector_options[:similarity] =
+              case distance
+              when "cosine"
+                "cosine"
+              when "euclidean"
+                "l2_norm"
+              when "inner_product"
+                "max_inner_product"
+              else
+                raise ArgumentError, "Unknown distance: #{distance}"
+              end
+          end
+
+          mapping[field.to_s] = vector_options
+        end
+      end
+
       if options[:inheritance]
         mapping[:type] = keyword_mapping
       end
@@ -525,7 +600,7 @@ module Searchkick
     def set_deep_paging(settings)
       if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
         settings[:index] ||= {}
-        settings[:index][:max_result_window] = 1_000_000_000
+        settings[:index][:max_result_window] = options[:max_result_window] || 1_000_000_000
       end
     end
 

data/lib/searchkick/log_subscriber.rb

@@ -31,7 +31,7 @@ module Searchkick
         params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
       end
 
-      debug "  #{color(name, YELLOW, true)}  #{index}#{type ? "/#{type.join(',')}" : ''}/_search#{params.any? ? '?' + params.join('&') : nil} #{payload[:query][:body].to_json}"
+      debug "  #{color(name, YELLOW, bold: true)}  #{index}#{type ? "/#{type.join(',')}" : ''}/_search#{params.any? ? '?' + params.join('&') : nil} #{payload[:query][:body].to_json}"
     end
 
     def request(event)
@@ -41,7 +41,7 @@ module Searchkick
       payload = event.payload
       name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
 
-      debug "  #{color(name, YELLOW, true)}  #{payload.except(:name).to_json}"
+      debug "  #{color(name, YELLOW, bold: true)}  #{payload.except(:name).to_json}"
     end
 
     def multi_search(event)
@@ -51,7 +51,7 @@ module Searchkick
       payload = event.payload
       name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
 
-      debug "  #{color(name, YELLOW, true)}  _msearch #{payload[:body]}"
+      debug "  #{color(name, YELLOW, bold: true)}  _msearch #{payload[:body]}"
     end
   end
 end

data/lib/searchkick/middleware.rb

@@ -3,8 +3,15 @@ require "faraday"
 module Searchkick
   class Middleware < Faraday::Middleware
     def call(env)
-      if env[:method] == :get && env[:url].path.to_s.end_with?("/_search")
+      path = env[:url].path.to_s
+      if path.end_with?("/_search")
         env[:request][:timeout] = Searchkick.search_timeout
+      elsif path.end_with?("/_msearch")
+        # assume no concurrent searches for timeout for now
+        searches = env[:request_body].count("\n") / 2
+        # do not allow timeout to exceed Searchkick.timeout
+        timeout = [Searchkick.search_timeout * searches, Searchkick.timeout].min
+        env[:request][:timeout] = timeout
       end
       @app.call(env)
     end

data/lib/searchkick/model.rb

@@ -4,8 +4,8 @@ module Searchkick
       options = Searchkick.model_options.merge(options)
 
       unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
-        :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
-        :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
+        :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :knn, :language,
+        :locations, :mappings, :match, :max_result_window, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
         :special_characters, :stem, :stemmer, :stem_conversions, :stem_exclusion, :stemmer_override, :suggest, :synonyms, :text_end,
         :text_middle, :text_start, :unscope, :word, :word_end, :word_middle, :word_start]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
@@ -66,7 +66,7 @@ module Searchkick
           alias_method Searchkick.search_method_name, :searchkick_search if Searchkick.search_method_name
 
           def searchkick_index(name: nil)
-            index_name = name || searchkick_index_name
+            index_name = name || searchkick_klass.searchkick_index_name
             index_name = index_name.call if index_name.respond_to?(:call)
             index_cache = class_variable_get(:@@searchkick_index_cache)
             index_cache.fetch(index_name) { Searchkick::Index.new(index_name, searchkick_options) }
@@ -88,9 +88,9 @@ module Searchkick
               if options[:index_name]
                 options[:index_name]
               elsif options[:index_prefix].respond_to?(:call)
-                -> { [options[:index_prefix].call, searchkick_klass.model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
+                -> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
               else
-                [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, searchkick_klass.model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
+                [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
               end
             end
           end

data/lib/searchkick/query.rb

@@ -9,7 +9,7 @@ module Searchkick
     attr_accessor :body
 
     def_delegators :execute, :map, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary,
-      :records, :results, :suggestions, :each_with_hit, :with_details, :aggregations, :aggs,
+      :results, :suggestions, :each_with_hit, :with_details, :aggregations, :aggs,
       :took, :error, :model_name, :entry_name, :total_count, :total_entries,
       :current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
       :offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
@@ -19,7 +19,7 @@ module Searchkick
     def initialize(klass, term = "*", **options)
       unknown_keywords = options.keys - [:aggs, :block, :body, :body_options, :boost,
         :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :explain,
-        :fields, :highlight, :includes, :index_name, :indices_boost, :limit, :load,
+        :fields, :highlight, :includes, :index_name, :indices_boost, :knn, :limit, :load,
         :match, :misspellings, :models, :model_includes, :offset, :operator, :order, :padding, :page, :per_page, :profile,
         :request_params, :routing, :scope_results, :scroll, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
@@ -191,7 +191,7 @@ module Searchkick
     end
 
     def retry_misspellings?(response)
-      @misspellings_below && Results.new(searchkick_klass, response).total_count < @misspellings_below
+      @misspellings_below && response["error"].nil? && Results.new(searchkick_klass, response).total_count < @misspellings_below
     end
 
     private
@@ -199,7 +199,11 @@ module Searchkick
     def handle_error(e)
       status_code = e.message[1..3].to_i
       if status_code == 404
-        raise MissingIndexError, "Index missing - run #{reindex_command}"
+        if e.message.include?("No search context found for id")
+          raise MissingIndexError, "No search context found for id"
+        else
+          raise MissingIndexError, "Index missing - run #{reindex_command}"
+        end
       elsif status_code == 500 && (
         e.message.include?("IllegalArgumentException[minimumSimilarity >= 1]") ||
         e.message.include?("No query registered for [multi_match]") ||
@@ -215,7 +219,7 @@ module Searchkick
         )
 
           raise UnsupportedVersionError
-        elsif e.message =~ /analyzer \[searchkick_.+\] not found/
+        elsif e.message.match?(/analyzer \[searchkick_.+\] not found/)
           raise InvalidQueryError, "Bad mapping - run #{reindex_command}"
         else
           raise InvalidQueryError, e.message
@@ -251,9 +255,16 @@ module Searchkick
       default_limit = searchkick_options[:deep_paging] ? 1_000_000_000 : 10_000
       per_page = (options[:limit] || options[:per_page] || default_limit).to_i
       padding = [options[:padding].to_i, 0].max
-      offset = options[:offset] || (page - 1) * per_page + padding
+      offset = (options[:offset] || (page - 1) * per_page + padding).to_i
       scroll = options[:scroll]
 
+      max_result_window = searchkick_options[:max_result_window]
+      original_per_page = per_page
+      if max_result_window
+        offset = max_result_window if offset > max_result_window
+        per_page = max_result_window - offset if offset + per_page > max_result_window
+      end
+
       # model and eager loading
       load = options[:load].nil? ? true : options[:load]
 
@@ -363,7 +374,7 @@ module Searchkick
             field_misspellings = misspellings && (!misspellings_fields || misspellings_fields.include?(base_field(field)))
 
             if field == "_all" || field.end_with?(".analyzed")
-              shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false || (!below73? && !track_total_hits?)
+              shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false || (!below73? && !track_total_hits?) || match_type == :match_phrase || !below80? || Searchkick.opensearch?
               qs << shared_options.merge(analyzer: "searchkick_search")
 
               # searchkick_search and searchkick_search2 are the same for some languages
@@ -377,7 +388,7 @@ module Searchkick
               exclude_field = f
               exclude_analyzer = "keyword"
             else
-              analyzer = field =~ /\.word_(start|middle|end)\z/ ? "searchkick_word_search" : "searchkick_autocomplete_search"
+              analyzer = field.match?(/\.word_(start|middle|end)\z/) ? "searchkick_word_search" : "searchkick_autocomplete_search"
               qs << shared_options.merge(analyzer: analyzer)
               exclude_analyzer = analyzer
             end
@@ -499,7 +510,7 @@ module Searchkick
         set_highlights(payload, fields) if options[:highlight]
 
         # timeout shortly after client times out
-        payload[:timeout] ||= "#{Searchkick.search_timeout + 1}s"
+        payload[:timeout] ||= "#{((Searchkick.search_timeout + 1) * 1000).round}ms"
 
         # An empty array will cause only the _id and _type for each hit to be returned
         # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html
@@ -515,6 +526,9 @@ module Searchkick
         end
       end
 
+      # knn
+      set_knn(payload, options[:knn], per_page, offset) if options[:knn]
+
       # pagination
       pagination_options = options[:page] || options[:limit] || options[:per_page] || options[:offset] || options[:padding]
       if !options[:body] || pagination_options
@@ -548,7 +562,7 @@ module Searchkick
 
       @body = payload
       @page = page
-      @per_page = per_page
+      @per_page = original_per_page
       @padding = padding
       @load = load
       @scroll = scroll
@@ -865,6 +879,119 @@ module Searchkick
       end
     end
 
+    def set_knn(payload, knn, per_page, offset)
+      if term != "*"
+        raise ArgumentError, "Use Searchkick.multi_search for hybrid search"
+      end
+
+      field = knn[:field]
+      field_options = searchkick_options.dig(:knn, field.to_sym) || searchkick_options.dig(:knn, field.to_s) || {}
+      vector = knn[:vector]
+      distance = knn[:distance] || field_options[:distance]
+      exact = knn[:exact]
+      exact = field_options[:distance].nil? || distance != field_options[:distance] if exact.nil?
+      k = per_page + offset
+      filter = payload.delete(:query)
+
+      if distance.nil?
+        raise ArgumentError, "distance required"
+      elsif !exact && distance != field_options[:distance]
+        raise ArgumentError, "distance must match searchkick options for approximate search"
+      end
+
+      if Searchkick.opensearch?
+        if exact
+          # https://opensearch.org/docs/latest/search-plugins/knn/knn-score-script/#spaces
+          space_type =
+            case distance
+            when "cosine"
+              "cosinesimil"
+            when "euclidean"
+              "l2"
+            when "taxicab"
+              "l1"
+            when "inner_product"
+              "innerproduct"
+            when "chebyshev"
+              "linf"
+            else
+              raise ArgumentError, "Unknown distance: #{distance}"
+            end
+
+          payload[:query] = {
+            script_score: {
+              query: {
+                bool: {
+                  must: [filter, {exists: {field: field}}]
+                }
+              },
+              script: {
+                source: "knn_score",
+                lang: "knn",
+                params: {
+                  field: field,
+                  query_value: vector,
+                  space_type: space_type
+                }
+              },
+              boost: distance == "cosine" ? 0.5 : 1.0
+            }
+          }
+        else
+          payload[:query] = {
+            knn: {
+              field.to_sym => {
+                vector: vector,
+                k: k,
+                filter: filter
+              }
+            }
+          }
+        end
+      else
+        if exact
+          # https://github.com/elastic/elasticsearch/blob/main/docs/reference/vectors/vector-functions.asciidoc
+          source =
+            case distance
+            when "cosine"
+              "(cosineSimilarity(params.query_vector, params.field) + 1.0) * 0.5"
+            when "euclidean"
+              "double l2 = l2norm(params.query_vector, params.field); 1 / (1 + l2 * l2)"
+            when "taxicab"
+              "1 / (1 + l1norm(params.query_vector, params.field))"
+            when "inner_product"
+              "double dot = dotProduct(params.query_vector, params.field); dot > 0 ? dot + 1 : 1 / (1 - dot)"
+            else
+              raise ArgumentError, "Unknown distance: #{distance}"
+            end
+
+          payload[:query] = {
+            script_score: {
+              query: {
+                bool: {
+                  must: [filter, {exists: {field: field}}]
+                }
+              },
+              script: {
+                source: source,
+                params: {
+                  field: field,
+                  query_vector: vector
+                }
+              }
+            }
+          }
+        else
+          payload[:knn] = {
+            field: field,
+            query_vector: vector,
+            k: k,
+            filter: filter
+          }
+        end
+      end
+    end
+
     def set_post_filters(payload, post_filters)
       payload[:post_filter] = {
         bool: {
@@ -874,7 +1001,8 @@ module Searchkick
     end
 
     def set_order(payload)
-      payload[:sort] = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
+      value = options[:order]
+      payload[:sort] = value.is_a?(Enumerable) ? value : {value => :asc}
     end
 
     # provides *very* basic protection from unfiltered parameters
@@ -898,8 +1026,12 @@ module Searchkick
           filters << {bool: {must_not: where_filters(value)}}
         elsif field == :_and
           filters << {bool: {must: value.map { |or_statement| {bool: {filter: where_filters(or_statement)}} }}}
-        # elsif field == :_script
-        #   filters << {script: {script: {source: value, lang: "painless"}}}
+        elsif field == :_script
+          unless value.is_a?(Script)
+            raise TypeError, "expected Searchkick::Script"
+          end
+
+          filters << {script: {script: {source: value.source, lang: value.lang, params: value.params}}}
         else
           # expand ranges
           if value.is_a?(Range)
@@ -992,6 +1124,11 @@ module Searchkick
               when :in
                 filters << term_filters(field, op_value)
               when :exists
+                # TODO add support for false in Searchkick 6
+                if op_value != true
+                  # TODO raise error in Searchkick 6
+                  Searchkick.warn("Passing a value other than true to exists is not supported")
+                end
                 filters << {exists: {field: field}}
               else
                 range_query =
@@ -1160,5 +1297,9 @@ module Searchkick
     def below710?
       Searchkick.server_below?("7.10.0")
     end
+
+    def below80?
+      Searchkick.server_below?("8.0.0")
+    end
   end
 end

data/lib/searchkick/reindex_queue.rb

@@ -10,7 +10,7 @@ module Searchkick
 
     # supports single and multiple ids
     def push(record_ids)
-      Searchkick.with_redis { |r| r.lpush(redis_key, record_ids) }
+      Searchkick.with_redis { |r| r.call("LPUSH", redis_key, record_ids) }
     end
 
     def push_records(records)
@@ -34,11 +34,11 @@ module Searchkick
     # TODO use reliable queuing
     def reserve(limit: 1000)
       if supports_rpop_with_count?
-        Searchkick.with_redis { |r| r.call("rpop", redis_key, limit) }.to_a
+        Searchkick.with_redis { |r| r.call("RPOP", redis_key, limit) }.to_a
       else
         record_ids = []
         Searchkick.with_redis do |r|
-          while record_ids.size < limit && (record_id = r.rpop(redis_key))
+          while record_ids.size < limit && (record_id = r.call("RPOP", redis_key))
             record_ids << record_id
           end
         end
@@ -47,11 +47,11 @@ module Searchkick
     end
 
     def clear
-      Searchkick.with_redis { |r| r.del(redis_key) }
+      Searchkick.with_redis { |r| r.call("DEL", redis_key) }
     end
 
     def length
-      Searchkick.with_redis { |r| r.llen(redis_key) }
+      Searchkick.with_redis { |r| r.call("LLEN", redis_key) }
     end
 
     private
@@ -65,11 +65,16 @@ module Searchkick
     end
 
     def redis_version
-      @redis_version ||= Searchkick.with_redis { |r| Gem::Version.new(r.info["redis_version"]) }
+      @redis_version ||=
+        Searchkick.with_redis do |r|
+          info = r.call("INFO")
+          matches = /redis_version:(\S+)/.match(info)
+          Gem::Version.new(matches[1])
+        end
     end
 
     def escape(value)
-      value.gsub("|", "||")
+      value.to_s.gsub("|", "||")
     end
   end
 end