searchkick 5.3.1 → 5.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8646c1e2b5f827947bc43c6d9f3761a3050007d67c8d051a3e70014700148c17
-  data.tar.gz: 23d8ab0656e1f2cbaf11aeffedd99ed3b9991271248c2bddc5bf6150cfdfff2c
+  metadata.gz: 5dfa66d383b0e1a91288b4636fef42b22ac73ec63294a7955ac20298851df1c1
+  data.tar.gz: fbe59c3e85352b01674c16831f67b3906367e6a032769582ee552038853194f8
 SHA512:
-  metadata.gz: 13fa0dd9d4c0fa8f77368403b0a9f717eaf5bcfd714022bcdd23400161ef8939baf7803f3a0773cf6848b6deea9ad46b9e9a67e50871cab07911e4a16f171775
-  data.tar.gz: bb2da50a0761be5e90074896c8c020aa7568a3483579bfa0743101199ef20b370b7b0cf3235818c86e69b6e4438cf05e67643a1a192715b5c952253411912dab
+  metadata.gz: 25d7df6cf3861522a99851c8c561f0726a6e922d4aafb0fe646883b86c37ba4c1ec78735940a5eb70056433422fa2987989701260ec7b1e2969fb452758d2d43
+  data.tar.gz: d9b637bba6c90e1c08de0b9cd8239e6b8b03aa86eb0a77aebc6b855ff0744220d775c8b7c8a12e1df736f98ee44dac3c6f11ad3a0506aa51e5c22f120f3c13d3

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 5.4.0 (2024-09-04)
+
+- Added `knn` option
+- Added `rrf` method
+- Added experimental support for scripting to `where` option
+- Added warning for `exists` with non-`true` values
+- Added warning for full reindex and `:queue` mode
+- Fixed `per_page` method when paginating beyond `max_result_window`
+- Dropped support for Ruby < 3.1
+
 ## 5.3.1 (2023-11-28)
 
 - Fixed error with misspellings below and failed queries

data/README.md

@@ -26,7 +26,7 @@ Check out [Searchjoy](https://github.com/ankane/searchjoy) for analytics and [Au
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
-[![Build Status](https://github.com/ankane/searchkick/workflows/build/badge.svg?branch=master)](https://github.com/ankane/searchkick/actions)
+[![Build Status](https://github.com/ankane/searchkick/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/searchkick/actions)
 
 ## Contents
 
@@ -120,9 +120,9 @@ where: {
   category: /frozen .+/,         # regexp
   category: {prefix: "frozen"},  # prefix
   store_id: {exists: true},      # exists
+  _not: {store_id: 1},           # negate a condition
   _or: [{in_stock: true}, {backordered: true}],
-  _and: [{in_stock: true}, {backordered: true}],
-  _not: {store_id: 1}            # negate a condition
+  _and: [{in_stock: true}, {backordered: true}]
 }
 ```
 
@@ -815,7 +815,7 @@ Product.search("milk", boost_where: {orderer_ids: current_user.id})
 
 Autocomplete predicts what a user will type, making the search experience faster and easier.
 
-![Autocomplete](https://gist.github.com/ankane/b6988db2802aca68a589b31e41b44195/raw/40febe948427e5bc53ec4e5dc248822855fef76f/autocomplete.png)
+![Autocomplete](https://gist.githubusercontent.com/ankane/b6988db2802aca68a589b31e41b44195/raw/40febe948427e5bc53ec4e5dc248822855fef76f/autocomplete.png)
 
 **Note:** To autocomplete on search terms rather than results, check out [Autosuggest](https://github.com/ankane/autosuggest).
 
@@ -881,7 +881,7 @@ Then add the search box and JavaScript code to a view.
 
 ## Suggestions
 
-![Suggest](https://gist.github.com/ankane/b6988db2802aca68a589b31e41b44195/raw/40febe948427e5bc53ec4e5dc248822855fef76f/recursion.png)
+![Suggest](https://gist.githubusercontent.com/ankane/b6988db2802aca68a589b31e41b44195/raw/40febe948427e5bc53ec4e5dc248822855fef76f/recursion.png)
 
 ```ruby
 class Product < ApplicationRecord
@@ -900,7 +900,7 @@ products.suggestions # ["peanut butter"]
 
 [Aggregations](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html) provide aggregated search data.
 
-![Aggregations](https://gist.github.com/ankane/b6988db2802aca68a589b31e41b44195/raw/40febe948427e5bc53ec4e5dc248822855fef76f/facets.png)
+![Aggregations](https://gist.githubusercontent.com/ankane/b6988db2802aca68a589b31e41b44195/raw/40febe948427e5bc53ec4e5dc248822855fef76f/facets.png)
 
 ```ruby
 products = Product.search("chuck taylor", aggs: [:product_type, :gender, :brand])
@@ -1845,9 +1845,86 @@ To query nested data, use dot notation.
 Product.search("san", fields: ["store.city"], where: {"store.zip_code" => 12345})
 ```
 
-## Nearest Neighbors
+## Nearest Neighbor Search
 
-You can use custom mapping and searching to index vectors and perform k-nearest neighbor search. See the examples for [Elasticsearch](examples/elasticsearch_knn.rb) and [OpenSearch](examples/opensearch_knn.rb).
+*Available for Elasticsearch 8.6+ and OpenSearch 2.4+*
+
+```ruby
+class Product < ApplicationRecord
+  searchkick knn: {embedding: {dimensions: 3, distance: "cosine"}}
+end
+```
+
+Also supports `euclidean` and `inner_product`
+
+Reindex and search with:
+
+```ruby
+Product.search(knn: {field: :embedding, vector: [1, 2, 3]}, limit: 10)
+```
+
+## Semantic Search
+
+First, add [nearest neighbor search](#nearest-neighbor-search-unreleased-experimental) to your model
+
+```ruby
+class Product < ApplicationRecord
+  searchkick knn: {embedding: {dimensions: 768, distance: "cosine"}}
+end
+```
+
+Generate an embedding for each record (you can use an external service or a library like [Informers](https://github.com/ankane/informers))
+
+```ruby
+embed = Informers.pipeline("embedding", "Snowflake/snowflake-arctic-embed-m-v1.5")
+embed_options = {model_output: "sentence_embedding", pooling: "none"} # specific to embedding model
+
+Product.find_each do |product|
+  embedding = embed.(product.name, **embed_options)
+  product.update!(embedding: embedding)
+end
+```
+
+For search, generate an embedding for the query (the query prefix is specific to the [embedding model](https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5))
+
+```ruby
+query_prefix = "Represent this sentence for searching relevant passages: "
+query_embedding = embed.(query_prefix + query, **embed_options)
+```
+
+And perform nearest neighbor search
+
+```ruby
+Product.search(knn: {field: :embedding, vector: query_embedding}, limit: 20)
+```
+
+See a [full example](examples/semantic.rb)
+
+## Hybrid Search
+
+Perform keyword search and semantic search in parallel
+
+```ruby
+keyword_search = Product.search(query, limit: 20)
+semantic_search = Product.search(knn: {field: :embedding, vector: query_embedding}, limit: 20)
+Searchkick.multi_search([keyword_search, semantic_search])
+```
+
+To combine the results, use Reciprocal Rank Fusion (RRF)
+
+```ruby
+Searchkick::Reranking.rrf(keyword_search, semantic_search).first(5)
+```
+
+Or a reranking model
+
+```ruby
+rerank = Informers.pipeline("reranking", "mixedbread-ai/mxbai-rerank-xsmall-v1")
+results = (keyword_search.to_a + semantic_search.to_a).uniq
+rerank.(query, results.map(&:name)).first(5).map { |v| results[v[:doc_id]] }
+```
+
+See a [full example](examples/hybrid.rb)
 
 ## Reference
 

data/lib/searchkick/index.rb

@@ -99,7 +99,7 @@ module Searchkick
       record_data = RecordData.new(self, record).record_data
 
       # remove underscore
-      get_options = record_data.to_h { |k, v| [k.to_s.sub(/\A_/, "").to_sym, v] }
+      get_options = record_data.to_h { |k, v| [k.to_s.delete_prefix("_").to_sym, v] }
 
       client.get(get_options)["_source"]
     end
@@ -352,6 +352,8 @@ module Searchkick
     # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
     def full_reindex(relation, import: true, resume: false, retain: false, mode: nil, refresh_interval: nil, scope: nil, wait: nil)
       raise ArgumentError, "wait only available in :async mode" if !wait.nil? && mode != :async
+      # TODO raise ArgumentError in Searchkick 6
+      Searchkick.warn("Full reindex does not support :queue mode - use :async mode instead") if mode == :queue
 
       if resume
         index_name = all_indices.sort.last

data/lib/searchkick/index_options.rb

@@ -169,6 +169,21 @@ module Searchkick
         max_shingle_diff: 4
       }
 
+      if options[:knn]
+        unless Searchkick.knn_support?
+          if Searchkick.opensearch?
+            raise Error, "knn requires OpenSearch 2.4+"
+          else
+            raise Error, "knn requires Elasticsearch 8.6+"
+          end
+        end
+
+        if Searchkick.opensearch? && options[:knn].any? { |_, v| !v[:distance].nil? }
+          # only enable if doing approximate search
+          settings[:index][:knn] = true
+        end
+      end
+
       if options[:case_sensitive]
         settings[:analysis][:analyzer].each do |_, analyzer|
           analyzer[:filter].delete("lowercase")
@@ -406,6 +421,66 @@ module Searchkick
         mapping[field] = shape_options.merge(type: "geo_shape")
       end
 
+      (options[:knn] || []).each do |field, knn_options|
+        distance = knn_options[:distance]
+
+        if Searchkick.opensearch?
+          if distance.nil?
+            # avoid server crash if method not specified
+            raise ArgumentError, "Must specify a distance for OpenSearch"
+          end
+
+          vector_options = {
+            type: "knn_vector",
+            dimension: knn_options[:dimensions]
+          }
+
+          if !distance.nil?
+            space_type =
+              case distance
+              when "cosine"
+                "cosinesimil"
+              when "euclidean"
+                "l2"
+              when "inner_product"
+                "innerproduct"
+              else
+                raise ArgumentError, "Unknown distance: #{distance}"
+              end
+
+            vector_options[:method] = {
+              name: "hnsw",
+              space_type: space_type,
+              engine: "lucene"
+            }
+          end
+
+          mapping[field.to_s] = vector_options
+        else
+          vector_options = {
+            type: "dense_vector",
+            dims: knn_options[:dimensions],
+            index: !distance.nil?
+          }
+
+          if !distance.nil?
+            vector_options[:similarity] =
+              case distance
+              when "cosine"
+                "cosine"
+              when "euclidean"
+                "l2_norm"
+              when "inner_product"
+                "max_inner_product"
+              else
+                raise ArgumentError, "Unknown distance: #{distance}"
+              end
+          end
+
+          mapping[field.to_s] = vector_options
+        end
+      end
+
       if options[:inheritance]
         mapping[:type] = keyword_mapping
       end

data/lib/searchkick/model.rb

@@ -4,7 +4,7 @@ module Searchkick
       options = Searchkick.model_options.merge(options)
 
       unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
-        :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
+        :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :knn, :language,
         :locations, :mappings, :match, :max_result_window, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
         :special_characters, :stem, :stemmer, :stem_conversions, :stem_exclusion, :stemmer_override, :suggest, :synonyms, :text_end,
         :text_middle, :text_start, :unscope, :word, :word_end, :word_middle, :word_start]

data/lib/searchkick/query.rb

@@ -9,7 +9,7 @@ module Searchkick
     attr_accessor :body
 
     def_delegators :execute, :map, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary,
-      :records, :results, :suggestions, :each_with_hit, :with_details, :aggregations, :aggs,
+      :results, :suggestions, :each_with_hit, :with_details, :aggregations, :aggs,
       :took, :error, :model_name, :entry_name, :total_count, :total_entries,
       :current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
       :offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
@@ -19,7 +19,7 @@ module Searchkick
     def initialize(klass, term = "*", **options)
       unknown_keywords = options.keys - [:aggs, :block, :body, :body_options, :boost,
         :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :explain,
-        :fields, :highlight, :includes, :index_name, :indices_boost, :limit, :load,
+        :fields, :highlight, :includes, :index_name, :indices_boost, :knn, :limit, :load,
         :match, :misspellings, :models, :model_includes, :offset, :operator, :order, :padding, :page, :per_page, :profile,
         :request_params, :routing, :scope_results, :scroll, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
@@ -219,7 +219,7 @@ module Searchkick
         )
 
           raise UnsupportedVersionError
-        elsif e.message =~ /analyzer \[searchkick_.+\] not found/
+        elsif e.message.match?(/analyzer \[searchkick_.+\] not found/)
           raise InvalidQueryError, "Bad mapping - run #{reindex_command}"
         else
           raise InvalidQueryError, e.message
@@ -259,6 +259,7 @@ module Searchkick
       scroll = options[:scroll]
 
       max_result_window = searchkick_options[:max_result_window]
+      original_per_page = per_page
       if max_result_window
         offset = max_result_window if offset > max_result_window
         per_page = max_result_window - offset if offset + per_page > max_result_window
@@ -387,7 +388,7 @@ module Searchkick
               exclude_field = f
               exclude_analyzer = "keyword"
             else
-              analyzer = field =~ /\.word_(start|middle|end)\z/ ? "searchkick_word_search" : "searchkick_autocomplete_search"
+              analyzer = field.match?(/\.word_(start|middle|end)\z/) ? "searchkick_word_search" : "searchkick_autocomplete_search"
               qs << shared_options.merge(analyzer: analyzer)
               exclude_analyzer = analyzer
             end
@@ -525,6 +526,9 @@ module Searchkick
         end
       end
 
+      # knn
+      set_knn(payload, options[:knn], per_page, offset) if options[:knn]
+
       # pagination
       pagination_options = options[:page] || options[:limit] || options[:per_page] || options[:offset] || options[:padding]
       if !options[:body] || pagination_options
@@ -558,7 +562,7 @@ module Searchkick
 
       @body = payload
       @page = page
-      @per_page = per_page
+      @per_page = original_per_page
       @padding = padding
       @load = load
       @scroll = scroll
@@ -875,6 +879,119 @@ module Searchkick
       end
     end
 
+    def set_knn(payload, knn, per_page, offset)
+      if term != "*"
+        raise ArgumentError, "Use Searchkick.multi_search for hybrid search"
+      end
+
+      field = knn[:field]
+      field_options = searchkick_options.dig(:knn, field.to_sym) || searchkick_options.dig(:knn, field.to_s) || {}
+      vector = knn[:vector]
+      distance = knn[:distance] || field_options[:distance]
+      exact = knn[:exact]
+      exact = field_options[:distance].nil? || distance != field_options[:distance] if exact.nil?
+      k = per_page + offset
+      filter = payload.delete(:query)
+
+      if distance.nil?
+        raise ArgumentError, "distance required"
+      elsif !exact && distance != field_options[:distance]
+        raise ArgumentError, "distance must match searchkick options for approximate search"
+      end
+
+      if Searchkick.opensearch?
+        if exact
+          # https://opensearch.org/docs/latest/search-plugins/knn/knn-score-script/#spaces
+          space_type =
+            case distance
+            when "cosine"
+              "cosinesimil"
+            when "euclidean"
+              "l2"
+            when "taxicab"
+              "l1"
+            when "inner_product"
+              "innerproduct"
+            when "chebyshev"
+              "linf"
+            else
+              raise ArgumentError, "Unknown distance: #{distance}"
+            end
+
+          payload[:query] = {
+            script_score: {
+              query: {
+                bool: {
+                  must: [filter, {exists: {field: field}}]
+                }
+              },
+              script: {
+                source: "knn_score",
+                lang: "knn",
+                params: {
+                  field: field,
+                  query_value: vector,
+                  space_type: space_type
+                }
+              },
+              boost: distance == "cosine" ? 0.5 : 1.0
+            }
+          }
+        else
+          payload[:query] = {
+            knn: {
+              field.to_sym => {
+                vector: vector,
+                k: k,
+                filter: filter
+              }
+            }
+          }
+        end
+      else
+        if exact
+          # https://github.com/elastic/elasticsearch/blob/main/docs/reference/vectors/vector-functions.asciidoc
+          source =
+            case distance
+            when "cosine"
+              "(cosineSimilarity(params.query_vector, params.field) + 1.0) * 0.5"
+            when "euclidean"
+              "double l2 = l2norm(params.query_vector, params.field); 1 / (1 + l2 * l2)"
+            when "taxicab"
+              "1 / (1 + l1norm(params.query_vector, params.field))"
+            when "inner_product"
+              "double dot = dotProduct(params.query_vector, params.field); dot > 0 ? dot + 1 : 1 / (1 - dot)"
+            else
+              raise ArgumentError, "Unknown distance: #{distance}"
+            end
+
+          payload[:query] = {
+            script_score: {
+              query: {
+                bool: {
+                  must: [filter, {exists: {field: field}}]
+                }
+              },
+              script: {
+                source: source,
+                params: {
+                  field: field,
+                  query_vector: vector
+                }
+              }
+            }
+          }
+        else
+          payload[:knn] = {
+            field: field,
+            query_vector: vector,
+            k: k,
+            filter: filter
+          }
+        end
+      end
+    end
+
     def set_post_filters(payload, post_filters)
       payload[:post_filter] = {
         bool: {
@@ -884,7 +1001,8 @@ module Searchkick
     end
 
     def set_order(payload)
-      payload[:sort] = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
+      value = options[:order]
+      payload[:sort] = value.is_a?(Enumerable) ? value : {value => :asc}
     end
 
     # provides *very* basic protection from unfiltered parameters
@@ -908,8 +1026,12 @@ module Searchkick
           filters << {bool: {must_not: where_filters(value)}}
         elsif field == :_and
           filters << {bool: {must: value.map { |or_statement| {bool: {filter: where_filters(or_statement)}} }}}
-        # elsif field == :_script
-        #   filters << {script: {script: {source: value, lang: "painless"}}}
+        elsif field == :_script
+          unless value.is_a?(Script)
+            raise TypeError, "expected Searchkick::Script"
+          end
+
+          filters << {script: {script: {source: value.source, lang: value.lang, params: value.params}}}
         else
           # expand ranges
           if value.is_a?(Range)
@@ -1002,6 +1124,11 @@ module Searchkick
               when :in
                 filters << term_filters(field, op_value)
               when :exists
+                # TODO add support for false in Searchkick 6
+                if op_value != true
+                  # TODO raise error in Searchkick 6
+                  Searchkick.warn("Passing a value other than true to exists is not supported")
+                end
                 filters << {exists: {field: field}}
               else
                 range_query =

data/lib/searchkick/reranking.rb

@@ -0,0 +1,28 @@
+module Searchkick
+  module Reranking
+    def self.rrf(first_ranking, *rankings, k: 60)
+      rankings.unshift(first_ranking)
+      rankings.map!(&:to_ary)
+
+      ranks = []
+      results = []
+      rankings.each do |ranking|
+        ranks << ranking.map.with_index.to_h { |v, i| [v, i + 1] }
+        results.concat(ranking)
+      end
+
+      results =
+        results.uniq.map do |result|
+          score =
+            ranks.sum do |rank|
+              r = rank[result]
+              r ? 1.0 / (k + r) : 0.0
+            end
+
+          {result: result, score: score}
+        end
+
+      results.sort_by { |v| -v[:score] }
+    end
+  end
+end

data/lib/searchkick/script.rb

@@ -0,0 +1,11 @@
+module Searchkick
+  class Script
+    attr_reader :source, :lang, :params
+
+    def initialize(source, lang: "painless", params: {})
+      @source = source
+      @lang = lang
+      @params = params
+    end
+  end
+end

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "5.3.1"
+  VERSION = "5.4.0"
 end

data/lib/searchkick.rb

@@ -25,7 +25,9 @@ require_relative "searchkick/record_data"
 require_relative "searchkick/record_indexer"
 require_relative "searchkick/relation"
 require_relative "searchkick/relation_indexer"
+require_relative "searchkick/reranking"
 require_relative "searchkick/results"
+require_relative "searchkick/script"
 require_relative "searchkick/version"
 require_relative "searchkick/where"
 
@@ -141,6 +143,15 @@ module Searchkick
     Gem::Version.new(server_version.split("-")[0]) < Gem::Version.new(version.split("-")[0])
   end
 
+  # private
+  def self.knn_support?
+    if opensearch?
+      !server_below?("2.4.0", true)
+    else
+      !server_below?("8.6.0")
+    end
+  end
+
   def self.search(term = "*", model: nil, **options, &block)
     options = options.dup
     klass = model
@@ -182,13 +193,20 @@ module Searchkick
     queries = queries.map { |q| q.send(:query) }
     event = {
       name: "Multi Search",
-      body: queries.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join,
+      body: queries.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join
     }
     ActiveSupport::Notifications.instrument("multi_search.searchkick", event) do
       MultiSearch.new(queries).perform
     end
   end
 
+  # script
+
+  # experimental
+  def self.script(source, **options)
+    Script.new(source, **options)
+  end
+
   # callbacks
 
   def self.enable_callbacks

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 5.3.1
+  version: 5.4.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-28 00:00:00.000000000 Z
+date: 2024-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -69,7 +69,9 @@ files:
 - lib/searchkick/reindex_v2_job.rb
 - lib/searchkick/relation.rb
 - lib/searchkick/relation_indexer.rb
+- lib/searchkick/reranking.rb
 - lib/searchkick/results.rb
+- lib/searchkick/script.rb
 - lib/searchkick/version.rb
 - lib/searchkick/where.rb
 - lib/tasks/searchkick.rake
@@ -85,14 +87,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: Intelligent search made easy with Rails and Elasticsearch or OpenSearch