searchkick 5.0.2 → 5.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ebb326348913a8532f1f4e7771bcdb57dda1abe4cc668e4f5bf8fe44bdbc85b
-  data.tar.gz: 57b63c4444b9dbe26bb8decc4ab4dca81e4e037e6642d5728b96549aea0b2131
+  metadata.gz: 18ce029db59c59e668c7c5f4d1e05702d9631f307736251856658eb9b6a20618
+  data.tar.gz: c4f72347ed409d994bb44d16a48da881165ce825e0dc00192a7acba2b687318f
 SHA512:
-  metadata.gz: f901dfae13328bb168e60cc72559ee2fd624f801456fee1628272c83ed402c4d00a83d3cac2ba978e6ce67c76853ca2dc5c1c0048b11ac34d74bb2b18bc3b974
-  data.tar.gz: '086eb84edef491f27c254cb09ee5bce9ec07bcec46f79b2ca65404041e67bfccd877e30e92e6393ca2117c353ec71e8c632c50615c71e481e89c30c071934dd5'
+  metadata.gz: b17985dfd054b0fe5579ea1057ab9f052a7dbf54c091628ade76fb86616b7c938ea616f3d0367d486e63d66d037b86866543ae902350c24f9b63f25e079e3859
+  data.tar.gz: b576abbc0681e466638164b9b92c8edc5e91c771fd3c538d5b4f8c07597c939be882d4dc9b3cb89e8e43ae913804960c93fdfa2ab8eab1b1cee9566f86adcbd5

data/CHANGELOG.md

@@ -1,3 +1,61 @@
+## 5.3.0 (2023-07-02)
+
+- Fixed error with `cutoff_frequency`
+- Dropped support for Ruby < 3 and Active Record < 6.1
+- Dropped support for Mongoid < 7
+
+## 5.2.4 (2023-05-11)
+
+- Fixed error with non-string routing and `:async` mode
+
+## 5.2.3 (2023-04-12)
+
+- Fixed error with missing records and multiple models
+
+## 5.2.2 (2023-04-01)
+
+- Fixed `total_docs` method
+- Fixed deprecation warning with Active Support 7.1
+
+## 5.2.1 (2023-02-21)
+
+- Added support for `redis-client` gem
+
+## 5.2.0 (2023-02-08)
+
+- Added model name to warning about missing records
+- Fixed unnecessary data loading when reindexing relations with `:async` and `:queue` modes
+
+## 5.1.2 (2023-01-29)
+
+- Fixed error with missing point in time
+
+## 5.1.1 (2022-12-05)
+
+- Added support for strings for `offset` and `per_page`
+
+## 5.1.0 (2022-10-12)
+
+- Added support for fractional search timeout
+- Fixed search timeout with `elasticsearch` 8+ and `opensearch-ruby` gems
+- Fixed search timeout not applying to `multi_search`
+
+## 5.0.5 (2022-10-09)
+
+- Added `model` method to `Searchkick::Relation`
+- Fixed deprecation warning with `redis` gem
+- Fixed `respond_to?` method on relation loading relation
+- Fixed `Relation loaded` error for non-mutating methods on relation
+
+## 5.0.4 (2022-06-16)
+
+- Added `max_result_window` option
+- Improved error message for unsupported versions of Elasticsearch
+
+## 5.0.3 (2022-03-13)
+
+- Fixed context for index name for inherited models
+
 ## 5.0.2 (2022-03-03)
 
 - Fixed index name for inherited models

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013-2021 Andrew Kane
+Copyright (c) 2013-2023 Andrew Kane
 
 MIT License
 

data/README.md

@@ -43,15 +43,13 @@ Check out [Searchjoy](https://github.com/ankane/searchjoy) for analytics and [Au
 - [Reference](#reference)
 - [Contributing](#contributing)
 
-Searchkick 5.0 was recently released! See [how to upgrade](#upgrading)
-
 ## Getting Started
 
 Install [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) or [OpenSearch](https://opensearch.org/downloads.html). For Homebrew, use:
 
 ```sh
-brew install elasticsearch
-brew services start elasticsearch
+brew install elastic/tap/elasticsearch-full
+brew services start elasticsearch-full
 # or
 brew install opensearch
 brew services start opensearch
@@ -66,7 +64,7 @@ gem "elasticsearch"   # select one
 gem "opensearch-ruby" # select one
 ```
 
-The latest version works with Elasticsearch 7 and 8 and OpenSearch 1. For Elasticsearch 6, use version 4.6.3 and [this readme](https://github.com/ankane/searchkick/blob/v4.6.3/README.md).
+The latest version works with Elasticsearch 7 and 8 and OpenSearch 1 and 2. For Elasticsearch 6, use version 4.6.3 and [this readme](https://github.com/ankane/searchkick/blob/v4.6.3/README.md).
 
 Add searchkick to models you want to search.
 
@@ -292,12 +290,18 @@ Option | Matches | Example
 
 The default is `:word`. The most matches will happen with `:word_middle`.
 
+To specify different matching for different fields, use:
+
+```ruby
+Product.search(query, fields: [{name: :word_start}, {brand: :word_middle}])
+```
+
 ### Exact Matches
 
 To match a field exactly (case-sensitive), use:
 
 ```ruby
-Product.search(query, fields: [{email: :exact}, :name])
+Product.search(query, fields: [{name: :exact}])
 ```
 
 ### Phrase Matches
@@ -323,11 +327,11 @@ end
 See the [list of languages](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-stemmer-tokenfilter.html#analysis-stemmer-tokenfilter-configure-parms). A few languages require plugins:
 
 - `chinese` - [analysis-ik plugin](https://github.com/medcl/elasticsearch-analysis-ik)
-- `chinese2` - [analysis-smartcn plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/7.4/analysis-smartcn.html)
-- `japanese` - [analysis-kuromoji plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/7.4/analysis-kuromoji.html)
+- `chinese2` - [analysis-smartcn plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html)
+- `japanese` - [analysis-kuromoji plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html)
 - `korean` - [analysis-openkoreantext plugin](https://github.com/open-korean-text/elasticsearch-analysis-openkoreantext)
-- `korean2` - [analysis-nori plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/7.4/analysis-nori.html)
-- `polish` - [analysis-stempel plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/7.4/analysis-stempel.html)
+- `korean2` - [analysis-nori plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-nori.html)
+- `polish` - [analysis-stempel plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-stempel.html)
 - `ukrainian` - [analysis-ukrainian plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/7.4/analysis-ukrainian.html)
 - `vietnamese` - [analysis-vietnamese plugin](https://github.com/duydo/elasticsearch-analysis-vietnamese)
 
@@ -592,6 +596,14 @@ There are four strategies for keeping the index synced with your database.
   end
   ```
 
+  And reindex a record or relation manually.
+
+  ```ruby
+  product.reindex
+  # or
+  store.products.reindex(mode: :async)
+  ```
+
 You can also do bulk updates.
 
 ```ruby
@@ -608,6 +620,12 @@ Searchkick.callbacks(false) do
 end
 ```
 
+Or override the model’s strategy.
+
+```ruby
+product.reindex(mode: :async) # :inline or :queue
+```
+
 ### Associations
 
 Data is **not** automatically synced when an association is updated. If this is desired, add a callback to reindex:
@@ -654,20 +672,16 @@ The best starting point to improve your search **by far** is to track searches a
 Product.search("apple", track: {user_id: current_user.id})
 ```
 
-[See the docs](https://github.com/ankane/searchjoy) for how to install and use.
-
-Focus on:
+[See the docs](https://github.com/ankane/searchjoy) for how to install and use. Focus on top searches with a low conversion rate.
 
-- top searches with low conversions
-- top searches with no results
-
-Searchkick can then use the conversion data to learn what users are looking for. If a user searches for “ice cream” and adds Ben & Jerry’s Chunky Monkey to the cart (our conversion metric at Instacart), that item gets a little more weight for similar searches.
+Searchkick can then use the conversion data to learn what users are looking for. If a user searches for “ice cream” and adds Ben & Jerry’s Chunky Monkey to the cart (our conversion metric at Instacart), that item gets a little more weight for similar searches. This can make a huge difference on the quality of your search.
 
 Add conversion data with:
 
 ```ruby
 class Product < ApplicationRecord
-  has_many :searches, class_name: "Searchjoy::Search", as: :convertable
+  has_many :conversions, class_name: "Searchjoy::Conversion", as: :convertable
+  has_many :searches, class_name: "Searchjoy::Search", through: :conversions
 
   searchkick conversions: [:conversions] # name of field
 
@@ -681,15 +695,100 @@ class Product < ApplicationRecord
 end
 ```
 
-Reindex and set up a cron job to add new conversions daily.
+Reindex and set up a cron job to add new conversions daily. For zero downtime deployment, temporarily set `conversions: false` in your search calls until the data is reindexed.
+
+### Performant Conversions
+
+A performant way to do conversions is to cache them to prevent N+1 queries. For Postgres, create a migration with:
 
 ```ruby
-rake searchkick:reindex CLASS=Product
+add_column :products, :search_conversions, :jsonb
 ```
 
-This can make a huge difference on the quality of your search.
+For MySQL, use `:json`, and for others, use `:text` with a [JSON serializer](https://api.rubyonrails.org/classes/ActiveRecord/AttributeMethods/Serialization/ClassMethods.html).
+
+Next, update your model. Create a separate method for conversion data so you can use [partial reindexing](#partial-reindexing).
+
+```ruby
+class Product < ApplicationRecord
+  searchkick conversions: [:conversions]
 
-For a more performant way to reindex conversion data, check out [performant conversions](#performant-conversions).
+  def search_data
+    {
+      name: name,
+      category: category
+    }.merge(conversions_data)
+  end
+
+  def conversions_data
+    {
+      conversions: search_conversions || {}
+    }
+  end
+end
+```
+
+Deploy and reindex your data. For zero downtime deployment, temporarily set `conversions: false` in your search calls until the data is reindexed.
+
+```ruby
+Product.reindex
+```
+
+Then, create a job to update the conversions column and reindex records with new conversions. Here’s one you can use for Searchjoy:
+
+```ruby
+class UpdateConversionsJob < ApplicationJob
+  def perform(class_name, since: nil, update: true, reindex: true)
+    model = Searchkick.load_model(class_name)
+
+    # get records that have a recent conversion
+    recently_converted_ids =
+      Searchjoy::Conversion.where(convertable_type: class_name).where(created_at: since..)
+      .order(:convertable_id).distinct.pluck(:convertable_id)
+
+    # split into batches
+    recently_converted_ids.in_groups_of(1000, false) do |ids|
+      if update
+        # fetch conversions
+        conversions =
+          Searchjoy::Conversion.where(convertable_id: ids, convertable_type: class_name)
+          .joins(:search).where.not(searchjoy_searches: {user_id: nil})
+          .group(:convertable_id, :query).distinct.count(:user_id)
+
+        # group by record
+        conversions_by_record = {}
+        conversions.each do |(id, query), count|
+          (conversions_by_record[id] ||= {})[query] = count
+        end
+
+        # update conversions column
+        model.transaction do
+          conversions_by_record.each do |id, conversions|
+            model.where(id: id).update_all(search_conversions: conversions)
+          end
+        end
+      end
+
+      if reindex
+        # reindex conversions data
+        model.where(id: ids).reindex(:conversions_data)
+      end
+    end
+  end
+end
+```
+
+Run the job:
+
+```ruby
+UpdateConversionsJob.perform_now("Product")
+```
+
+And set it up to run daily.
+
+```ruby
+UpdateConversionsJob.perform_later("Product", since: 1.day.ago)
+```
 
 ## Personalized Results
 
@@ -1011,7 +1110,7 @@ Restaurant.search("soup", where: {bounds: {geo_shape: {type: "polygon", coordina
 Falling entirely within the query shape
 
 ```ruby
-Restaurant.search("salad", where: {bounds: {geo_shape: {type: "circle", relation: "within", coordinates: [{lat: 38, lon: -123}], radius: "1km"}}})
+Restaurant.search("salad", where: {bounds: {geo_shape: {type: "circle", relation: "within", coordinates: {lat: 38, lon: -123}, radius: "1km"}}})
 ```
 
 Not touching the query shape
@@ -1575,11 +1674,12 @@ Reindex a subset of attributes to reduce time spent generating search data and c
 class Product < ApplicationRecord
   def search_data
     {
-      name: name
-    }.merge(search_prices)
+      name: name,
+      category: category
+    }.merge(prices_data)
   end
 
-  def search_prices
+  def prices_data
     {
       price: price,
       sale_price: sale_price
@@ -1591,68 +1691,7 @@ end
 And use:
 
 ```ruby
-Product.reindex(:search_prices)
-```
-
-### Performant Conversions
-
-Split out conversions into a separate method so you can use partial reindexing, and cache conversions to prevent N+1 queries. Be sure to use a centralized cache store like Memcached or Redis.
-
-```ruby
-class Product < ApplicationRecord
-  def search_data
-    {
-      name: name
-    }.merge(search_conversions)
-  end
-
-  def search_conversions
-    {
-      conversions: Rails.cache.read("search_conversions:#{self.class.name}:#{id}") || {}
-    }
-  end
-end
-```
-
-Create a job to update the cache and reindex records with new conversions.
-
-```ruby
-class ReindexConversionsJob < ApplicationJob
-  def perform(class_name)
-    # get records that have a recent conversion
-    recently_converted_ids =
-      Searchjoy::Search.where("convertable_type = ? AND converted_at > ?", class_name, 1.day.ago)
-      .order(:convertable_id).distinct.pluck(:convertable_id)
-
-    # split into groups
-    recently_converted_ids.in_groups_of(1000, false) do |ids|
-      # fetch conversions
-      conversions =
-        Searchjoy::Search.where(convertable_id: ids, convertable_type: class_name)
-        .group(:convertable_id, :query).distinct.count(:user_id)
-
-      # group conversions by record
-      conversions_by_record = {}
-      conversions.each do |(id, query), count|
-        (conversions_by_record[id] ||= {})[query] = count
-      end
-
-      # write to cache
-      conversions_by_record.each do |id, conversions|
-        Rails.cache.write("search_conversions:#{class_name}:#{id}", conversions)
-      end
-
-      # partial reindex
-      class_name.constantize.where(id: ids).reindex(:search_conversions)
-    end
-  end
-end
-```
-
-Run the job with:
-
-```ruby
-ReindexConversionsJob.perform_later("Product")
+Product.reindex(:prices_data)
 ```
 
 ## Advanced
@@ -1798,6 +1837,10 @@ To query nested data, use dot notation.
 Product.search("san", fields: ["store.city"], where: {"store.zip_code" => 12345})
 ```
 
+## Nearest Neighbors
+
+You can use custom mapping and searching to index vectors and perform k-nearest neighbor search. See the examples for [Elasticsearch](examples/elasticsearch_knn.rb) and [OpenSearch](examples/opensearch_knn.rb).
+
 ## Reference
 
 Reindex one record
@@ -2036,12 +2079,24 @@ Turn on misspellings after a certain number of characters
 Product.search("api", misspellings: {prefix_length: 2}) # api, apt, no ahi
 ```
 
-**Note:** With this option, if the query length is the same as `prefix_length`, misspellings are turned off with Elasticsearch 7 and OpenSearch
+**Note:** With this option, if the query length is the same as `prefix_length`, misspellings are turned off with Elasticsearch 7 and OpenSearch 1
 
 ```ruby
 Product.search("ah", misspellings: {prefix_length: 2}) # ah, no aha
 ```
 
+BigDecimal values are indexed as floats by default so they can be used for boosting. Convert them to strings to keep full precision.
+
+```ruby
+class Product < ApplicationRecord
+  def search_data
+    {
+      units: units.to_s("F")
+    }
+  end
+end
+```
+
 ## Gotchas
 
 ### Consistency

data/lib/searchkick/index.rb

@@ -67,7 +67,8 @@ module Searchkick
           index: name,
           body: {
             query: {match_all: {}},
-            size: 0
+            size: 0,
+            track_total_hits: true
           }
         )
 
@@ -418,7 +419,7 @@ module Searchkick
         true
       end
     rescue => e
-      if Searchkick.transport_error?(e) && e.message.include?("No handler for type [text]")
+      if Searchkick.transport_error?(e) && (e.message.include?("No handler for type [text]") || e.message.include?("class java.util.ArrayList cannot be cast to class java.util.Map"))
         raise UnsupportedVersionError
       end
 

data/lib/searchkick/index_options.rb

@@ -19,7 +19,7 @@ module Searchkick
         mappings = generate_mappings.deep_symbolize_keys.deep_merge(custom_mappings)
       end
 
-      set_deep_paging(settings) if options[:deep_paging]
+      set_deep_paging(settings) if options[:deep_paging] || options[:max_result_window]
 
       {
         settings: settings,
@@ -525,7 +525,7 @@ module Searchkick
     def set_deep_paging(settings)
       if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
         settings[:index] ||= {}
-        settings[:index][:max_result_window] = 1_000_000_000
+        settings[:index][:max_result_window] = options[:max_result_window] || 1_000_000_000
       end
     end
 

data/lib/searchkick/log_subscriber.rb

@@ -31,7 +31,7 @@ module Searchkick
         params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
       end
 
-      debug "  #{color(name, YELLOW, true)}  #{index}#{type ? "/#{type.join(',')}" : ''}/_search#{params.any? ? '?' + params.join('&') : nil} #{payload[:query][:body].to_json}"
+      debug "  #{color(name, YELLOW, bold: true)}  #{index}#{type ? "/#{type.join(',')}" : ''}/_search#{params.any? ? '?' + params.join('&') : nil} #{payload[:query][:body].to_json}"
     end
 
     def request(event)
@@ -41,7 +41,7 @@ module Searchkick
       payload = event.payload
       name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
 
-      debug "  #{color(name, YELLOW, true)}  #{payload.except(:name).to_json}"
+      debug "  #{color(name, YELLOW, bold: true)}  #{payload.except(:name).to_json}"
     end
 
     def multi_search(event)
@@ -51,7 +51,7 @@ module Searchkick
       payload = event.payload
       name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
 
-      debug "  #{color(name, YELLOW, true)}  _msearch #{payload[:body]}"
+      debug "  #{color(name, YELLOW, bold: true)}  _msearch #{payload[:body]}"
     end
   end
 end

data/lib/searchkick/middleware.rb

@@ -3,8 +3,15 @@ require "faraday"
 module Searchkick
   class Middleware < Faraday::Middleware
     def call(env)
-      if env[:method] == :get && env[:url].path.to_s.end_with?("/_search")
+      path = env[:url].path.to_s
+      if path.end_with?("/_search")
         env[:request][:timeout] = Searchkick.search_timeout
+      elsif path.end_with?("/_msearch")
+        # assume no concurrent searches for timeout for now
+        searches = env[:request_body].count("\n") / 2
+        # do not allow timeout to exceed Searchkick.timeout
+        timeout = [Searchkick.search_timeout * searches, Searchkick.timeout].min
+        env[:request][:timeout] = timeout
       end
       @app.call(env)
     end

data/lib/searchkick/model.rb

@@ -5,7 +5,7 @@ module Searchkick
 
       unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
         :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
-        :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
+        :locations, :mappings, :match, :max_result_window, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
         :special_characters, :stem, :stemmer, :stem_conversions, :stem_exclusion, :stemmer_override, :suggest, :synonyms, :text_end,
         :text_middle, :text_start, :unscope, :word, :word_end, :word_middle, :word_start]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
@@ -66,7 +66,7 @@ module Searchkick
           alias_method Searchkick.search_method_name, :searchkick_search if Searchkick.search_method_name
 
           def searchkick_index(name: nil)
-            index_name = name || searchkick_index_name
+            index_name = name || searchkick_klass.searchkick_index_name
             index_name = index_name.call if index_name.respond_to?(:call)
             index_cache = class_variable_get(:@@searchkick_index_cache)
             index_cache.fetch(index_name) { Searchkick::Index.new(index_name, searchkick_options) }
@@ -88,9 +88,9 @@ module Searchkick
               if options[:index_name]
                 options[:index_name]
               elsif options[:index_prefix].respond_to?(:call)
-                -> { [options[:index_prefix].call, searchkick_klass.model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
+                -> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
               else
-                [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, searchkick_klass.model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
+                [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
               end
             end
           end

data/lib/searchkick/query.rb

@@ -199,7 +199,11 @@ module Searchkick
     def handle_error(e)
       status_code = e.message[1..3].to_i
       if status_code == 404
-        raise MissingIndexError, "Index missing - run #{reindex_command}"
+        if e.message.include?("No search context found for id")
+          raise MissingIndexError, "No search context found for id"
+        else
+          raise MissingIndexError, "Index missing - run #{reindex_command}"
+        end
       elsif status_code == 500 && (
         e.message.include?("IllegalArgumentException[minimumSimilarity >= 1]") ||
         e.message.include?("No query registered for [multi_match]") ||
@@ -251,9 +255,15 @@ module Searchkick
       default_limit = searchkick_options[:deep_paging] ? 1_000_000_000 : 10_000
       per_page = (options[:limit] || options[:per_page] || default_limit).to_i
       padding = [options[:padding].to_i, 0].max
-      offset = options[:offset] || (page - 1) * per_page + padding
+      offset = (options[:offset] || (page - 1) * per_page + padding).to_i
       scroll = options[:scroll]
 
+      max_result_window = searchkick_options[:max_result_window]
+      if max_result_window
+        offset = max_result_window if offset > max_result_window
+        per_page = max_result_window - offset if offset + per_page > max_result_window
+      end
+
       # model and eager loading
       load = options[:load].nil? ? true : options[:load]
 
@@ -363,7 +373,7 @@ module Searchkick
             field_misspellings = misspellings && (!misspellings_fields || misspellings_fields.include?(base_field(field)))
 
             if field == "_all" || field.end_with?(".analyzed")
-              shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false || (!below73? && !track_total_hits?)
+              shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false || (!below73? && !track_total_hits?) || match_type == :match_phrase || !below80? || Searchkick.opensearch?
               qs << shared_options.merge(analyzer: "searchkick_search")
 
               # searchkick_search and searchkick_search2 are the same for some languages
@@ -499,7 +509,7 @@ module Searchkick
         set_highlights(payload, fields) if options[:highlight]
 
         # timeout shortly after client times out
-        payload[:timeout] ||= "#{Searchkick.search_timeout + 1}s"
+        payload[:timeout] ||= "#{((Searchkick.search_timeout + 1) * 1000).round}ms"
 
         # An empty array will cause only the _id and _type for each hit to be returned
         # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html
@@ -1160,5 +1170,9 @@ module Searchkick
     def below710?
       Searchkick.server_below?("7.10.0")
     end
+
+    def below80?
+      Searchkick.server_below?("8.0.0")
+    end
   end
 end

data/lib/searchkick/reindex_queue.rb

@@ -10,7 +10,7 @@ module Searchkick
 
     # supports single and multiple ids
     def push(record_ids)
-      Searchkick.with_redis { |r| r.lpush(redis_key, record_ids) }
+      Searchkick.with_redis { |r| r.call("LPUSH", redis_key, record_ids) }
     end
 
     def push_records(records)
@@ -34,11 +34,11 @@ module Searchkick
     # TODO use reliable queuing
     def reserve(limit: 1000)
       if supports_rpop_with_count?
-        Searchkick.with_redis { |r| r.call("rpop", redis_key, limit) }.to_a
+        Searchkick.with_redis { |r| r.call("RPOP", redis_key, limit) }.to_a
       else
         record_ids = []
         Searchkick.with_redis do |r|
-          while record_ids.size < limit && (record_id = r.rpop(redis_key))
+          while record_ids.size < limit && (record_id = r.call("RPOP", redis_key))
             record_ids << record_id
           end
         end
@@ -47,11 +47,11 @@ module Searchkick
     end
 
     def clear
-      Searchkick.with_redis { |r| r.del(redis_key) }
+      Searchkick.with_redis { |r| r.call("DEL", redis_key) }
     end
 
     def length
-      Searchkick.with_redis { |r| r.llen(redis_key) }
+      Searchkick.with_redis { |r| r.call("LLEN", redis_key) }
     end
 
     private
@@ -65,11 +65,16 @@ module Searchkick
     end
 
     def redis_version
-      @redis_version ||= Searchkick.with_redis { |r| Gem::Version.new(r.info["redis_version"]) }
+      @redis_version ||=
+        Searchkick.with_redis do |r|
+          info = r.call("INFO")
+          matches = /redis_version:(\S+)/.match(info)
+          Gem::Version.new(matches[1])
+        end
     end
 
     def escape(value)
-      value.gsub("|", "||")
+      value.to_s.gsub("|", "||")
     end
   end
 end

data/lib/searchkick/relation.rb

@@ -8,6 +8,9 @@ module Searchkick
     delegate :body, :params, to: :query
     delegate_missing_to :private_execute
 
+    attr_reader :model
+    alias_method :klass, :model
+
     def initialize(model, term = "*", **options)
       @model = model
       @term = term
@@ -26,20 +29,22 @@ module Searchkick
 
     def execute
       Searchkick.warn("The execute method is no longer needed")
-      private_execute
-      self
+      load
     end
 
+    # experimental
     def limit(value)
       clone.limit!(value)
     end
 
+    # experimental
     def limit!(value)
       check_loaded
       @options[:limit] = value
       self
     end
 
+    # experimental
     def offset(value = NO_DEFAULT_VALUE)
       # TODO remove in Searchkick 6
       if value == NO_DEFAULT_VALUE
@@ -49,22 +54,26 @@ module Searchkick
       end
     end
 
+    # experimental
     def offset!(value)
       check_loaded
       @options[:offset] = value
       self
     end
 
+    # experimental
     def page(value)
       clone.page!(value)
     end
 
+    # experimental
     def page!(value)
       check_loaded
       @options[:page] = value
       self
     end
 
+    # experimental
     def per_page(value = NO_DEFAULT_VALUE)
       # TODO remove in Searchkick 6
       if value == NO_DEFAULT_VALUE
@@ -74,24 +83,139 @@ module Searchkick
       end
     end
 
+    # experimental
     def per_page!(value)
       check_loaded
       @options[:per_page] = value
       self
     end
 
+    # experimental
+    def where(value = NO_DEFAULT_VALUE)
+      if value == NO_DEFAULT_VALUE
+        Where.new(self)
+      else
+        clone.where!(value)
+      end
+    end
+
+    # experimental
+    def where!(value)
+      check_loaded
+      if @options[:where]
+        @options[:where] = {_and: [@options[:where], ensure_permitted(value)]}
+      else
+        @options[:where] = ensure_permitted(value)
+      end
+      self
+    end
+
+    # experimental
+    def rewhere(value)
+      clone.rewhere!(value)
+    end
+
+    # experimental
+    def rewhere!(value)
+      check_loaded
+      @options[:where] = ensure_permitted(value)
+      self
+    end
+
+    # experimental
+    def order(*values)
+      clone.order!(*values)
+    end
+
+    # experimental
+    def order!(*values)
+      values = values.first if values.size == 1 && values.first.is_a?(Array)
+      check_loaded
+      (@options[:order] ||= []).concat(values)
+      self
+    end
+
+    # experimental
+    def reorder(*values)
+      clone.reorder!(*values)
+    end
+
+    # experimental
+    def reorder!(*values)
+      check_loaded
+      @options[:order] = values
+      self
+    end
+
+    # experimental
+    def select(*values, &block)
+      if block_given?
+        private_execute.select(*values, &block)
+      else
+        clone.select!(*values)
+      end
+    end
+
+    # experimental
+    def select!(*values)
+      check_loaded
+      (@options[:select] ||= []).concat(values)
+      self
+    end
+
+    # experimental
+    def reselect(*values)
+      clone.reselect!(*values)
+    end
+
+    # experimental
+    def reselect!(*values)
+      check_loaded
+      @options[:select] = values
+      self
+    end
+
+    # experimental
+    def includes(*values)
+      clone.includes!(*values)
+    end
+
+    # experimental
+    def includes!(*values)
+      check_loaded
+      (@options[:includes] ||= []).concat(values)
+      self
+    end
+
+    # experimental
     def only(*keys)
       Relation.new(@model, @term, **@options.slice(*keys))
     end
 
+    # experimental
     def except(*keys)
       Relation.new(@model, @term, **@options.except(*keys))
     end
 
+    # experimental
+    def load
+      private_execute
+      self
+    end
+
     def loaded?
       !@execute.nil?
     end
 
+    def respond_to_missing?(method_name, include_all)
+      Results.new(nil, nil, nil).respond_to?(method_name, include_all) || super
+    end
+
+    # TODO uncomment in 6.0
+    # def to_yaml
+    #   private_execute.to_a.to_yaml
+    # end
+
     private
 
     def private_execute
@@ -108,5 +232,16 @@ module Searchkick
       # reset query since options will change
       @query = nil
     end
+
+    # provides *very* basic protection from unfiltered parameters
+    # this is not meant to be comprehensive and may be expanded in the future
+    def ensure_permitted(obj)
+      obj.to_h
+    end
+
+    def initialize_copy(other)
+      super
+      @execute = nil
+    end
   end
 end

data/lib/searchkick/relation_indexer.rb

@@ -14,12 +14,17 @@ module Searchkick
         relation = relation.search_import
       end
 
-      # remove unneeded loading for async
-      if mode == :async
+      # remove unneeded loading for async and queue
+      if mode == :async || mode == :queue
         if relation.respond_to?(:primary_key)
-          relation = relation.select(relation.primary_key).except(:includes, :preload)
+          relation = relation.except(:includes, :preload)
+          unless mode == :queue && relation.klass.method_defined?(:search_routing)
+            relation = relation.except(:select).select(relation.primary_key)
+          end
         elsif relation.respond_to?(:only)
-          relation = relation.only(:_id)
+          unless mode == :queue && relation.klass.method_defined?(:search_routing)
+            relation = relation.only(:_id)
+          end
         end
       end
 
@@ -42,11 +47,11 @@ module Searchkick
     end
 
     def batches_left
-      Searchkick.with_redis { |r| r.scard(batches_key) }
+      Searchkick.with_redis { |r| r.call("SCARD", batches_key) }
     end
 
     def batch_completed(batch_id)
-      Searchkick.with_redis { |r| r.srem(batches_key, batch_id) }
+      Searchkick.with_redis { |r| r.call("SREM", batches_key, [batch_id]) }
     end
 
     private
@@ -134,7 +139,7 @@ module Searchkick
     end
 
     def batch_job(class_name, batch_id, record_ids)
-      Searchkick.with_redis { |r| r.sadd(batches_key, batch_id) }
+      Searchkick.with_redis { |r| r.call("SADD", batches_key, [batch_id]) }
       Searchkick::BulkReindexJob.perform_later(
         class_name: class_name,
         index_name: index.name,

data/lib/searchkick/results.rb

@@ -3,6 +3,7 @@ module Searchkick
     include Enumerable
     extend Forwardable
 
+    # TODO remove klass and options in 6.0
     attr_reader :klass, :response, :options
 
     def_delegators :results, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary
@@ -13,6 +14,7 @@ module Searchkick
       @options = options
     end
 
+    # TODO make private in 6.0
     def results
       @results ||= with_hit.map(&:first)
     end
@@ -302,7 +304,7 @@ module Searchkick
     def build_hits
       @build_hits ||= begin
         if missing_records.any?
-          Searchkick.warn("Records in search index do not exist in database: #{missing_records.map { |v| v[:id] }.join(", ")}")
+          Searchkick.warn("Records in search index do not exist in database: #{missing_records.map { |v| "#{Array(v[:model]).map(&:model_name).sort.join("/")} #{v[:id]}" }.join(", ")}")
         end
         with_hit_and_missing_records[0]
       end

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "5.0.2"
+  VERSION = "5.3.0"
 end

data/lib/searchkick/where.rb

@@ -0,0 +1,11 @@
+module Searchkick
+  class Where
+    def initialize(relation)
+      @relation = relation
+    end
+
+    def not(value)
+      @relation.where(_not: value)
+    end
+  end
+end

data/lib/searchkick.rb

@@ -10,26 +10,27 @@ require "hashie"
 require "forwardable"
 
 # modules
-require "searchkick/controller_runtime"
-require "searchkick/index"
-require "searchkick/index_cache"
-require "searchkick/index_options"
-require "searchkick/indexer"
-require "searchkick/hash_wrapper"
-require "searchkick/log_subscriber"
-require "searchkick/model"
-require "searchkick/multi_search"
-require "searchkick/query"
-require "searchkick/reindex_queue"
-require "searchkick/record_data"
-require "searchkick/record_indexer"
-require "searchkick/relation"
-require "searchkick/relation_indexer"
-require "searchkick/results"
-require "searchkick/version"
+require_relative "searchkick/controller_runtime"
+require_relative "searchkick/index"
+require_relative "searchkick/index_cache"
+require_relative "searchkick/index_options"
+require_relative "searchkick/indexer"
+require_relative "searchkick/hash_wrapper"
+require_relative "searchkick/log_subscriber"
+require_relative "searchkick/model"
+require_relative "searchkick/multi_search"
+require_relative "searchkick/query"
+require_relative "searchkick/reindex_queue"
+require_relative "searchkick/record_data"
+require_relative "searchkick/record_indexer"
+require_relative "searchkick/relation"
+require_relative "searchkick/relation_indexer"
+require_relative "searchkick/results"
+require_relative "searchkick/version"
+require_relative "searchkick/where"
 
 # integrations
-require "searchkick/railtie" if defined?(Rails)
+require_relative "searchkick/railtie" if defined?(Rails)
 
 module Searchkick
   # requires faraday
@@ -134,8 +135,9 @@ module Searchkick
     @opensearch
   end
 
-  def self.server_below?(version)
-    server_version = opensearch? ? "7.10.2" : self.server_version
+  # TODO always check true version in Searchkick 6
+  def self.server_below?(version, true_version = false)
+    server_version = !true_version && opensearch? ? "7.10.2" : self.server_version
     Gem::Version.new(server_version.split("-")[0]) < Gem::Version.new(version.split("-")[0])
   end
 
@@ -283,7 +285,7 @@ module Searchkick
     relation
   end
 
-  # private
+  # public (for reindexing conversions)
   def self.load_model(class_name, allow_child: false)
     model = class_name.safe_constantize
     raise Error, "Could not find class: #{class_name}" unless model

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 5.0.2
+  version: 5.3.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-03 00:00:00.000000000 Z
+date: 2023-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: hashie
   requirement: !ruby/object:Gem::Requirement
@@ -71,6 +71,7 @@ files:
 - lib/searchkick/relation_indexer.rb
 - lib/searchkick/results.rb
 - lib/searchkick/version.rb
+- lib/searchkick/where.rb
 - lib/tasks/searchkick.rake
 homepage: https://github.com/ankane/searchkick
 licenses:
@@ -84,14 +85,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Intelligent search made easy with Rails and Elasticsearch or OpenSearch