searchkick 4.4.1 → 4.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff49d45a4473f44b07e8f1654d47e9b0590fd71eb3c6567f62b2fbcdc1e837b9
-  data.tar.gz: fd6ac4c931c7250787bc6f905c6c72b8412355e6b505bb2ef9d8cf5731d2939d
+  metadata.gz: 6937cc5456846216a4ea52c2274dfc43220e7f848dde276f0102ce48de3184a6
+  data.tar.gz: 0bfcc6a4e5a893629f9cac734498efba357dbd19e6641c21af7ea8e82f94f1ba
 SHA512:
-  metadata.gz: 05630a20b302fd406e5545935861b6616db79e2dd12e2eb11fe82bb3733c8b21af1087b7ad8d0fd4bd93eb3a07d2bd67461815a293746a2d2c2d3f2148d7147b
-  data.tar.gz: 571528f3967fd921e92dda6413313d29bd9048ee832e44bdb1a31a297178c3ee2f0b31dead3c573b297da96b676fdd74467bd7aa514c6a97f2a38b2500901057
+  metadata.gz: 0e7e41267464a8d32641edf6a578a6c65a97170f40e2f129effc8ff1da4144c30f00b88e52ddf6c51ffba366ec3cdfb263890e1539165ad47581c1723bf6b992
+  data.tar.gz: 49511b973a2fb7dce8d7952c0e5ccd676db357d2d965f46321e1b56bd5213f9f0c44f461f0010344f585e799dcbb13d9c4482772e20ab996462be86430c040cf

data/CHANGELOG.md

@@ -1,3 +1,56 @@
+## 4.6.3 (2021-11-19)
+
+- Added support for reloadable synonyms for OpenSearch
+- Added experimental support for `opensearch` gem
+- Removed `elasticsearch-xpack` dependency for reloadable synonyms
+
+## 4.6.2 (2021-11-15)
+
+- Added support for beginless ranges to `where` option
+- Fixed `like` and `ilike` with `+` character
+- Fixed warning about accessing system indices when no model or index specified
+
+## 4.6.1 (2021-09-25)
+
+- Added `ilike` operator for Elasticsearch 7.10+
+- Fixed missing methods with `multi_search`
+
+## 4.6.0 (2021-08-22)
+
+- Added support for case-insensitive regular expressions with Elasticsearch 7.10+
+- Added support for `OPENSEARCH_URL`
+- Fixed error with `debug` option
+
+## 4.5.2 (2021-08-05)
+
+- Fixed error with reindex queue
+- Fixed error with `model_name` method with multiple models
+- Fixed error with `debug` option with elasticsearch-ruby 7.14
+
+## 4.5.1 (2021-08-03)
+
+- Improved performance of reindex queue
+
+## 4.5.0 (2021-06-07)
+
+- Added experimental support for OpenSearch
+- Added support for synonyms in Japanese
+
+## 4.4.4 (2021-03-12)
+
+- Fixed `too_long_frame_exception` with `scroll` method
+- Fixed multi-word emoji tokenization
+
+## 4.4.3 (2021-02-25)
+
+- Added support for Hunspell
+- Fixed warning about accessing system indices
+
+## 4.4.2 (2020-11-23)
+
+- Added `missing_records` method to results
+- Fixed issue with `like` and special characters
+
 ## 4.4.1 (2020-06-24)
 
 - Added `stem_exclusion` and `stemmer_override` options

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013-2020 Andrew Kane
+Copyright (c) 2013-2021 Andrew Kane
 
 MIT License
 

data/README.md

@@ -20,13 +20,13 @@ Plus:
 - autocomplete
 - “Did you mean” suggestions
 - supports many languages
-- works with ActiveRecord, Mongoid, and NoBrainer
+- works with Active Record, Mongoid, and NoBrainer
 
-:tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
+Check out [Searchjoy](https://github.com/ankane/searchjoy) for analytics and [Autosuggest](https://github.com/ankane/autosuggest) for query suggestions
 
-:speech_balloon: Get [handcrafted updates](https://chartkick.us7.list-manage.com/subscribe?u=952c861f99eb43084e0a49f98&id=6ea6541e8e&group[0][4]=true) for new features
+:tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
-[![Build Status](https://travis-ci.org/ankane/searchkick.svg?branch=master)](https://travis-ci.org/ankane/searchkick)
+[![Build Status](https://github.com/ankane/searchkick/workflows/build/badge.svg?branch=master)](https://github.com/ankane/searchkick/actions)
 
 ## Contents
 
@@ -45,11 +45,11 @@ Plus:
 
 ## Getting Started
 
-[Install Elasticsearch](https://www.elastic.co/downloads/elasticsearch). For Homebrew, use:
+Install [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) or [OpenSearch](https://opensearch.org/downloads.html). For Homebrew, use:
 
 ```sh
-brew install elasticsearch
-brew services start elasticsearch
+brew install elasticsearch # or opensearch
+brew services start elasticsearch # or opensearch
 ```
 
 Add this line to your application’s Gemfile:
@@ -58,7 +58,7 @@ Add this line to your application’s Gemfile:
 gem 'searchkick'
 ```
 
-The latest version works with Elasticsearch 6 and 7. For Elasticsearch 5, use version 3.1.3 and [this readme](https://github.com/ankane/searchkick/blob/v3.1.3/README.md).
+The latest version works with Elasticsearch 6 and 7 and OpenSearch 1. For Elasticsearch 5, use version 3.1.3 and [this readme](https://github.com/ankane/searchkick/blob/v3.1.3/README.md).
 
 Add searchkick to models you want to search.
 
@@ -103,19 +103,20 @@ Where
 
 ```ruby
 where: {
-  expires_at: {gt: Time.now},   # lt, gte, lte also available
-  orders_count: 1..10,          # equivalent to {gte: 1, lte: 10}
-  aisle_id: [25, 30],           # in
-  store_id: {not: 2},           # not
-  aisle_id: {not: [25, 30]},    # not in
-  user_ids: {all: [1, 3]},      # all elements in array
-  category: {like: "%frozen%"}, # like
-  category: /frozen .+/,        # regexp
-  category: {prefix: "frozen"}, # prefix
-  store_id: {exists: true},     # exists
+  expires_at: {gt: Time.now},    # lt, gte, lte also available
+  orders_count: 1..10,           # equivalent to {gte: 1, lte: 10}
+  aisle_id: [25, 30],            # in
+  store_id: {not: 2},            # not
+  aisle_id: {not: [25, 30]},     # not in
+  user_ids: {all: [1, 3]},       # all elements in array
+  category: {like: "%frozen%"},  # like
+  category: {ilike: "%frozen%"}, # ilike
+  category: /frozen .+/,         # regexp
+  category: {prefix: "frozen"},  # prefix
+  store_id: {exists: true},      # exists
   _or: [{in_stock: true}, {backordered: true}],
   _and: [{in_stock: true}, {backordered: true}],
-  _not: {store_id: 1}           # negate a condition
+  _not: {store_id: 1}            # negate a condition
 }
 ```
 
@@ -125,7 +126,7 @@ Order
 order: {_score: :desc} # most relevant first - default
 ```
 
-[All of these sort options are supported](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html)
+[All of these sort options are supported](https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html)
 
 Limit / offset
 
@@ -139,7 +140,7 @@ Select
 select: [:name]
 ```
 
-[These source filtering options are supported](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#request-body-search-source-filtering)
+[These source filtering options are supported](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-fields.html#source-filtering)
 
 ### Results
 
@@ -176,7 +177,7 @@ Get the full response from Elasticsearch
 results.response
 ```
 
-**Note:** By default, Elasticsearch [limits paging](#deep-paging-master) to the first 10,000 results for performance. With Elasticsearch 7, this applies to the total count as well.
+**Note:** By default, Elasticsearch [limits paging](#deep-paging) to the first 10,000 results for performance. With Elasticsearch 7, this applies to the total count as well.
 
 ### Boosting
 
@@ -209,7 +210,7 @@ boost_by_recency: {created_at: {scale: "7d", decay: 0.5}}
 
 You can also boost by:
 
-- [Conversions](#keep-getting-better)
+- [Conversions](#intelligent-search)
 - [Distance](#boost-by-distance)
 
 ### Get Everything
@@ -311,7 +312,7 @@ class Product < ApplicationRecord
 end
 ```
 
-See the [list of stemmers](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-stemmer-tokenfilter.html). A few languages require plugins:
+See the [list of languages](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-stemmer-tokenfilter.html#analysis-stemmer-tokenfilter-configure-parms). A few languages require plugins:
 
 - `chinese` - [analysis-ik plugin](https://github.com/medcl/elasticsearch-analysis-ik)
 - `chinese2` - [analysis-smartcn plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/7.4/analysis-smartcn.html)
@@ -322,6 +323,14 @@ See the [list of stemmers](https://www.elastic.co/guide/en/elasticsearch/referen
 - `ukrainian` - [analysis-ukrainian plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/7.4/analysis-ukrainian.html)
 - `vietnamese` - [analysis-vietnamese plugin](https://github.com/duydo/elasticsearch-analysis-vietnamese)
 
+You can also use a Hunspell dictionary for stemming.
+
+```ruby
+class Product < ApplicationRecord
+  searchkick stemmer: {type: "hunspell", locale: "en_US"}
+end
+```
+
 Disable stemming with:
 
 ```ruby
@@ -366,9 +375,9 @@ search_synonyms: ["lightbulb => halogenlamp"]
 
 The above approach works well when your synonym list is static, but in practice, this is often not the case. When you analyze search conversions, you often want to add new synonyms without a full reindex.
 
-#### Elasticsearch 7.3+
+#### Elasticsearch 7.3+ or OpenSearch
 
-For Elasticsearch 7.3+, we recommend placing synonyms in a file on the Elasticsearch server (in the `config` directory). This allows you to reload synonyms without reindexing.
+For Elasticsearch 7.3+ or OpenSearch, we recommend placing synonyms in a file on the Elasticsearch or OpenSearch server (in the `config` directory). This allows you to reload synonyms without reindexing.
 
 ```txt
 pop, soda
@@ -378,16 +387,12 @@ burger, hamburger
 Then use:
 
 ```ruby
-search_synonyms: "synonyms.txt"
-```
-
-Add [elasticsearch-xpack](https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-xpack) to your Gemfile:
-
-```ruby
-gem 'elasticsearch-xpack', '>= 7.8.0'
+class Product < ApplicationRecord
+  searchkick search_synonyms: "synonyms.txt"
+end
 ```
 
-And use:
+And reload with:
 
 ```ruby
 Product.search_index.reload_synonyms
@@ -641,7 +646,7 @@ class Product < ApplicationRecord
   def search_data
     {
       name: name,
-      conversions: searches.group(:query).uniq.count(:user_id)
+      conversions: searches.group(:query).distinct.count(:user_id)
       # {"ice cream" => 234, "chocolate" => 67, "cream" => 2}
     }
   end
@@ -891,7 +896,7 @@ Additional options can be specified for each field:
 Band.search "cinema", fields: [:name], highlight: {fields: {name: {fragment_size: 200}}}
 ```
 
-You can find available highlight options in the [Elasticsearch reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-highlighting.html#_highlighted_fragments).
+You can find available highlight options in the [Elasticsearch reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/highlighting.html).
 
 ## Similar Items
 
@@ -942,7 +947,7 @@ Boost results by distance - closer results are boosted more
 Restaurant.search "noodles", boost_by_distance: {location: {origin: {lat: 37, lon: -122}}}
 ```
 
-Also supports [additional options](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html#_decay_functions)
+Also supports [additional options](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html#function-decay)
 
 ```ruby
 Restaurant.search "wings", boost_by_distance: {location: {origin: {lat: 37, lon: -122}, function: "linear", scale: "30mi", decay: 0.5}}
@@ -1202,14 +1207,28 @@ end
 FactoryBot.create(:product, :some_trait, :reindex, some_attribute: "foo")
 ```
 
+### GitHub Actions
+
+Check out [setup-elasticsearch](https://github.com/ankane/setup-elasticsearch) for an easy way to install Elasticsearch:
+
+```yml
+    - uses: ankane/setup-elasticsearch@v1
+```
+
+And [setup-opensearch](https://github.com/ankane/setup-opensearch) for an easy way to install OpenSearch:
+
+```yml
+    - uses: ankane/setup-opensearch@v1
+```
+
 ## Deployment
 
 Searchkick uses `ENV["ELASTICSEARCH_URL"]` for the Elasticsearch server. This defaults to `http://localhost:9200`.
 
 - [Elastic Cloud](#elastic-cloud)
 - [Heroku](#heroku)
-- [Amazon Elasticsearch Service](#amazon-elasticsearch-service)
-- [Self-Hosted and Other](#other)
+- [Amazon OpenSearch Service](#amazon-opensearch-service)
+- [Self-Hosted and Other](#self-hosted-and-other)
 
 ### Elastic Cloud
 
@@ -1232,7 +1251,7 @@ Choose an add-on: [Bonsai](https://elements.heroku.com/addons/bonsai), [SearchBo
 For Bonsai:
 
 ```sh
-heroku addons:create bonsai
+heroku addons:create bonsai # use --engine=opensearch for OpenSearch
 heroku config:set ELASTICSEARCH_URL=`heroku config:get BONSAI_URL`
 ```
 
@@ -1268,7 +1287,7 @@ Then deploy and reindex:
 heroku run rake searchkick:reindex:all
 ```
 
-### Amazon Elasticsearch Service
+### Amazon OpenSearch Service
 
 Create an initializer `config/initializers/elasticsearch.rb` with:
 
@@ -1461,7 +1480,7 @@ Product.search_index.promote(index_name, update_refresh_interval: true)
 
 ### Queuing
 
-Push ids of records needing reindexed to a queue and reindex in bulk for better performance. First, set up Redis in an initializer. We recommend using [connection_pool](https://github.com/mperham/connection_pool).
+Push ids of records needing reindexing to a queue and reindex in bulk for better performance. First, set up Redis in an initializer. We recommend using [connection_pool](https://github.com/mperham/connection_pool).
 
 ```ruby
 Searchkick.redis = ConnectionPool.new { Redis.new }
@@ -1564,14 +1583,14 @@ class ReindexConversionsJob < ApplicationJob
     # get records that have a recent conversion
     recently_converted_ids =
       Searchjoy::Search.where("convertable_type = ? AND converted_at > ?", class_name, 1.day.ago)
-      .order(:convertable_id).uniq.pluck(:convertable_id)
+      .order(:convertable_id).distinct.pluck(:convertable_id)
 
     # split into groups
     recently_converted_ids.in_groups_of(1000, false) do |ids|
       # fetch conversions
       conversions =
         Searchjoy::Search.where(convertable_id: ids, convertable_type: class_name)
-        .group(:convertable_id, :query).uniq.count(:user_id)
+        .group(:convertable_id, :query).distinct.count(:user_id)
 
       # group conversions by record
       conversions_by_record = {}
@@ -1695,7 +1714,7 @@ Check out [this great post](https://www.tiagoamaro.com.br/2014/12/11/multi-tenan
 
 ## Scroll API
 
-Searchkick also supports the [scroll API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html). Scrolling is not intended for real time user requests, but rather for processing large amounts of data.
+Searchkick also supports the [scroll API](https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#scroll-search-results). Scrolling is not intended for real time user requests, but rather for processing large amounts of data.
 
 ```ruby
 Product.search("*", scroll: "1m").scroll do |batch|
@@ -1823,7 +1842,7 @@ class Product < ApplicationRecord
   def search_data
     {
       name: name,
-      unique_user_conversions: searches.group(:query).uniq.count(:user_id),
+      unique_user_conversions: searches.group(:query).distinct.count(:user_id),
       # {"ice cream" => 234, "chocolate" => 67, "cream" => 2}
       total_conversions: searches.group(:query).count
       # {"ice cream" => 412, "chocolate" => 117, "cream" => 6}
@@ -1948,7 +1967,7 @@ products = Product.search("carrots", execute: false)
 products.each { ... } # search not executed until here
 ```
 
-Add [request parameters](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-uri-request.html), like `search_type` and `query_cache`
+Add [request parameters](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html#search-search-api-query-params) like `search_type`
 
 ```ruby
 Product.search("carrots", request_params: {search_type: "dfs_query_then_fetch"})

data/lib/searchkick/index.rb

@@ -84,7 +84,8 @@ module Searchkick
       old_indices =
         begin
           client.indices.get_alias(name: name).keys
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        rescue => e
+          raise e unless Searchkick.not_found_error?(e)
           {}
         end
       actions = old_indices.map { |old_name| {remove: {index: old_name, alias: name}} } + [{add: {index: new_name, alias: name}}]
@@ -105,11 +106,12 @@ module Searchkick
       indices =
         begin
           if client.indices.respond_to?(:get_alias)
-            client.indices.get_alias
+            client.indices.get_alias(index: "#{name}*")
           else
             client.indices.get_aliases
           end
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        rescue => e
+          raise e unless Searchkick.not_found_error?(e)
           {}
         end
       indices = indices.select { |_k, v| v.empty? || v["aliases"].empty? } if unaliased
@@ -161,6 +163,7 @@ module Searchkick
       RecordData.new(self, record).document_type
     end
 
+    # TODO use like: [{_index: ..., _id: ...}] in Searchkick 5
     def similar_record(record, **options)
       like_text = retrieve(record).to_hash
         .keep_if { |k, _| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
@@ -177,13 +180,15 @@ module Searchkick
     end
 
     def reload_synonyms
-      require "elasticsearch/xpack"
-      raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
-      raise Error, "Requires elasticsearch-xpack 7.8+" unless client.xpack.respond_to?(:indices)
-      begin
-        client.xpack.indices.reload_search_analyzers(index: name)
-      rescue Elasticsearch::Transport::Transport::Errors::MethodNotAllowed
-        raise Error, "Requires non-OSS version of Elasticsearch"
+      if Searchkick.opensearch?
+        client.transport.perform_request "POST", "_plugins/_refresh_search_analyzers/#{CGI.escape(name)}"
+      else
+        raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
+        begin
+          client.transport.perform_request("GET", "#{CGI.escape(name)}/_reload_search_analyzers")
+        rescue Elasticsearch::Transport::Transport::Errors::MethodNotAllowed
+          raise Error, "Requires non-OSS version of Elasticsearch"
+        end
       end
     end
 
@@ -360,9 +365,9 @@ module Searchkick
         index.refresh
         true
       end
-    rescue Elasticsearch::Transport::Transport::Errors::BadRequest => e
-      if e.message.include?("No handler for type [text]")
-        raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
+    rescue => e
+      if Searchkick.transport_error?(e) && e.message.include?("No handler for type [text]")
+        raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 6 or greater"
       end
 
       raise e

data/lib/searchkick/index_options.rb

@@ -153,6 +153,7 @@ module Searchkick
         }
       }
 
+      raise ArgumentError, "Can't pass both language and stemmer" if options[:stemmer] && language
       update_language(settings, language)
       update_stemming(settings)
 
@@ -234,6 +235,27 @@ module Searchkick
             type: "kuromoji"
           }
         )
+      when "japanese2"
+        analyzer = {
+          type: "custom",
+          tokenizer: "kuromoji_tokenizer",
+          filter: [
+            "kuromoji_baseform",
+            "kuromoji_part_of_speech",
+            "cjk_width",
+            "ja_stop",
+            "searchkick_stemmer",
+            "lowercase"
+          ]
+        }
+        settings[:analysis][:analyzer].merge!(
+          default_analyzer => analyzer.deep_dup,
+          searchkick_search: analyzer.deep_dup,
+          searchkick_search2: analyzer.deep_dup
+        )
+        settings[:analysis][:filter][:searchkick_stemmer] = {
+          type: "kuromoji_stemmer"
+        }
       when "korean"
         settings[:analysis][:analyzer].merge!(
           default_analyzer => {
@@ -286,6 +308,18 @@ module Searchkick
     end
 
     def update_stemming(settings)
+      if options[:stemmer]
+        stemmer = options[:stemmer]
+        # could also support snowball and stemmer
+        case stemmer[:type]
+        when "hunspell"
+          # supports all token filter options
+          settings[:analysis][:filter][:searchkick_stemmer] = stemmer
+        else
+          raise ArgumentError, "Unknown stemmer: #{stemmer[:type]}"
+        end
+      end
+
       stem = options[:stem]
 
       # language analyzer used
@@ -499,8 +533,18 @@ module Searchkick
         end
         settings[:analysis][:filter][:searchkick_synonym_graph] = synonym_graph
 
-        [:searchkick_search2, :searchkick_word_search].each do |analyzer|
-          settings[:analysis][:analyzer][analyzer][:filter].insert(2, "searchkick_synonym_graph")
+        if options[:language] == "japanese2"
+          [:searchkick_search, :searchkick_search2].each do |analyzer|
+            settings[:analysis][:analyzer][analyzer][:filter].insert(4, "searchkick_synonym_graph")
+          end
+        else
+          [:searchkick_search2, :searchkick_word_search].each do |analyzer|
+            unless settings[:analysis][:analyzer][analyzer].key?(:filter)
+              raise Searchkick::Error, "Search synonyms are not supported yet for language"
+            end
+
+            settings[:analysis][:analyzer][analyzer][:filter].insert(2, "searchkick_synonym_graph")
+          end
         end
       end
     end

data/lib/searchkick/model.rb

@@ -6,7 +6,7 @@ module Searchkick
       unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
         :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
         :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
-        :special_characters, :stem, :stem_conversions, :stem_exclusion, :stemmer_override, :suggest, :synonyms, :text_end,
+        :special_characters, :stem, :stemmer, :stem_conversions, :stem_exclusion, :stemmer_override, :suggest, :synonyms, :text_end,
         :text_middle, :text_start, :word, :wordnet, :word_end, :word_middle, :word_start]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
 

data/lib/searchkick/process_queue_job.rb

@@ -11,7 +11,7 @@ module Searchkick
         if record_ids.any?
           batch_options = {
             class_name: class_name,
-            record_ids: record_ids,
+            record_ids: record_ids.uniq,
             index_name: index_name
           }
 

data/lib/searchkick/query.rb

@@ -1,5 +1,6 @@
 module Searchkick
   class Query
+    include Enumerable
     extend Forwardable
 
     @@metric_aggs = [:avg, :cardinality, :max, :min, :sum]
@@ -12,7 +13,8 @@ module Searchkick
       :took, :error, :model_name, :entry_name, :total_count, :total_entries,
       :current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
       :offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
-      :out_of_range?, :hits, :response, :to_a, :first, :scroll
+      :out_of_range?, :hits, :response, :to_a, :first, :scroll, :highlights, :with_highlights,
+      :with_score, :misspellings?, :scroll_id, :clear_scroll, :missing_records, :with_hit
 
     def initialize(klass, term = "*", **options)
       unknown_keywords = options.keys - [:aggs, :block, :body, :body_options, :boost,
@@ -25,7 +27,7 @@ module Searchkick
       term = term.to_s
 
       if options[:emoji]
-        term = EmojiParser.parse_unicode(term) { |e| " #{e.name} " }.strip
+        term = EmojiParser.parse_unicode(term) { |e| " #{e.name.tr('_', ' ')} " }.strip
       end
 
       @klass = klass
@@ -73,7 +75,8 @@ module Searchkick
         elsif searchkick_index
           searchkick_index.name
         else
-          "_all"
+          # fixes warning about accessing system indices
+          "*,-.*"
         end
 
       params = {
@@ -109,7 +112,12 @@ module Searchkick
       request_params = query.except(:index, :type, :body)
 
       # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
+      host =
+        if Searchkick.client.transport.respond_to?(:transport)
+          Searchkick.client.transport.transport.hosts.first
+        else
+          Searchkick.client.transport.hosts.first
+        end
       credentials = host[:user] || host[:password] ? "#{host[:user]}:#{host[:password]}@" : nil
       params = ["pretty"]
       request_params.each do |k, v|
@@ -353,8 +361,8 @@ module Searchkick
               shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false || (!below73? && !track_total_hits?)
               qs << shared_options.merge(analyzer: "searchkick_search")
 
-              # searchkick_search and searchkick_search2 are the same for ukrainian
-              unless %w(japanese korean polish ukrainian vietnamese).include?(searchkick_options[:language])
+              # searchkick_search and searchkick_search2 are the same for some languages
+              unless %w(japanese japanese2 korean polish ukrainian vietnamese).include?(searchkick_options[:language])
                 qs << shared_options.merge(analyzer: "searchkick_search2")
               end
               exclude_analyzer = "searchkick_search2"
@@ -864,10 +872,11 @@ module Searchkick
       }
     end
 
-    # TODO id transformation for arrays
     def set_order(payload)
       order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
       id_field = :_id
+      # TODO no longer map id to _id in Searchkick 5
+      # since sorting on _id is deprecated in Elasticsearch
       payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
     end
 
@@ -896,12 +905,7 @@ module Searchkick
         else
           # expand ranges
           if value.is_a?(Range)
-            # infinite? added in Ruby 2.4
-            if value.end.nil? || (value.end.respond_to?(:infinite?) && value.end.infinite?)
-              value = {gte: value.first}
-            else
-              value = {gte: value.first, (value.exclude_end? ? :lt : :lte) => value.last}
-            end
+            value = expand_range(value)
           end
 
           value = {in: value} if value.is_a?(Array)
@@ -953,7 +957,7 @@ module Searchkick
                     }
                   }
                 }
-              when :like
+              when :like, :ilike
                 # based on Postgres
                 # https://www.postgresql.org/docs/current/functions-matching.html
                 # % matches zero or more characters
@@ -961,13 +965,22 @@ module Searchkick
                 # \ is escape character
                 # escape Lucene reserved characters
                 # https://www.elastic.co/guide/en/elasticsearch/reference/current/regexp-syntax.html#regexp-optional-operators
-                reserved = %w(. ? + * | { } [ ] ( ) " \\)
+                reserved = %w(\\ . ? + * | { } [ ] ( ) ")
                 regex = op_value.dup
                 reserved.each do |v|
-                  regex.gsub!(v, "\\" + v)
+                  regex.gsub!(v, "\\\\" + v)
                 end
                 regex = regex.gsub(/(?<!\\)%/, ".*").gsub(/(?<!\\)_/, ".").gsub("\\%", "%").gsub("\\_", "_")
-                filters << {regexp: {field => {value: regex}}}
+
+                if op == :ilike
+                  if below710?
+                    raise ArgumentError, "ilike requires Elasticsearch 7.10+"
+                  else
+                    filters << {regexp: {field => {value: regex, flags: "NONE", case_insensitive: true}}}
+                  end
+                else
+                  filters << {regexp: {field => {value: regex, flags: "NONE"}}}
+                end
               when :prefix
                 filters << {prefix: {field => {value: op_value}}}
               when :regexp # support for regexp queries without using a regexp ruby object
@@ -1022,10 +1035,6 @@ module Searchkick
       elsif value.nil?
         {bool: {must_not: {exists: {field: field}}}}
       elsif value.is_a?(Regexp)
-        if value.casefold?
-          Searchkick.warn("Case-insensitive flag does not work with Elasticsearch")
-        end
-
         source = value.source
         unless source.start_with?("\\A") && source.end_with?("\\z")
           # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-regexp-query.html
@@ -1047,7 +1056,14 @@ module Searchkick
           # source = "#{source}.*"
         end
 
-        {regexp: {field => {value: source, flags: "NONE"}}}
+        if below710?
+          if value.casefold?
+            Searchkick.warn("Case-insensitive flag does not work with Elasticsearch < 7.10")
+          end
+          {regexp: {field => {value: source, flags: "NONE"}}}
+        else
+          {regexp: {field => {value: source, flags: "NONE", case_insensitive: value.casefold?}}}
+        end
       else
         # TODO add this for other values
         if value.as_json.is_a?(Enumerable)
@@ -1118,6 +1134,17 @@ module Searchkick
       end
     end
 
+    def expand_range(range)
+      expanded = {}
+      expanded[:gte] = range.begin if range.begin
+
+      if range.end && !(range.end.respond_to?(:infinite?) && range.end.infinite?)
+        expanded[range.exclude_end? ? :lt : :lte] = range.end
+      end
+
+      expanded
+    end
+
     def base_field(k)
       k.sub(/\.(analyzed|word_start|word_middle|word_end|text_start|text_middle|text_end|exact)\z/, "")
     end
@@ -1145,5 +1172,9 @@ module Searchkick
     def below75?
       Searchkick.server_below?("7.5.0")
     end
+
+    def below710?
+      Searchkick.server_below?("7.10.0")
+    end
   end
 end

data/lib/searchkick/record_indexer.rb

@@ -64,8 +64,9 @@ module Searchkick
       if record.destroyed? || !record.persisted? || !record.should_index?
         begin
           index.remove(record)
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          # do nothing
+        rescue => e
+          raise e unless Searchkick.not_found_error?(e)
+          # do nothing if not found
         end
       else
         if method_name

data/lib/searchkick/reindex_queue.rb

@@ -14,11 +14,17 @@ module Searchkick
 
     # TODO use reliable queuing
     def reserve(limit: 1000)
-      record_ids = Set.new
-      while record_ids.size < limit && (record_id = Searchkick.with_redis { |r| r.rpop(redis_key) })
-        record_ids << record_id
+      if supports_rpop_with_count?
+        Searchkick.with_redis { |r| r.call("rpop", redis_key, limit) }.to_a
+      else
+        record_ids = []
+        Searchkick.with_redis do |r|
+          while record_ids.size < limit && (record_id = r.rpop(redis_key))
+            record_ids << record_id
+          end
+        end
+        record_ids
       end
-      record_ids.to_a
     end
 
     def clear
@@ -34,5 +40,13 @@ module Searchkick
     def redis_key
       "searchkick:reindex_queue:#{name}"
     end
+
+    def supports_rpop_with_count?
+      redis_version >= Gem::Version.new("6.2")
+    end
+
+    def redis_version
+      @redis_version ||= Searchkick.with_redis { |r| Gem::Version.new(r.info["redis_version"]) }
+    end
   end
 end

data/lib/searchkick/results.rb

@@ -22,76 +22,17 @@ module Searchkick
     # TODO return enumerator like with_score
     def with_hit
       @with_hit ||= begin
-        if options[:load]
-          # results can have different types
-          results = {}
-
-          hits.group_by { |hit, _| hit["_index"] }.each do |index, grouped_hits|
-            klasses =
-              if @klass
-                [@klass]
-              else
-                index_alias = index.split("_")[0..-2].join("_")
-                Array((options[:index_mapping] || {})[index_alias])
-              end
-            raise Searchkick::Error, "Unknown model for index: #{index}" unless klasses.any?
-
-            results[index] = {}
-            klasses.each do |klass|
-              results[index].merge!(results_query(klass, grouped_hits).to_a.index_by { |r| r.id.to_s })
-            end
-          end
-
-          missing_ids = []
-
-          # sort
-          results =
-            hits.map do |hit|
-              result = results[hit["_index"]][hit["_id"].to_s]
-              if result && !(options[:load].is_a?(Hash) && options[:load][:dumpable])
-                if (hit["highlight"] || options[:highlight]) && !result.respond_to?(:search_highlights)
-                  highlights = hit_highlights(hit)
-                  result.define_singleton_method(:search_highlights) do
-                    highlights
-                  end
-                end
-              end
-              [result, hit]
-            end.select do |result, hit|
-              missing_ids << hit["_id"] unless result
-              result
-            end
-
-          if missing_ids.any?
-            Searchkick.warn("Records in search index do not exist in database: #{missing_ids.join(", ")}")
-          end
-
-          results
-        else
-          hits.map do |hit|
-            result =
-              if hit["_source"]
-                hit.except("_source").merge(hit["_source"])
-              elsif hit["fields"]
-                hit.except("fields").merge(hit["fields"])
-              else
-                hit
-              end
-
-            if hit["highlight"] || options[:highlight]
-              highlight = Hash[hit["highlight"].to_a.map { |k, v| [base_field(k), v.first] }]
-              options[:highlighted_fields].map { |k| base_field(k) }.each do |k|
-                result["highlighted_#{k}"] ||= (highlight[k] || result[k])
-              end
-            end
-
-            result["id"] ||= result["_id"] # needed for legacy reasons
-            [HashWrapper.new(result), hit]
-          end
+        if missing_records.any?
+          Searchkick.warn("Records in search index do not exist in database: #{missing_records.map { |v| v[:id] }.join(", ")}")
         end
+        with_hit_and_missing_records[0]
       end
     end
 
+    def missing_records
+      @missing_records ||= with_hit_and_missing_records[1]
+    end
+
     def suggestions
       if response["suggest"]
         response["suggest"].values.flat_map { |v| v.first["options"] }.sort_by { |o| -o["score"] }.map { |o| o["text"] }.uniq
@@ -130,7 +71,11 @@ module Searchkick
     end
 
     def model_name
-      klass.model_name
+      if klass.nil?
+        ActiveModel::Name.new(self.class, nil, 'Result')
+      else
+        klass.model_name
+      end
     end
 
     def entry_name(options = {})
@@ -247,16 +192,11 @@ module Searchkick
 
         records.clear_scroll
       else
-        params = {
-          scroll: options[:scroll],
-          scroll_id: scroll_id
-        }
-
         begin
           # TODO Active Support notifications for this scroll call
-          Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
-          if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
+          Searchkick::Results.new(@klass, Searchkick.client.scroll(scroll: options[:scroll], body: {scroll_id: scroll_id}), @options)
+        rescue => e
+          if Searchkick.not_found_error?(e) && e.message =~ /search_context_missing_exception/i
             raise Searchkick::Error, "Scroll id has expired"
           else
             raise e
@@ -271,13 +211,97 @@ module Searchkick
         # not required as scroll will expire
         # but there is a cost to open scrolls
         Searchkick.client.clear_scroll(scroll_id: scroll_id)
-      rescue Elasticsearch::Transport::Transport::Error
-        # do nothing
+      rescue => e
+        raise e unless Searchkick.transport_error?(e)
       end
     end
 
     private
 
+    def with_hit_and_missing_records
+      @with_hit_and_missing_records ||= begin
+        missing_records = []
+
+        if options[:load]
+          grouped_hits = hits.group_by { |hit, _| hit["_index"] }
+
+          # determine models
+          index_models = {}
+          grouped_hits.each do |index, _|
+            models =
+              if @klass
+                [@klass]
+              else
+                index_alias = index.split("_")[0..-2].join("_")
+                Array((options[:index_mapping] || {})[index_alias])
+              end
+            raise Searchkick::Error, "Unknown model for index: #{index}. Pass the `models` option to the search method." unless models.any?
+            index_models[index] = models
+          end
+
+          # fetch results
+          results = {}
+          grouped_hits.each do |index, index_hits|
+            results[index] = {}
+            index_models[index].each do |model|
+              results[index].merge!(results_query(model, index_hits).to_a.index_by { |r| r.id.to_s })
+            end
+          end
+
+          # sort
+          results =
+            hits.map do |hit|
+              result = results[hit["_index"]][hit["_id"].to_s]
+              if result && !(options[:load].is_a?(Hash) && options[:load][:dumpable])
+                if (hit["highlight"] || options[:highlight]) && !result.respond_to?(:search_highlights)
+                  highlights = hit_highlights(hit)
+                  result.define_singleton_method(:search_highlights) do
+                    highlights
+                  end
+                end
+              end
+              [result, hit]
+            end.select do |result, hit|
+              unless result
+                models = index_models[hit["_index"]]
+                missing_records << {
+                  id: hit["_id"],
+                  # may be multiple models for inheritance with child models
+                  # not ideal to return different types
+                  # but this situation shouldn't be common
+                  model: models.size == 1 ? models.first : models
+                }
+              end
+              result
+            end
+        else
+          results =
+            hits.map do |hit|
+              result =
+                if hit["_source"]
+                  hit.except("_source").merge(hit["_source"])
+                elsif hit["fields"]
+                  hit.except("fields").merge(hit["fields"])
+                else
+                  hit
+                end
+
+              if hit["highlight"] || options[:highlight]
+                highlight = Hash[hit["highlight"].to_a.map { |k, v| [base_field(k), v.first] }]
+                options[:highlighted_fields].map { |k| base_field(k) }.each do |k|
+                  result["highlighted_#{k}"] ||= (highlight[k] || result[k])
+                end
+              end
+
+              result["id"] ||= result["_id"] # needed for legacy reasons
+              [HashWrapper.new(result), hit]
+            end
+        end
+
+       [results, missing_records]
+      end
+    end
+
     def results_query(records, hits)
       ids = hits.map { |hit| hit["_id"] }
       if options[:includes] || options[:model_includes]

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "4.4.1"
+  VERSION = "4.6.3"
 end

data/lib/searchkick.rb

@@ -34,6 +34,7 @@ module Searchkick
   class Error < StandardError; end
   class MissingIndexError < Error; end
   class UnsupportedVersionError < Error; end
+  # TODO switch to Error
   class InvalidQueryError < Elasticsearch::Transport::Transport::Errors::BadRequest; end
   class DangerousOperation < Error; end
   class ImportError < Error; end
@@ -56,7 +57,7 @@ module Searchkick
       require "typhoeus/adapters/faraday" if defined?(Typhoeus) && Gem::Version.new(Faraday::VERSION) < Gem::Version.new("0.14.0")
 
       Elasticsearch::Client.new({
-        url: ENV["ELASTICSEARCH_URL"],
+        url: ENV["ELASTICSEARCH_URL"] || ENV["OPENSEARCH_URL"],
         transport_options: {request: {timeout: timeout}, headers: {content_type: "application/json"}},
         retry_on_failure: 2
       }.deep_merge(client_options)) do |f|
@@ -74,11 +75,24 @@ module Searchkick
     (defined?(@search_timeout) && @search_timeout) || timeout
   end
 
+  # private
+  def self.server_info
+    @server_info ||= client.info
+  end
+
   def self.server_version
-    @server_version ||= client.info["version"]["number"]
+    @server_version ||= server_info["version"]["number"]
+  end
+
+  def self.opensearch?
+    unless defined?(@opensearch)
+      @opensearch = server_info["version"]["distribution"] == "opensearch"
+    end
+    @opensearch
   end
 
   def self.server_below?(version)
+    server_version = opensearch? ? "7.10.2" : self.server_version
     Gem::Version.new(server_version.split("-")[0]) < Gem::Version.new(version.split("-")[0])
   end
 
@@ -265,6 +279,18 @@ module Searchkick
       !Mongoid::Threaded.current_scope(klass).nil?
     end
   end
+
+  # private
+  def self.not_found_error?(e)
+    (defined?(Elasticsearch) && e.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound)) ||
+    (defined?(OpenSearch) && e.is_a?(OpenSearch::Transport::Transport::Errors::NotFound))
+  end
+
+  # private
+  def self.transport_error?(e)
+    (defined?(Elasticsearch) && e.is_a?(Elasticsearch::Transport::Transport::Error)) ||
+    (defined?(OpenSearch) && e.is_a?(OpenSearch::Transport::Transport::Error))
+  end
 end
 
 require "active_model/callbacks"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 4.4.1
+  version: 4.6.3
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-25 00:00:00.000000000 Z
+date: 2021-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -31,6 +31,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -38,6 +41,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.14'
 - !ruby/object:Gem::Dependency
   name: hashie
   requirement: !ruby/object:Gem::Requirement
@@ -52,8 +58,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
-email: andrew@chartkick.com
+description:
+email: andrew@ankane.org
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -87,7 +93,7 @@ homepage: https://github.com/ankane/searchkick
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -102,8 +108,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
-summary: Intelligent search made easy with Rails and Elasticsearch
+summary: Intelligent search made easy with Rails and Elasticsearch or OpenSearch
 test_files: []