searchkick 4.4.4 → 4.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 864e600f47d8f3cc55c5664f04b71a1eb054bdad20acd5eb206a77bdeed03503
-  data.tar.gz: 2dcd69102ab91f46209dd345cf180309729d470423fb0832fa8e6ed0772cfa45
+  metadata.gz: d93159910ecc579b56511b5d8c6908b34fecbce17b49ccbc264383068610feb9
+  data.tar.gz: a60319ffc12e2aac95bf96f616bb65afcb8e75b77a89598e960193c8424c677a
 SHA512:
-  metadata.gz: c687faaae288a53c7ec5260c501c390c0c27cef372cb76dea4caeaa4bdbd6554ccf627ba10a82fe8202f236d84ee00e8872632316f66e0cc89fe4f3900e017a0
-  data.tar.gz: c61486f67dc88dee390022aacb91f45278559d23cf62dd7a9f9468aa2e317b0d643a7f500162608e81496c750987e39391b1200a1d2f0528985187b32b094f53
+  metadata.gz: 1ce03474756a2adf3ada602580c6e56d2a39d39d93096b9d796167f9bc841a92b5d321914a05ab75ca9e10e87cf6e77ed8590b6e9934cae1ec0e56ba590f65c8
+  data.tar.gz: ae617b74a6ed27f6caa6bf9f3494a62d35ab4176e87bf868a5c04e60ae1b78462bee07de1f182cd3d81cac5a3bcaa22a57aad5e75f61c384aa8ceb6344822486

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+## 4.6.0 (2021-08-22)
+
+- Added support for case-insensitive regular expressions with Elasticsearch 7.10+
+- Added support for `OPENSEARCH_URL`
+- Fixed error with `debug` option
+
+## 4.5.2 (2021-08-05)
+
+- Fixed error with reindex queue
+- Fixed error with `model_name` method with multiple models
+- Fixed error with `debug` option with elasticsearch-ruby 7.14
+
+## 4.5.1 (2021-08-03)
+
+- Improved performance of reindex queue
+
+## 4.5.0 (2021-06-07)
+
+- Added experimental support for OpenSearch
+- Added support for synonyms in Japanese
+
 ## 4.4.4 (2021-03-12)
 
 - Fixed `too_long_frame_exception` with `scroll` method

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013-2020 Andrew Kane
+Copyright (c) 2013-2021 Andrew Kane
 
 MIT License
 

data/README.md

@@ -20,7 +20,7 @@ Plus:
 - autocomplete
 - “Did you mean” suggestions
 - supports many languages
-- works with ActiveRecord, Mongoid, and NoBrainer
+- works with Active Record, Mongoid, and NoBrainer
 
 Check out [Searchjoy](https://github.com/ankane/searchjoy) for analytics and [Autosuggest](https://github.com/ankane/autosuggest) for query suggestions
 
@@ -45,11 +45,11 @@ Check out [Searchjoy](https://github.com/ankane/searchjoy) for analytics and [Au
 
 ## Getting Started
 
-[Install Elasticsearch](https://www.elastic.co/downloads/elasticsearch). For Homebrew, use:
+Install [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) or [OpenSearch](https://opensearch.org/downloads.html). For Homebrew, use:
 
 ```sh
-brew install elasticsearch
-brew services start elasticsearch
+brew install elasticsearch # or opensearch
+brew services start elasticsearch # or opensearch
 ```
 
 Add this line to your application’s Gemfile:
@@ -58,7 +58,7 @@ Add this line to your application’s Gemfile:
 gem 'searchkick'
 ```
 
-The latest version works with Elasticsearch 6 and 7. For Elasticsearch 5, use version 3.1.3 and [this readme](https://github.com/ankane/searchkick/blob/v3.1.3/README.md).
+The latest version works with Elasticsearch 6 and 7 and OpenSearch 1. For Elasticsearch 5, use version 3.1.3 and [this readme](https://github.com/ankane/searchkick/blob/v3.1.3/README.md).
 
 Add searchkick to models you want to search.
 
@@ -125,7 +125,7 @@ Order
 order: {_score: :desc} # most relevant first - default
 ```
 
-[All of these sort options are supported](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html)
+[All of these sort options are supported](https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html)
 
 Limit / offset
 
@@ -139,7 +139,7 @@ Select
 select: [:name]
 ```
 
-[These source filtering options are supported](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#request-body-search-source-filtering)
+[These source filtering options are supported](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-fields.html#source-filtering)
 
 ### Results
 
@@ -392,7 +392,7 @@ search_synonyms: "synonyms.txt"
 Add [elasticsearch-xpack](https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-xpack) to your Gemfile:
 
 ```ruby
-gem 'elasticsearch-xpack', '>= 7.8.0'
+gem 'elasticsearch-xpack', '>= 7.8', '< 7.14'
 ```
 
 And use:
@@ -401,7 +401,7 @@ And use:
 Product.search_index.reload_synonyms
 ```
 
-#### Elasticsearch < 7.3
+#### Elasticsearch < 7.3 or OpenSearch
 
 You can use a library like [ActsAsTaggableOn](https://github.com/mbleigh/acts-as-taggable-on) and do:
 
@@ -649,7 +649,7 @@ class Product < ApplicationRecord
   def search_data
     {
       name: name,
-      conversions: searches.group(:query).uniq.count(:user_id)
+      conversions: searches.group(:query).distinct.count(:user_id)
       # {"ice cream" => 234, "chocolate" => 67, "cream" => 2}
     }
   end
@@ -899,7 +899,7 @@ Additional options can be specified for each field:
 Band.search "cinema", fields: [:name], highlight: {fields: {name: {fragment_size: 200}}}
 ```
 
-You can find available highlight options in the [Elasticsearch reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-highlighting.html#_highlighted_fragments).
+You can find available highlight options in the [Elasticsearch reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/highlighting.html).
 
 ## Similar Items
 
@@ -950,7 +950,7 @@ Boost results by distance - closer results are boosted more
 Restaurant.search "noodles", boost_by_distance: {location: {origin: {lat: 37, lon: -122}}}
 ```
 
-Also supports [additional options](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html#_decay_functions)
+Also supports [additional options](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-function-score-query.html#function-decay)
 
 ```ruby
 Restaurant.search "wings", boost_by_distance: {location: {origin: {lat: 37, lon: -122}, function: "linear", scale: "30mi", decay: 0.5}}
@@ -1212,12 +1212,18 @@ FactoryBot.create(:product, :some_trait, :reindex, some_attribute: "foo")
 
 ### GitHub Actions
 
-Check out [setup-elasticsearch](https://github.com/ankane/setup-elasticsearch) for an easy way to install Elasticsearch.
+Check out [setup-elasticsearch](https://github.com/ankane/setup-elasticsearch) for an easy way to install Elasticsearch:
 
 ```yml
     - uses: ankane/setup-elasticsearch@v1
 ```
 
+And [setup-opensearch](https://github.com/ankane/setup-opensearch) for an easy way to install OpenSearch:
+
+```yml
+    - uses: ankane/setup-opensearch@v1
+```
+
 ## Deployment
 
 Searchkick uses `ENV["ELASTICSEARCH_URL"]` for the Elasticsearch server. This defaults to `http://localhost:9200`.
@@ -1248,7 +1254,7 @@ Choose an add-on: [Bonsai](https://elements.heroku.com/addons/bonsai), [SearchBo
 For Bonsai:
 
 ```sh
-heroku addons:create bonsai
+heroku addons:create bonsai # use --engine=opensearch for OpenSearch
 heroku config:set ELASTICSEARCH_URL=`heroku config:get BONSAI_URL`
 ```
 
@@ -1580,14 +1586,14 @@ class ReindexConversionsJob < ApplicationJob
     # get records that have a recent conversion
     recently_converted_ids =
       Searchjoy::Search.where("convertable_type = ? AND converted_at > ?", class_name, 1.day.ago)
-      .order(:convertable_id).uniq.pluck(:convertable_id)
+      .order(:convertable_id).distinct.pluck(:convertable_id)
 
     # split into groups
     recently_converted_ids.in_groups_of(1000, false) do |ids|
       # fetch conversions
       conversions =
         Searchjoy::Search.where(convertable_id: ids, convertable_type: class_name)
-        .group(:convertable_id, :query).uniq.count(:user_id)
+        .group(:convertable_id, :query).distinct.count(:user_id)
 
       # group conversions by record
       conversions_by_record = {}
@@ -1711,7 +1717,7 @@ Check out [this great post](https://www.tiagoamaro.com.br/2014/12/11/multi-tenan
 
 ## Scroll API
 
-Searchkick also supports the [scroll API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html). Scrolling is not intended for real time user requests, but rather for processing large amounts of data.
+Searchkick also supports the [scroll API](https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#scroll-search-results). Scrolling is not intended for real time user requests, but rather for processing large amounts of data.
 
 ```ruby
 Product.search("*", scroll: "1m").scroll do |batch|
@@ -1839,7 +1845,7 @@ class Product < ApplicationRecord
   def search_data
     {
       name: name,
-      unique_user_conversions: searches.group(:query).uniq.count(:user_id),
+      unique_user_conversions: searches.group(:query).distinct.count(:user_id),
       # {"ice cream" => 234, "chocolate" => 67, "cream" => 2}
       total_conversions: searches.group(:query).count
       # {"ice cream" => 412, "chocolate" => 117, "cream" => 6}
@@ -1964,7 +1970,7 @@ products = Product.search("carrots", execute: false)
 products.each { ... } # search not executed until here
 ```
 
-Add [request parameters](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-uri-request.html), like `search_type` and `query_cache`
+Add [request parameters](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html#search-search-api-query-params) like `search_type`
 
 ```ruby
 Product.search("carrots", request_params: {search_type: "dfs_query_then_fetch"})

data/lib/searchkick.rb

@@ -56,7 +56,7 @@ module Searchkick
       require "typhoeus/adapters/faraday" if defined?(Typhoeus) && Gem::Version.new(Faraday::VERSION) < Gem::Version.new("0.14.0")
 
       Elasticsearch::Client.new({
-        url: ENV["ELASTICSEARCH_URL"],
+        url: ENV["ELASTICSEARCH_URL"] || ENV["OPENSEARCH_URL"],
         transport_options: {request: {timeout: timeout}, headers: {content_type: "application/json"}},
         retry_on_failure: 2
       }.deep_merge(client_options)) do |f|
@@ -74,11 +74,24 @@ module Searchkick
     (defined?(@search_timeout) && @search_timeout) || timeout
   end
 
+  # private
+  def self.server_info
+    @server_info ||= client.info
+  end
+
   def self.server_version
-    @server_version ||= client.info["version"]["number"]
+    @server_version ||= server_info["version"]["number"]
+  end
+
+  def self.opensearch?
+    unless defined?(@opensearch)
+      @opensearch = server_info["version"]["distribution"] == "opensearch"
+    end
+    @opensearch
   end
 
   def self.server_below?(version)
+    server_version = opensearch? ? "7.10.2" : self.server_version
     Gem::Version.new(server_version.split("-")[0]) < Gem::Version.new(version.split("-")[0])
   end
 

data/lib/searchkick/index_options.rb

@@ -235,6 +235,27 @@ module Searchkick
             type: "kuromoji"
           }
         )
+      when "japanese2"
+        analyzer = {
+          type: "custom",
+          tokenizer: "kuromoji_tokenizer",
+          filter: [
+            "kuromoji_baseform",
+            "kuromoji_part_of_speech",
+            "cjk_width",
+            "ja_stop",
+            "searchkick_stemmer",
+            "lowercase"
+          ]
+        }
+        settings[:analysis][:analyzer].merge!(
+          default_analyzer => analyzer.deep_dup,
+          searchkick_search: analyzer.deep_dup,
+          searchkick_search2: analyzer.deep_dup
+        )
+        settings[:analysis][:filter][:searchkick_stemmer] = {
+          type: "kuromoji_stemmer"
+        }
       when "korean"
         settings[:analysis][:analyzer].merge!(
           default_analyzer => {
@@ -512,8 +533,18 @@ module Searchkick
         end
         settings[:analysis][:filter][:searchkick_synonym_graph] = synonym_graph
 
-        [:searchkick_search2, :searchkick_word_search].each do |analyzer|
-          settings[:analysis][:analyzer][analyzer][:filter].insert(2, "searchkick_synonym_graph")
+        if options[:language] == "japanese2"
+          [:searchkick_search, :searchkick_search2].each do |analyzer|
+            settings[:analysis][:analyzer][analyzer][:filter].insert(4, "searchkick_synonym_graph")
+          end
+        else
+          [:searchkick_search2, :searchkick_word_search].each do |analyzer|
+            unless settings[:analysis][:analyzer][analyzer].key?(:filter)
+              raise Searchkick::Error, "Search synonyms are not supported yet for language"
+            end
+
+            settings[:analysis][:analyzer][analyzer][:filter].insert(2, "searchkick_synonym_graph")
+          end
         end
       end
     end

data/lib/searchkick/process_queue_job.rb

@@ -11,7 +11,7 @@ module Searchkick
         if record_ids.any?
           batch_options = {
             class_name: class_name,
-            record_ids: record_ids,
+            record_ids: record_ids.uniq,
             index_name: index_name
           }
 

data/lib/searchkick/query.rb

@@ -109,7 +109,12 @@ module Searchkick
       request_params = query.except(:index, :type, :body)
 
       # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
+      host =
+        if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("7.14.0")
+          Searchkick.client.transport.transport.hosts.first
+        else
+          Searchkick.client.transport.hosts.first
+        end
       credentials = host[:user] || host[:password] ? "#{host[:user]}:#{host[:password]}@" : nil
       params = ["pretty"]
       request_params.each do |k, v|
@@ -353,8 +358,8 @@ module Searchkick
               shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false || (!below73? && !track_total_hits?)
               qs << shared_options.merge(analyzer: "searchkick_search")
 
-              # searchkick_search and searchkick_search2 are the same for ukrainian
-              unless %w(japanese korean polish ukrainian vietnamese).include?(searchkick_options[:language])
+              # searchkick_search and searchkick_search2 are the same for some languages
+              unless %w(japanese japanese2 korean polish ukrainian vietnamese).include?(searchkick_options[:language])
                 qs << shared_options.merge(analyzer: "searchkick_search2")
               end
               exclude_analyzer = "searchkick_search2"
@@ -864,10 +869,11 @@ module Searchkick
       }
     end
 
-    # TODO id transformation for arrays
     def set_order(payload)
       order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
       id_field = :_id
+      # TODO no longer map id to _id in Searchkick 5
+      # since sorting on _id is deprecated in Elasticsearch
       payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
     end
 
@@ -1022,10 +1028,6 @@ module Searchkick
       elsif value.nil?
         {bool: {must_not: {exists: {field: field}}}}
       elsif value.is_a?(Regexp)
-        if value.casefold?
-          Searchkick.warn("Case-insensitive flag does not work with Elasticsearch")
-        end
-
         source = value.source
         unless source.start_with?("\\A") && source.end_with?("\\z")
           # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-regexp-query.html
@@ -1047,7 +1049,14 @@ module Searchkick
           # source = "#{source}.*"
         end
 
-        {regexp: {field => {value: source, flags: "NONE"}}}
+        if below710?
+          if value.casefold?
+            Searchkick.warn("Case-insensitive flag does not work with Elasticsearch < 7.10")
+          end
+          {regexp: {field => {value: source, flags: "NONE"}}}
+        else
+          {regexp: {field => {value: source, flags: "NONE", case_insensitive: value.casefold?}}}
+        end
       else
         # TODO add this for other values
         if value.as_json.is_a?(Enumerable)
@@ -1145,5 +1154,9 @@ module Searchkick
     def below75?
       Searchkick.server_below?("7.5.0")
     end
+
+    def below710?
+      Searchkick.server_below?("7.10.0")
+    end
   end
 end

data/lib/searchkick/reindex_queue.rb

@@ -14,11 +14,17 @@ module Searchkick
 
     # TODO use reliable queuing
     def reserve(limit: 1000)
-      record_ids = Set.new
-      while record_ids.size < limit && (record_id = Searchkick.with_redis { |r| r.rpop(redis_key) })
-        record_ids << record_id
+      if supports_rpop_with_count?
+        Searchkick.with_redis { |r| r.call("rpop", redis_key, limit) }.to_a
+      else
+        record_ids = []
+        Searchkick.with_redis do |r|
+          while record_ids.size < limit && (record_id = r.rpop(redis_key))
+            record_ids << record_id
+          end
+        end
+        record_ids
       end
-      record_ids.to_a
     end
 
     def clear
@@ -34,5 +40,13 @@ module Searchkick
     def redis_key
       "searchkick:reindex_queue:#{name}"
     end
+
+    def supports_rpop_with_count?
+      redis_version >= Gem::Version.new("6.2")
+    end
+
+    def redis_version
+      @redis_version ||= Searchkick.with_redis { |r| Gem::Version.new(r.info["redis_version"]) }
+    end
   end
 end

data/lib/searchkick/results.rb

@@ -71,7 +71,11 @@ module Searchkick
     end
 
     def model_name
-      klass.model_name
+      if klass.nil?
+        ActiveModel::Name.new(self.class, nil, 'Result')
+      else
+        klass.model_name
+      end
     end
 
     def entry_name(options = {})
@@ -231,7 +235,7 @@ module Searchkick
                 index_alias = index.split("_")[0..-2].join("_")
                 Array((options[:index_mapping] || {})[index_alias])
               end
-            raise Searchkick::Error, "Unknown model for index: #{index}" unless models.any?
+            raise Searchkick::Error, "Unknown model for index: #{index}. Pass the `models` option to the search method." unless models.any?
             index_models[index] = models
           end
 

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "4.4.4"
+  VERSION = "4.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 4.4.4
+  version: 4.6.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-12 00:00:00.000000000 Z
+date: 2021-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -31,6 +31,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -38,6 +41,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.14'
 - !ruby/object:Gem::Dependency
   name: hashie
   requirement: !ruby/object:Gem::Requirement
@@ -53,7 +59,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description:
-email: andrew@chartkick.com
+email: andrew@ankane.org
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -102,8 +108,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
-summary: Intelligent search made easy with Rails and Elasticsearch
+summary: Intelligent search made easy with Rails and Elasticsearch or OpenSearch
 test_files: []