searchkick 3.0.0 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0114d21ec3a6707c49408e641fa27a5591cc9eca
-  data.tar.gz: d723af7de81aa5ff40818fcd229029724a0fcda7
+  metadata.gz: bf72e82ba04fdb0dad02352ce84d8d7844b9097d
+  data.tar.gz: 53739b3ac5968665fb53d73f6966b63e2c9bf3c3
 SHA512:
-  metadata.gz: 04b026c1acdd413476a2099f7b0b335c207f286e4aadd9d323f0f5d8f5bc18cd15da2219040830a12f258dba497c9412510815fd2af70682e3579239caeaabba
-  data.tar.gz: e41240ef1831b869311c588a3a6d87c054607fedd2d3da3d56db4d220ae7c1038a317cb703f220b36f06c2ada8b99f2a34cd8e3b7f7c046d6ad2fb2115004a3c
+  metadata.gz: f1e26f41eda3b67dae70f0fe0502f46bef521565bdf69b70d1762430292e941f1fb581f6a485779705afe869f2728d9bd5e18abcd7537de1bcb2307831a84c75
+  data.tar.gz: 03c79e33d55fd712d4b75ff9830d00c9fa78acdc65aeb80a78cc23931055ecec0a3e892d239491236ecedc7184c8d37a062a0ece6a45779b1244e9117eed28d9

data/CHANGELOG.md

@@ -1,6 +1,12 @@
+## 3.0.1
+
+- Added `scope` option for partial reindex
+- Added support for Japanese, Polish, and Ukranian
+
 ## 3.0.0
 
 - Added support for Chinese
+- No longer requires fields to query for Elasticsearch 6
 - Results can be marshaled by default (unless using `highlight` option)
 
 Breaking changes

data/Gemfile

@@ -4,13 +4,13 @@ source "https://rubygems.org"
 gemspec
 
 gem "sqlite3"
-gem "activerecord", "~> 5.2.0.rc1"
+gem "activerecord"
 gem "gemoji-parser"
 gem "typhoeus"
-gem "activejob", "~> 5.2.0.rc1"
+gem "activejob"
 gem "redis"
 gem "connection_pool"
 
 # kaminari
-gem "actionpack", "~> 5.2.0.rc1"
+gem "actionpack"
 gem "kaminari"

data/README.md

@@ -19,6 +19,7 @@ Plus:
 - easily personalize results for each user
 - autocomplete
 - “Did you mean” suggestions
+- supports many languages
 - works with ActiveRecord, Mongoid, and NoBrainer
 
 :speech_balloon: Get [handcrafted updates](http://chartkick.us7.list-manage.com/subscribe?u=952c861f99eb43084e0a49f98&id=6ea6541e8e&group[0][4]=true) for new features
@@ -299,13 +300,12 @@ end
 
 [See the list of stemmers](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-stemmer-tokenfilter.html)
 
-For Chinese, install the [elasticsearch-analysis-ik plugin](https://github.com/medcl/elasticsearch-analysis-ik) and use:
+A few languages require plugins:
 
-```ruby
-class Product < ApplicationRecord
-  searchkick language: "chinese"
-end
-```
+- `chinese` - [elasticsearch-analysis-ik plugin](https://github.com/medcl/elasticsearch-analysis-ik)
+- `japanese` - [analysis-kuromoji plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-kuromoji.html)
+- `polish` - [analysis-stempel plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-stempel.html)
+- `ukrainian` - [analysis-ukrainian plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-ukrainian.html)
 
 ### Synonyms
 
@@ -1344,7 +1344,7 @@ end
 Create a job to update the cache and reindex records with new conversions.
 
 ```ruby
-class ReindexConversionsJob < ActiveJob::Base
+class ReindexConversionsJob < ApplicationJob
   def perform(class_name)
     # get records that have a recent conversion
     recently_converted_ids =

data/lib/searchkick/bulk_indexer.rb

@@ -6,31 +6,34 @@ module Searchkick
       @index = index
     end
 
-    def import_scope(scope, resume: false, method_name: nil, async: false, batch: false, batch_id: nil, full: false)
-      # use scope for import
-      scope = scope.search_import if scope.respond_to?(:search_import)
+    def import_scope(relation, resume: false, method_name: nil, async: false, batch: false, batch_id: nil, full: false, scope: nil)
+      if scope
+        relation = relation.send(scope)
+      elsif relation.respond_to?(:search_import)
+        relation = relation.search_import
+      end
 
       if batch
-        import_or_update scope.to_a, method_name, async
+        import_or_update relation.to_a, method_name, async
         Searchkick.with_redis { |r| r.srem(batches_key, batch_id) } if batch_id
       elsif full && async
-        full_reindex_async(scope)
-      elsif scope.respond_to?(:find_in_batches)
+        full_reindex_async(relation)
+      elsif relation.respond_to?(:find_in_batches)
         if resume
           # use total docs instead of max id since there's not a great way
           # to get the max _id without scripting since it's a string
 
           # TODO use primary key and prefix with table name
-          scope = scope.where("id > ?", total_docs)
+          relation = relation.where("id > ?", total_docs)
         end
 
-        scope = scope.select("id").except(:includes, :preload) if async
+        relation = relation.select("id").except(:includes, :preload) if async
 
-        scope.find_in_batches batch_size: batch_size do |items|
+        relation.find_in_batches batch_size: batch_size do |items|
           import_or_update items, method_name, async
         end
       else
-        each_batch(scope) do |items|
+        each_batch(relation) do |items|
           import_or_update items, method_name, async
         end
       end

data/lib/searchkick/index.rb

@@ -178,22 +178,22 @@ module Searchkick
 
     # reindex
 
-    def reindex(scope, method_name, scoped:, full: false, **options)
+    def reindex(relation, method_name, scoped:, full: false, scope: nil, **options)
       refresh = options.fetch(:refresh, !scoped)
 
       if method_name
         # update
-        import_scope(scope, method_name: method_name)
+        import_scope(relation, method_name: method_name, scope: scope)
         self.refresh if refresh
         true
       elsif scoped && !full
         # reindex association
-        import_scope(scope)
+        import_scope(relation, scope: scope)
         self.refresh if refresh
         true
       else
         # full reindex
-        reindex_scope(scope, options)
+        reindex_scope(relation, scope: scope, **options)
       end
     end
 
@@ -204,8 +204,8 @@ module Searchkick
       index
     end
 
-    def import_scope(scope, **options)
-      bulk_indexer.import_scope(scope, **options)
+    def import_scope(relation, **options)
+      bulk_indexer.import_scope(relation, **options)
     end
 
     def batches_left
@@ -257,7 +257,7 @@ module Searchkick
 
     # https://gist.github.com/jarosan/3124884
     # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
-    def reindex_scope(scope, import: true, resume: false, retain: false, async: false, refresh_interval: nil)
+    def reindex_scope(relation, import: true, resume: false, retain: false, async: false, refresh_interval: nil, scope: nil)
       if resume
         index_name = all_indices.sort.last
         raise Searchkick::Error, "No index to resume" unless index_name
@@ -265,16 +265,23 @@ module Searchkick
       else
         clean_indices unless retain
 
-        index_options = scope.searchkick_index_options
+        index_options = relation.searchkick_index_options
         index_options.deep_merge!(settings: {index: {refresh_interval: refresh_interval}}) if refresh_interval
         index = create_index(index_options: index_options)
       end
 
+      import_options = {
+        resume: resume,
+        async: async,
+        full: true,
+        scope: scope
+      }
+
       # check if alias exists
       alias_exists = alias_exists?
       if alias_exists
         # import before promotion
-        index.import_scope(scope, resume: resume, async: async, full: true) if import
+        index.import_scope(relation, **import_options) if import
 
         # get existing indices to remove
         unless async
@@ -286,7 +293,7 @@ module Searchkick
         promote(index.name, update_refresh_interval: !refresh_interval.nil?)
 
         # import after promotion
-        index.import_scope(scope, resume: resume, async: async, full: true) if import
+        index.import_scope(relation, **import_options) if import
       end
 
       if async

data/lib/searchkick/index_options.rb

@@ -142,7 +142,8 @@ module Searchkick
           }
         }
 
-        if language == "chinese"
+        case language
+        when "chinese"
           settings[:analysis][:analyzer].merge!(
             default_analyzer => {
               type: "ik_smart"
@@ -156,6 +157,30 @@ module Searchkick
           )
 
           settings[:analysis][:filter].delete(:searchkick_stemmer)
+        when "japanese"
+          settings[:analysis][:analyzer].merge!(
+            default_analyzer => {
+              type: "kuromoji"
+            },
+            searchkick_search: {
+              type: "kuromoji"
+            },
+            searchkick_search2: {
+              type: "kuromoji"
+            }
+          )
+        when "polish", "ukrainian", "smartcn"
+          settings[:analysis][:analyzer].merge!(
+            default_analyzer => {
+              type: language
+            },
+            searchkick_search: {
+              type: language
+            },
+            searchkick_search2: {
+              type: language
+            }
+          )
         end
 
         if Searchkick.env == "test"

data/lib/searchkick/query.rb

@@ -314,10 +314,12 @@ module Searchkick
 
             if field == "_all" || field.end_with?(".analyzed")
               shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || misspellings == false
-              qs.concat [
-                shared_options.merge(analyzer: "searchkick_search"),
-                shared_options.merge(analyzer: "searchkick_search2")
-              ]
+              qs << shared_options.merge(analyzer: "searchkick_search")
+
+              # searchkick_search and searchkick_search2 are the same for ukrainian
+              unless %w(japanese polish ukrainian).include?(searchkick_options[:language])
+                qs << shared_options.merge(analyzer: "searchkick_search2")
+              end
               exclude_analyzer = "searchkick_search2"
             elsif field.end_with?(".exact")
               f = field.split(".")[0..-2].join(".")
@@ -850,7 +852,7 @@ module Searchkick
                 }
               when :regexp # support for regexp queries without using a regexp ruby object
                 filters << {regexp: {field => {value: op_value}}}
-              when :not # not equal
+              when :not, :_not # not equal
                 filters << {bool: {must_not: term_filters(field, op_value)}}
               when :all
                 op_value.each do |val|

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "3.0.0"
+  VERSION = "3.0.1"
 end

data/test/language_test.rb

@@ -2,15 +2,59 @@ require_relative "test_helper"
 
 class LanguageTest < Minitest::Test
   def setup
+    skip unless ENV["LANGUAGE"]
+
     Song.destroy_all
   end
 
   def test_chinese
-    skip unless ENV["CHINESE"]
-    Song.reindex
-    store_names ["中华人民共和国国歌"], Song
-    assert_search "中华人民共和国", ["中华人民共和国国歌"], {}, Song
-    assert_search "国歌", ["中华人民共和国国歌"], {}, Song
-    assert_search "人", [], {}, Song
+    # requires https://github.com/medcl/elasticsearch-analysis-ik
+    with_options(Song, language: "chinese") do
+      store_names ["中华人民共和国国歌"], Song
+      assert_language_search "中华人民共和国", ["中华人民共和国国歌"]
+      assert_language_search "国歌", ["中华人民共和国国歌"]
+      assert_language_search "人", []
+    end
+  end
+
+  # experimental
+  def test_smartcn
+    # requires https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-smartcn.html
+    with_options(Song, language: "smartcn") do
+      store_names ["中华人民共和国国歌"], Song
+      assert_language_search "中华人民共和国", ["中华人民共和国国歌"]
+      # assert_language_search "国歌", ["中华人民共和国国歌"]
+      assert_language_search "人", []
+    end
+  end
+
+  def test_japanese
+    # requires https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-kuromoji.html
+    with_options(Song, language: "japanese") do
+      store_names ["JR新宿駅の近くにビールを飲みに行こうか"], Song
+      assert_language_search "飲む", ["JR新宿駅の近くにビールを飲みに行こうか"]
+      assert_language_search "jr", ["JR新宿駅の近くにビールを飲みに行こうか"]
+      assert_language_search "新", []
+    end
+  end
+
+  def test_polish
+    # requires https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-stempel.html
+    with_options(Song, language: "polish") do
+      store_names ["polski"], Song
+      assert_language_search "polskimi", ["polski"]
+    end
+  end
+
+  def test_ukrainian
+    # requires https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-ukrainian.html
+    with_options(Song, language: "ukrainian") do
+      store_names ["ресторани"], Song
+      assert_language_search "ресторан", ["ресторани"]
+    end
+  end
+
+  def assert_language_search(term, expected)
+    assert_search term, expected, {misspellings: false}, Song
   end
 end

data/test/reindex_test.rb

@@ -1,6 +1,11 @@
 require_relative "test_helper"
 
 class ReindexTest < Minitest::Test
+  def setup
+    super
+    Sku.destroy_all
+  end
+
   def test_scoped
     skip if nobrainer? || cequel?
 

data/test/synonyms_test.rb

@@ -47,7 +47,9 @@ class SynonymsTest < Minitest::Test
   end
 
   def test_wordnet
-    skip unless ENV["TEST_WORDNET"]
+    # requires WordNet
+    skip unless ENV["WORDNET"]
+
     store_names ["Creature", "Beast", "Dragon"], Animal
     assert_search "animal", ["Creature", "Beast"], {}, Animal
   end

data/test/test_helper.rb

@@ -528,7 +528,7 @@ class Sku
 end
 
 class Song
-  searchkick language: "chinese"
+  searchkick
 end
 
 Product.searchkick_index.delete if Product.searchkick_index.exists?
@@ -547,7 +547,6 @@ class Minitest::Test
     Store.destroy_all
     Animal.destroy_all
     Speaker.destroy_all
-    Sku.destroy_all
   end
 
   protected
@@ -579,4 +578,16 @@ class Minitest::Test
   def assert_first(term, expected, options = {}, klass = Product)
     assert_equal expected, klass.search(term, options).map(&:name).first
   end
+
+  def with_options(klass, options)
+    previous_options = klass.searchkick_options.dup
+    begin
+      klass.searchkick_options.merge!(options)
+      klass.reindex
+      yield
+    ensure
+      klass.searchkick_options.clear
+      klass.searchkick_options.merge!(previous_options)
+    end
+  end
 end

data/test/where_test.rb

@@ -31,7 +31,9 @@ class WhereTest < Minitest::Test
     assert_search "product", ["Product A"], where: {store_id: 1...2}
     assert_search "product", ["Product A", "Product B"], where: {store_id: [1, 2]}
     assert_search "product", ["Product B", "Product C", "Product D"], where: {store_id: {not: 1}}
+    assert_search "product", ["Product B", "Product C", "Product D"], where: {store_id: {_not: 1}}
     assert_search "product", ["Product C", "Product D"], where: {store_id: {not: [1, 2]}}
+    assert_search "product", ["Product C", "Product D"], where: {store_id: {_not: [1, 2]}}
     assert_search "product", ["Product A"], where: {user_ids: {lte: 2, gte: 2}}
 
     # or
@@ -56,12 +58,15 @@ class WhereTest < Minitest::Test
 
     # any / nested terms
     assert_search "product", ["Product B", "Product C"], where: {user_ids: {not: [2], in: [1, 3]}}
+    assert_search "product", ["Product B", "Product C"], where: {user_ids: {_not: [2], in: [1, 3]}}
 
     # not / exists
     assert_search "product", ["Product D"], where: {user_ids: nil}
     assert_search "product", ["Product A", "Product B", "Product C"], where: {user_ids: {not: nil}}
+    assert_search "product", ["Product A", "Product B", "Product C"], where: {user_ids: {_not: nil}}
     assert_search "product", ["Product A", "Product C", "Product D"], where: {user_ids: [3, nil]}
     assert_search "product", ["Product B"], where: {user_ids: {not: [3, nil]}}
+    assert_search "product", ["Product B"], where: {user_ids: {_not: [3, nil]}}
   end
 
   def test_regexp

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.0.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-03 00:00:00.000000000 Z
+date: 2018-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel