searchkick 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93348e8748debeefc0be073340665c5024b961af
-  data.tar.gz: a4c73f4e8147ba2120d7e3388205fa056128d392
+  metadata.gz: fdf75cc3dce90b3109799a5ee97e30c319d450ac
+  data.tar.gz: 646d884beeb278d6a056f662d9a9df6de2e0b6a4
 SHA512:
-  metadata.gz: 1f75f44631fee7348ff9aaf3d24f99b1954d5cef1868cc17f73c44c1dc17918f6eaa158dd35c46a582b31ec7d226384782421cd3c15f33b5a5e1b1f426266d90
-  data.tar.gz: 6be3ea5ed0ac0002b4628ad3cc067c0352b4724b9318836e2affe2c05c167132113e00b1ea8515f9e806054a57e3d7cb98a6f3c904f549885960fad72caa4e98
+  metadata.gz: 3f97d141a413008ecb2ce03db8c29bd09aa0127af473f40428d0c21b350e4a0266931e347164f978c695a25ebd4c26b57cfd921ce39768355821387ffccad52e
+  data.tar.gz: 91b030868ca63ad185a67b91062d682a414d942f1e17b6af3aa1ff5dd6392831c240507cf0eb18bf025f4dc3522a40b042d411420608df85a2de6227a4be5378

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.5.0
+
+- Better control over partial matches
+- Added merge_mappings option
+- Added batch_size option
+- Fixed bug with nil where clauses
+
 ## 0.4.2
 
 - Added `should_index?` method to control which records are indexed

data/README.md

@@ -148,6 +148,32 @@ To change this, use:
 Product.search "fresh honey", partial: true # fresh OR honey
 ```
 
+By default, results must match the entire word - `back` will not match `backpack`. You can change this behavior with:
+
+```ruby
+class Product < ActiveRecord::Base
+  searchkick word_start: [:name]
+end
+```
+
+And to search:
+
+```ruby
+Product.search "back", fields: [{name: :word_start}]
+```
+
+Available options are:
+
+```ruby
+:word # default
+:word_start
+:word_middle
+:word_end
+:text_start
+:text_middle
+:text_end
+```
+
 ### Synonyms
 
 ```ruby
@@ -293,14 +319,14 @@ First, specify which fields use this feature.  This is necessary since autocompl
 
 ```ruby
 class City < ActiveRecord::Base
-  searchkick autocomplete: ["name"]
+  searchkick text_start: [:name]
 end
 ```
 
 Reindex and search with:
 
 ```ruby
-City.search "san fr", autocomplete: true
+City.search "san fr", fields: [{name: :text_start}]
 ```
 
 Typically, you want to use a Javascript library like [typeahead.js](http://twitter.github.io/typeahead.js/) or [jQuery UI](http://jqueryui.com/autocomplete/).
@@ -314,7 +340,7 @@ First, add a controller action.
 class CitiesController < ApplicationController
 
   def autocomplete
-    render json: City.search(params[:query], autocomplete: true, limit: 10).map(&:name)
+    render json: City.search(params[:query], fields: [{name: :text_start}], limit: 10).map(&:name)
   end
 
 end
@@ -530,6 +556,14 @@ And use the `query` option to search:
 Product.search query: {match: {name: "milk"}}
 ```
 
+[master] To keep the mappings and settings generated by Searchkick, use:
+
+```ruby
+class Product < ActiveRecord::Base
+  searchkick merge_mappings: true, mappings: {...}
+end
+```
+
 ## Reference
 
 Searchkick requires Elasticsearch `0.90.0` or higher.
@@ -601,6 +635,14 @@ class Product < ActiveRecord::Base
 end
 ```
 
+Change import batch size [master]
+
+```ruby
+class Product < ActiveRecord::Base
+  searchkick batch_size: 200 # defaults to 1000
+end
+```
+
 Reindex all models (Rails only)
 
 ```sh

data/lib/searchkick/reindex.rb

@@ -57,11 +57,13 @@ module Searchkick
     private
 
     def searchkick_import(index)
+      batch_size = searchkick_options[:batch_size] || 1000
+
       # use scope for import
       scope = searchkick_klass
       scope = scope.search_import if scope.respond_to?(:search_import)
       if scope.respond_to?(:find_in_batches)
-        scope.find_in_batches do |batch|
+        scope.find_in_batches batch_size: batch_size do |batch|
           index.import batch.select{|item| item.should_index? }
         end
       else
@@ -70,7 +72,7 @@ module Searchkick
         items = []
         scope.all.each do |item|
           items << item if item.should_index?
-          if items.length % 1000 == 0
+          if items.length % batch_size == 0
             index.import items
             items = []
           end
@@ -82,7 +84,7 @@ module Searchkick
     def searchkick_index_options
       options = searchkick_options
 
-      if options[:mappings]
+      if options[:mappings] and !options[:merge_mappings]
         settings = options[:settings] || {}
         mappings = options[:mappings]
       else
@@ -126,6 +128,41 @@ module Searchkick
                 type: "custom",
                 tokenizer: "standard",
                 filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
+              },
+              searchkick_suggest_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
+              },
+              searchkick_text_start_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+              },
+              searchkick_text_middle_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+              },
+              searchkick_text_end_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
+              },
+              searchkick_word_start_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+              },
+              searchkick_word_middle_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+              },
+              searchkick_word_end_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
               }
             },
             filter: {
@@ -143,6 +180,16 @@ module Searchkick
               searchkick_suggest_shingle: {
                 type: "shingle",
                 max_shingle_size: 5
+              },
+              searchkick_edge_ngram: {
+                type: "edgeNGram",
+                min_gram: 1,
+                max_gram: 50
+              },
+              searchkick_ngram: {
+                type: "nGram",
+                min_gram: 1,
+                max_gram: 50
               }
             },
             tokenizer: {
@@ -159,7 +206,7 @@ module Searchkick
           settings.merge!(number_of_shards: 1, number_of_replicas: 0)
         end
 
-        settings.merge!(options[:settings] || {})
+        settings.deep_merge!(options[:settings] || {})
 
         # synonyms
         synonyms = options[:synonyms] || []
@@ -200,10 +247,12 @@ module Searchkick
           }
         end
 
-        # autocomplete and suggest
-        autocomplete = (options[:autocomplete] || []).map(&:to_s)
-        suggest = (options[:suggest] || []).map(&:to_s)
-        (autocomplete + suggest).uniq.each do |field|
+        mapping_options = Hash[
+          [:autocomplete, :suggest, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end]
+            .map{|type| [type, (options[type] || []).map(&:to_s)] }
+        ]
+
+        mapping_options.values.flatten.uniq.each do |field|
           field_mapping = {
             type: "multi_field",
             fields: {
@@ -213,12 +262,13 @@ module Searchkick
               # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-highlighting.html#_fast_vector_highlighter
             }
           }
-          if autocomplete.include?(field)
-            field_mapping[:fields]["autocomplete"] = {type: "string", index: "analyzed", analyzer: "searchkick_autocomplete_index"}
-          end
-          if suggest.include?(field)
-            field_mapping[:fields]["suggest"] = {type: "string", index: "analyzed", analyzer: "searchkick_suggest_index"}
+
+          mapping_options.each do |type, fields|
+            if fields.include?(field)
+              field_mapping[:fields][type] = {type: "string", index: "analyzed", analyzer: "searchkick_#{type}_index"}
+            end
           end
+
           mapping[field] = field_mapping
         end
 
@@ -253,7 +303,7 @@ module Searchkick
               }
             ]
           }
-        }
+        }.deep_merge(options[:mappings] || {})
       end
 
       {

data/lib/searchkick/search.rb

@@ -14,7 +14,10 @@ module Searchkick
           if options[:autocomplete]
             options[:fields].map{|f| "#{f}.autocomplete" }
           else
-            options[:fields].map{|f| "#{f}.analyzed" }
+            options[:fields].map do |value|
+              k, v = value.is_a?(Hash) ? value.to_a.first : [value, :word]
+              "#{k}.#{v == :word ? "analyzed" : v}"
+            end
           end
         else
           if options[:autocomplete]
@@ -67,23 +70,37 @@ module Searchkick
             }
           }
         else
-          shared_options = {
-            fields: fields,
-            query: term,
-            use_dis_max: false,
-            operator: operator
-          }
-          queries = [
-            {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search")},
-            {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search2")}
-          ]
-          if options[:misspellings] != false
-            distance = (options[:misspellings].is_a?(Hash) && options[:misspellings][:distance]) || 1
-            queries.concat [
-              {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search")},
-              {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search2")}
-            ]
+          queries = []
+          fields.each do |field|
+            if field == "_all" or field.end_with?(".analyzed")
+              shared_options = {
+                fields: [field],
+                query: term,
+                use_dis_max: false,
+                operator: operator
+              }
+              queries.concat [
+                {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search")},
+                {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search2")}
+              ]
+              if options[:misspellings] != false
+                distance = (options[:misspellings].is_a?(Hash) && options[:misspellings][:distance]) || 1
+                queries.concat [
+                  {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search")},
+                  {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search2")}
+                ]
+              end
+            else
+              queries << {
+                multi_match: {
+                  fields: [field],
+                  query: term,
+                  analyzer: "searchkick_autocomplete_search"
+                }
+              }
+            end
           end
+
           payload = {
             dis_max: {
               queries: queries
@@ -164,6 +181,17 @@ module Searchkick
         payload[:sort] = order
       end
 
+      term_filters =
+        proc do |field, value|
+          if value.is_a?(Array) # in query
+            {or: value.map{|v| term_filters.call(field, v) }}
+          elsif value.nil?
+            {missing: {"field" => field, existence: true, null_value: true}}
+          else
+            {term: {field => value}}
+          end
+        end
+
       # where
       where_filters =
         proc do |where|
@@ -181,9 +209,7 @@ module Searchkick
                 value = {gte: value.first, (value.exclude_end? ? :lt : :lte) => value.last}
               end
 
-              if value.is_a?(Array) # in query
-                filters << {terms: {field => value}}
-              elsif value.is_a?(Hash)
+              if value.is_a?(Hash)
                 if value[:near]
                   filters << {
                     geo_distance: {
@@ -206,11 +232,7 @@ module Searchkick
 
                 value.each do |op, op_value|
                   if op == :not # not equal
-                    if op_value.is_a?(Array)
-                      filters << {not: {terms: {field => op_value}}}
-                    else
-                      filters << {not: {term: {field => op_value}}}
-                    end
+                    filters << {not: term_filters.call(field, op_value)}
                   elsif op == :all
                     filters << {terms: {field => op_value, execution: "and"}}
                   else
@@ -231,11 +253,7 @@ module Searchkick
                   end
                 end
               else
-                if value.nil?
-                  filters << {missing: {"field" => field, existence: true, null_value: true}}
-                else
-                  filters << {term: {field => value}}
-                end
+                filters << term_filters.call(field, value)
               end
             end
           end

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

data/test/autocomplete_test.rb

@@ -17,4 +17,34 @@ class TestAutocomplete < Minitest::Unit::TestCase
     assert_search "hum", ["Hummus"], autocomplete: true, fields: [:name]
   end
 
+  def test_text_start
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "whe", ["Where in the World is Carmen San Diego?"], fields: [{name: :text_start}]
+  end
+
+  def test_text_middle
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "n the wor", ["Where in the World is Carmen San Diego?"], fields: [{name: :text_middle}]
+  end
+
+  def test_text_end
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "ego?", ["Where in the World is Carmen San Diego?"], fields: [{name: :text_end}]
+  end
+
+  def test_word_start
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "car", ["Where in the World is Carmen San Diego?"], fields: [{name: :word_start}]
+  end
+
+  def test_word_middle
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "orl", ["Where in the World is Carmen San Diego?"], fields: [{name: :word_middle}]
+  end
+
+  def test_word_end
+    store_names ["Where in the World is Carmen San Diego?"]
+    assert_search "men", ["Where in the World is Carmen San Diego?"], fields: [{name: :word_end}]
+  end
+
 end

data/test/sql_test.rb

@@ -67,6 +67,11 @@ class TestSql < Minitest::Unit::TestCase
     # all
     assert_search "product", ["Product A"], where: {user_ids: {all: [1, 3]}}
     assert_search "product", [], where: {user_ids: {all: [1, 2, 3, 4]}}
+    # not / exists
+    assert_search "product", ["Product C", "Product D"], where: {user_ids: nil}
+    assert_search "product", ["Product A", "Product B"], where: {user_ids: {not: nil}}
+    assert_search "product", ["Product A", "Product C", "Product D"], where: {user_ids: [3, nil]}
+    assert_search "product", ["Product B"], where: {user_ids: {not: [3, nil]}}
   end
 
   def test_where_string

data/test/test_helper.rb

@@ -115,7 +115,13 @@ class Product
     suggest: [:name, :color],
     conversions: "conversions",
     personalize: "user_ids",
-    locations: ["location", "multiple_locations"]
+    locations: ["location", "multiple_locations"],
+    text_start: [:name],
+    text_middle: [:name],
+    text_end: [:name],
+    word_start: [:name],
+    word_middle: [:name],
+    word_end: [:name]
 
   attr_accessor :conversions, :user_ids
 

metadata

@@ -1,111 +1,111 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-29 00:00:00.000000000 Z
+date: 2014-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tire
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: tire-contrib
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.7'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Search made easy
@@ -115,8 +115,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -158,17 +158,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.2.0
 signing_key: 
 specification_version: 4
 summary: Search made easy