searchkick 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84aa6414f91ce1c76d0c247e9a11dfce4417c2ee
-  data.tar.gz: f072c08de08bac3808112ad165199b98b5b2e507
+  metadata.gz: 93348e8748debeefc0be073340665c5024b961af
+  data.tar.gz: a4c73f4e8147ba2120d7e3388205fa056128d392
 SHA512:
-  metadata.gz: b866b6ba7f26bca01b13043c42ae1e9917b664be9bf77ce0c269dde30298fc8d2a08a54ce247dd465e500ebe8e122e804685ab4855984c1a09088b46422900e7
-  data.tar.gz: ab9bae0de9a80123cb85e0a34f221b6dce56efc91471c3037fbce10b2c7e284635291704965c1894c419e3b7610da8e1ffa4d83cde4a76039a530ef0b26e7407
+  metadata.gz: 1f75f44631fee7348ff9aaf3d24f99b1954d5cef1868cc17f73c44c1dc17918f6eaa158dd35c46a582b31ec7d226384782421cd3c15f33b5a5e1b1f426266d90
+  data.tar.gz: 6be3ea5ed0ac0002b4628ad3cc067c0352b4724b9318836e2affe2c05c167132113e00b1ea8515f9e806054a57e3d7cb98a6f3c904f549885960fad72caa4e98

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.4.2
+
+- Added `should_index?` method to control which records are indexed
+- Added ability to temporarily disable callbacks
+- Added custom mappings
+
 ## 0.4.1
 
 - Fixed issue w/ inheritance mapping

data/README.md

@@ -72,6 +72,8 @@ Query like SQL
 Product.search "2% Milk", where: {in_stock: true}, limit: 10, offset: 50
 ```
 
+**Note:** If you prefer the Elasticsearch DSL, see the [Advanced section](#advanced)
+
 Search specific fields
 
 ```ruby
@@ -195,6 +197,16 @@ class Product < ActiveRecord::Base
 end
 ```
 
+By default, all records are indexed.  To control which records are indexed, use the `should_index?` method.
+
+```ruby
+class Product < ActiveRecord::Base
+  def should_index?
+    active # only index active records
+  end
+end
+```
+
 ### To Reindex, or Not to Reindex
 
 #### Reindex
@@ -494,6 +506,30 @@ Then deploy and reindex:
 rake searchkick:reindex CLASS=Product
 ```
 
+## Advanced
+
+Prefer to use the [Elasticsearch DSL](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-queries.html) but still want awesome features like zero-downtime reindexing?
+
+Create a custom mapping:
+
+```ruby
+class Product < ActiveRecord::Base
+  searchkick mappings: {
+    product: {
+      properties: {
+        name: {type: "string", analyzer: "keyword"}
+      }
+    }
+  }
+end
+```
+
+And use the `query` option to search:
+
+```ruby
+Product.search query: {match: {name: "milk"}}
+```
+
 ## Reference
 
 Searchkick requires Elasticsearch `0.90.0` or higher.
@@ -527,7 +563,7 @@ class Product < ActiveRecord::Base
 end
 ```
 
-Turn off callbacks
+Turn off callbacks permanently
 
 ```ruby
 class Product < ActiveRecord::Base
@@ -535,6 +571,15 @@ class Product < ActiveRecord::Base
 end
 ```
 
+or temporarily
+
+```ruby
+Product.disable_search_callbacks # use Searchkick.disable_callbacks for all models
+ExpensiveProductsTask.execute
+Product.enable_search_callbacks
+Product.reindex
+```
+
 Eager load associations
 
 ```ruby

data/lib/searchkick/model.rb

@@ -8,6 +8,7 @@ module Searchkick
         class_variable_set :@@searchkick_options, options.dup
         class_variable_set :@@searchkick_env, ENV["RACK_ENV"] || ENV["RAILS_ENV"] || "development"
         class_variable_set :@@searchkick_klass, self
+        class_variable_set :@@searchkick_callbacks, options[:callbacks] != false
 
         # set index name
         # TODO support proc
@@ -18,19 +19,34 @@ module Searchkick
         extend Searchkick::Reindex
         include Searchkick::Similar
 
-        def reindex
-          index = self.class.searchkick_index
-          if destroyed?
-            index.remove self
-          else
-            index.store self
-          end
+        after_save :reindex
+        after_destroy :reindex
+
+        def self.enable_search_callbacks
+          class_variable_set :@@searchkick_callbacks, true
+        end
+
+        def self.disable_search_callbacks
+          class_variable_set :@@searchkick_callbacks, false
+        end
+
+        def self.search_callbacks?
+          class_variable_get(:@@searchkick_callbacks) && Searchkick.callbacks?
+        end
+
+        def should_index?
+          true
         end
 
-        unless options[:callbacks] == false
-          # TODO ability to temporarily disable
-          after_save :reindex
-          after_destroy :reindex
+        def reindex
+          if self.class.search_callbacks?
+            index = self.class.searchkick_index
+            if destroyed? or !should_index?
+              index.remove self
+            else
+              index.store self
+            end
+          end
         end
 
         def search_data

data/lib/searchkick/reindex.rb

@@ -62,14 +62,14 @@ module Searchkick
       scope = scope.search_import if scope.respond_to?(:search_import)
       if scope.respond_to?(:find_in_batches)
         scope.find_in_batches do |batch|
-          index.import batch
+          index.import batch.select{|item| item.should_index? }
         end
       else
         # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
         # use cursor for Mongoid
         items = []
         scope.all.each do |item|
-          items << item
+          items << item if item.should_index?
           if items.length % 1000 == 0
             index.import items
             items = []
@@ -82,174 +82,179 @@ module Searchkick
     def searchkick_index_options
       options = searchkick_options
 
-      settings = {
-        analysis: {
-          analyzer: {
-            searchkick_keyword: {
-              type: "custom",
-              tokenizer: "keyword",
-              filter: ["lowercase", "snowball"]
-            },
-            default_index: {
-              type: "custom",
-              tokenizer: "standard",
-              # synonym should come last, after stemming and shingle
-              # shingle must come before snowball
-              filter: ["standard", "lowercase", "asciifolding", "stop", "searchkick_index_shingle", "snowball"]
-            },
-            searchkick_search: {
-              type: "custom",
-              tokenizer: "standard",
-              filter: ["standard", "lowercase", "asciifolding", "stop", "searchkick_search_shingle", "snowball"]
-            },
-            searchkick_search2: {
-              type: "custom",
-              tokenizer: "standard",
-              filter: ["standard", "lowercase", "asciifolding", "stop", "snowball"]
-            },
-            # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
-            searchkick_autocomplete_index: {
-              type: "custom",
-              tokenizer: "searchkick_autocomplete_ngram",
-              filter: ["lowercase", "asciifolding"]
-            },
-            searchkick_autocomplete_search: {
-              type: "custom",
-              tokenizer: "keyword",
-              filter: ["lowercase", "asciifolding"]
-            },
-            searchkick_suggest_index: {
-              type: "custom",
-              tokenizer: "standard",
-              filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
-            }
-          },
-          filter: {
-            searchkick_index_shingle: {
-              type: "shingle",
-              token_separator: ""
+      if options[:mappings]
+        settings = options[:settings] || {}
+        mappings = options[:mappings]
+      else
+        settings = {
+          analysis: {
+            analyzer: {
+              searchkick_keyword: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "snowball"]
+              },
+              default_index: {
+                type: "custom",
+                tokenizer: "standard",
+                # synonym should come last, after stemming and shingle
+                # shingle must come before snowball
+                filter: ["standard", "lowercase", "asciifolding", "stop", "searchkick_index_shingle", "snowball"]
+              },
+              searchkick_search: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["standard", "lowercase", "asciifolding", "stop", "searchkick_search_shingle", "snowball"]
+              },
+              searchkick_search2: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["standard", "lowercase", "asciifolding", "stop", "snowball"]
+              },
+              # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
+              searchkick_autocomplete_index: {
+                type: "custom",
+                tokenizer: "searchkick_autocomplete_ngram",
+                filter: ["lowercase", "asciifolding"]
+              },
+              searchkick_autocomplete_search: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding"]
+              },
+              searchkick_suggest_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
+              }
             },
-            # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7
-            searchkick_search_shingle: {
-              type: "shingle",
-              token_separator: "",
-              output_unigrams: false,
-              output_unigrams_if_no_shingles: true
+            filter: {
+              searchkick_index_shingle: {
+                type: "shingle",
+                token_separator: ""
+              },
+              # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7
+              searchkick_search_shingle: {
+                type: "shingle",
+                token_separator: "",
+                output_unigrams: false,
+                output_unigrams_if_no_shingles: true
+              },
+              searchkick_suggest_shingle: {
+                type: "shingle",
+                max_shingle_size: 5
+              }
             },
-            searchkick_suggest_shingle: {
-              type: "shingle",
-              max_shingle_size: 5
-            }
-          },
-          tokenizer: {
-            searchkick_autocomplete_ngram: {
-              type: "edgeNGram",
-              min_gram: 1,
-              max_gram: 50
+            tokenizer: {
+              searchkick_autocomplete_ngram: {
+                type: "edgeNGram",
+                min_gram: 1,
+                max_gram: 50
+              }
             }
           }
         }
-      }
 
-      if searchkick_env == "test"
-        settings.merge!(number_of_shards: 1, number_of_replicas: 0)
-      end
+        if searchkick_env == "test"
+          settings.merge!(number_of_shards: 1, number_of_replicas: 0)
+        end
 
-      settings.merge!(options[:settings] || {})
+        settings.merge!(options[:settings] || {})
 
-      # synonyms
-      synonyms = options[:synonyms] || []
-      if synonyms.any?
-        settings[:analysis][:filter][:searchkick_synonym] = {
-          type: "synonym",
-          synonyms: synonyms.select{|s| s.size > 1 }.map{|s| s.join(",") }
-        }
-        # choosing a place for the synonym filter when stemming is not easy
-        # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
-        # TODO use a snowball stemmer on synonyms when creating the token filter
+        # synonyms
+        synonyms = options[:synonyms] || []
+        if synonyms.any?
+          settings[:analysis][:filter][:searchkick_synonym] = {
+            type: "synonym",
+            synonyms: synonyms.select{|s| s.size > 1 }.map{|s| s.join(",") }
+          }
+          # choosing a place for the synonym filter when stemming is not easy
+          # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
+          # TODO use a snowball stemmer on synonyms when creating the token filter
 
-        # http://elasticsearch-users.115913.n3.nabble.com/synonym-multi-words-search-td4030811.html
-        # I find the following approach effective if you are doing multi-word synonyms (synonym phrases):
-        # - Only apply the synonym expansion at index time
-        # - Don't have the synonym filter applied search
-        # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
-        settings[:analysis][:analyzer][:default_index][:filter].insert(4, "searchkick_synonym")
-        settings[:analysis][:analyzer][:default_index][:filter] << "searchkick_synonym"
-      end
+          # http://elasticsearch-users.115913.n3.nabble.com/synonym-multi-words-search-td4030811.html
+          # I find the following approach effective if you are doing multi-word synonyms (synonym phrases):
+          # - Only apply the synonym expansion at index time
+          # - Don't have the synonym filter applied search
+          # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
+          settings[:analysis][:analyzer][:default_index][:filter].insert(4, "searchkick_synonym")
+          settings[:analysis][:analyzer][:default_index][:filter] << "searchkick_synonym"
+        end
 
-      if options[:special_characters] == false
-        settings[:analysis][:analyzer].each do |analyzer, analyzer_settings|
-          analyzer_settings[:filter].reject!{|f| f == "asciifolding" }
+        if options[:special_characters] == false
+          settings[:analysis][:analyzer].each do |analyzer, analyzer_settings|
+            analyzer_settings[:filter].reject!{|f| f == "asciifolding" }
+          end
         end
-      end
 
-      mapping = {}
+        mapping = {}
 
-      # conversions
-      if options[:conversions]
-        mapping[:conversions] = {
-          type: "nested",
-          properties: {
-            query: {type: "string", analyzer: "searchkick_keyword"},
-            count: {type: "integer"}
+        # conversions
+        if options[:conversions]
+          mapping[:conversions] = {
+            type: "nested",
+            properties: {
+              query: {type: "string", analyzer: "searchkick_keyword"},
+              count: {type: "integer"}
+            }
           }
-        }
-      end
+        end
 
-      # autocomplete and suggest
-      autocomplete = (options[:autocomplete] || []).map(&:to_s)
-      suggest = (options[:suggest] || []).map(&:to_s)
-      (autocomplete + suggest).uniq.each do |field|
-        field_mapping = {
-          type: "multi_field",
-          fields: {
-            field => {type: "string", index: "not_analyzed"},
-            "analyzed" => {type: "string", index: "analyzed"}
-            # term_vector: "with_positions_offsets" for fast / correct highlighting
-            # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-highlighting.html#_fast_vector_highlighter
+        # autocomplete and suggest
+        autocomplete = (options[:autocomplete] || []).map(&:to_s)
+        suggest = (options[:suggest] || []).map(&:to_s)
+        (autocomplete + suggest).uniq.each do |field|
+          field_mapping = {
+            type: "multi_field",
+            fields: {
+              field => {type: "string", index: "not_analyzed"},
+              "analyzed" => {type: "string", index: "analyzed"}
+              # term_vector: "with_positions_offsets" for fast / correct highlighting
+              # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-highlighting.html#_fast_vector_highlighter
+            }
           }
-        }
-        if autocomplete.include?(field)
-          field_mapping[:fields]["autocomplete"] = {type: "string", index: "analyzed", analyzer: "searchkick_autocomplete_index"}
-        end
-        if suggest.include?(field)
-          field_mapping[:fields]["suggest"] = {type: "string", index: "analyzed", analyzer: "searchkick_suggest_index"}
+          if autocomplete.include?(field)
+            field_mapping[:fields]["autocomplete"] = {type: "string", index: "analyzed", analyzer: "searchkick_autocomplete_index"}
+          end
+          if suggest.include?(field)
+            field_mapping[:fields]["suggest"] = {type: "string", index: "analyzed", analyzer: "searchkick_suggest_index"}
+          end
+          mapping[field] = field_mapping
         end
-        mapping[field] = field_mapping
-      end
 
-      (options[:locations] || []).each do |field|
-        mapping[field] = {
-          type: "geo_point"
-        }
-      end
+        (options[:locations] || []).each do |field|
+          mapping[field] = {
+            type: "geo_point"
+          }
+        end
 
-      mappings = {
-        _default_: {
-          properties: mapping,
-          # https://gist.github.com/kimchy/2898285
-          dynamic_templates: [
-            {
-              string_template: {
-                match: "*",
-                match_mapping_type: "string",
-                mapping: {
-                  # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
-                  type: "multi_field",
-                  fields: {
-                    # analyzed field must be the default field for include_in_all
+        mappings = {
+          _default_: {
+            properties: mapping,
+            # https://gist.github.com/kimchy/2898285
+            dynamic_templates: [
+              {
+                string_template: {
+                  match: "*",
+                  match_mapping_type: "string",
+                  mapping: {
                     # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
-                    # however, we can include the not_analyzed field in _all
-                    # and the _all index analyzer will take care of it
-                    "{name}" => {type: "string", index: "not_analyzed"},
-                    "analyzed" => {type: "string", index: "analyzed"}
+                    type: "multi_field",
+                    fields: {
+                      # analyzed field must be the default field for include_in_all
+                      # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
+                      # however, we can include the not_analyzed field in _all
+                      # and the _all index analyzer will take care of it
+                      "{name}" => {type: "string", index: "not_analyzed"},
+                      "analyzed" => {type: "string", index: "analyzed"}
+                    }
                   }
                 }
               }
-            }
-          ]
+            ]
+          }
         }
-      }
+      end
 
       {
         settings: settings,

data/lib/searchkick/search.rb

@@ -2,7 +2,13 @@ module Searchkick
   module Search
 
     def search(term, options = {})
-      term = term.to_s
+      if term.is_a?(Hash)
+        options = term
+        term = nil
+      else
+        term = term.to_s
+      end
+
       fields =
         if options[:fields]
           if options[:autocomplete]
@@ -35,7 +41,9 @@ module Searchkick
 
       all = term == "*"
 
-      if options[:similar]
+      if options[:query]
+        payload = options[:query]
+      elsif options[:similar]
         payload = {
           more_like_this: {
             fields: fields,

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
 end

data/lib/searchkick.rb

@@ -8,6 +8,22 @@ require "searchkick/model"
 require "searchkick/tasks"
 require "searchkick/logger" if defined?(Rails)
 
+module Searchkick
+  @callbacks = true
+
+  def self.enable_callbacks
+    @callbacks = true
+  end
+
+  def self.disable_callbacks
+    @callbacks = false
+  end
+
+  def self.callbacks?
+    @callbacks
+  end
+end
+
 # TODO find better ActiveModel hook
 ActiveModel::Callbacks.send(:include, Searchkick::Model)
 ActiveRecord::Base.send(:extend, Searchkick::Model) if defined?(ActiveRecord)

data/test/index_test.rb

@@ -26,4 +26,10 @@ class TestIndex < Minitest::Unit::TestCase
     assert !old_index.exists?
   end
 
+  def test_mapping
+    store_names ["Dollar Tree"], Store
+    assert_equal [], Store.search(query: {match: {name: "dollar"}}).map(&:name)
+    assert_equal ["Dollar Tree"], Store.search(query: {match: {name: "Dollar Tree"}}).map(&:name)
+  end
+
 end

data/test/inheritance_test.rb

@@ -2,11 +2,6 @@ require_relative "test_helper"
 
 class TestInheritance < Minitest::Unit::TestCase
 
-  def setup
-    super
-    Animal.destroy_all
-  end
-
   def test_child_reindex
     store_names ["Max"], Cat
     assert Dog.reindex

data/test/model_test.rb

@@ -0,0 +1,35 @@
+require_relative "test_helper"
+
+class TestModel < Minitest::Unit::TestCase
+
+  def test_disable_callbacks_model
+    store_names ["product a"]
+
+    Product.disable_search_callbacks
+    assert !Product.search_callbacks?
+
+    store_names ["product b"]
+    assert_search "product", ["product a"]
+
+    Product.enable_search_callbacks
+    Product.reindex
+
+    assert_search "product", ["product a", "product b"]
+  end
+
+  def test_disable_callbacks_global
+    store_names ["product a"]
+
+    Searchkick.disable_callbacks
+    assert !Searchkick.callbacks?
+
+    store_names ["product b"]
+    assert_search "product", ["product a"]
+
+    Searchkick.enable_callbacks
+    Product.reindex
+
+    assert_search "product", ["product a", "product b"]
+  end
+
+end

data/test/should_index_test.rb

@@ -0,0 +1,34 @@
+require_relative "test_helper"
+
+class TestShouldIndex < Minitest::Unit::TestCase
+
+  def test_basic
+    store_names ["INDEX", "DO NOT INDEX"]
+    assert_search "index", ["INDEX"]
+  end
+
+  def test_default_true
+    assert Animal.new.should_index?
+  end
+
+  def test_change_to_true
+    store_names ["DO NOT INDEX"]
+    assert_search "index", []
+    product = Product.first
+    product.name = "INDEX"
+    product.save!
+    Product.searchkick_index.refresh
+    assert_search "index", ["INDEX"]
+  end
+
+  def test_change_to_false
+    store_names ["INDEX"]
+    assert_search "index", ["INDEX"]
+    product = Product.first
+    product.name = "DO NOT INDEX"
+    product.save!
+    Product.searchkick_index.refresh
+    assert_search "index", []
+  end
+
+end

data/test/test_helper.rb

@@ -33,6 +33,8 @@ if defined?(Mongoid)
 
   class Store
     include Mongoid::Document
+
+    field :name
   end
 
   class Animal
@@ -73,6 +75,7 @@ else
   end
 
   ActiveRecord::Migration.create_table :stores, :force => true do |t|
+    t.string :name
   end
 
   ActiveRecord::Migration.create_table :animals, :force => true do |t|
@@ -119,6 +122,20 @@ class Product
   def search_data
     serializable_hash.merge conversions: conversions, user_ids: user_ids, location: [latitude, longitude], multiple_locations: [[latitude, longitude], [0, 0]]
   end
+
+  def should_index?
+    name != "DO NOT INDEX"
+  end
+end
+
+class Store
+  searchkick mappings: {
+    store: {
+      properties: {
+        name: {type: "string", analyzer: "keyword"}
+      }
+    }
+  }
 end
 
 class Animal
@@ -129,12 +146,14 @@ Product.searchkick_index.delete if Product.searchkick_index.exists?
 Product.reindex
 Product.reindex # run twice for both index paths
 
+Store.reindex
 Animal.reindex
 
 class Minitest::Unit::TestCase
 
   def setup
     Product.destroy_all
+    Animal.destroy_all
   end
 
   protected
@@ -151,16 +170,16 @@ class Minitest::Unit::TestCase
   end
 
   # no order
-  def assert_search(term, expected, options = {})
-    assert_equal expected.sort, Product.search(term, options).map(&:name).sort
+  def assert_search(term, expected, options = {}, klass = Product)
+    assert_equal expected.sort, klass.search(term, options).map(&:name).sort
   end
 
-  def assert_order(term, expected, options = {})
-    assert_equal expected, Product.search(term, options).map(&:name)
+  def assert_order(term, expected, options = {}, klass = Product)
+    assert_equal expected, klass.search(term, options).map(&:name)
   end
 
-  def assert_first(term, expected, options = {})
-    assert_equal expected, Product.search(term, options).map(&:name).first
+  def assert_first(term, expected, options = {}, klass = Product)
+    assert_equal expected, klass.search(term, options).map(&:name).first
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-20 00:00:00.000000000 Z
+date: 2013-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tire
@@ -141,6 +141,8 @@ files:
 - test/index_test.rb
 - test/inheritance_test.rb
 - test/match_test.rb
+- test/model_test.rb
+- test/should_index_test.rb
 - test/similar_test.rb
 - test/sql_test.rb
 - test/suggest_test.rb
@@ -178,6 +180,8 @@ test_files:
 - test/index_test.rb
 - test/inheritance_test.rb
 - test/match_test.rb
+- test/model_test.rb
+- test/should_index_test.rb
 - test/similar_test.rb
 - test/sql_test.rb
 - test/suggest_test.rb