searchkick 0.8.1 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dfbf6c27b23821e31c3de0338259ee8c2adf5130
-  data.tar.gz: f3b47e7a6d98f990f53d95410c88e2a2fc791b36
+  metadata.gz: 647df0215d082ec2eed37cabd3f1949aafc6c058
+  data.tar.gz: b3be78d876de64f7323fe31fbbe7829f7885ebc4
 SHA512:
-  metadata.gz: 1b0d6d2d11bde9c7a3f316a6ac6dc4886f639b8a02ddc6b49e280ae0535b38f7f43e53e1507528587cdc578b0b1ad165816aa23b89d4e21043a4e480790f181c
-  data.tar.gz: 39b962185cf9b507be764e1175e8bbe2d27822eed2005ad02923849577237c348d80b77e3ee8b3128676f7191ac43f23b2044075051751b532260a32597a877b
+  metadata.gz: 9326c663870b3dcf60384515ec29873a9b7c3ce6325922e3250ee0c7f75ebd933859d1a481275d035f2a8e00ac2b5c55ee0d342dfd55e698b575ca947f781f93
+  data.tar.gz: e6aaaa85a4ee9ccea8dba7bbd3077aef5164be1a5381ea60e1716881035050505575e1190494f9e53c1bef92b89852050b587a3fe3d84b4b35019eb9c8364ba8

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.8.2
+
+- Added `async` to `callbacks` option
+- Added `wordnet` option
+- Added `edit_distance` option to eventually replace `distance` option
+- Catch misspelling of `misspellings` option
+- Improved logging
+
 ## 0.8.1
 
 - Added `search_method_name` option

data/README.md

@@ -243,6 +243,27 @@ end
 
 Call `Product.reindex` after changing synonyms.
 
+### WordNet
+
+Prepopulate English synonyms with the [WordNet database](http://en.wikipedia.org/wiki/WordNet).
+
+Download [WordNet 3.0](http://wordnetcode.princeton.edu/3.0/WNprolog-3.0.tar.gz) to each Elasticsearch server and move `wn_s.pl` to the `/var/lib` directory.
+
+```sh
+cd /tmp
+curl -o wordnet.tar.gz http://wordnetcode.princeton.edu/3.0/WNprolog-3.0.tar.gz
+tar -zxvf wordnet.tar.gz
+mv prolog/wn_s.pl /var/lib
+```
+
+Tell each model to use it:
+
+```ruby
+class Product < ActiveRecord::Base
+  searchkick wordnet: true
+end
+```
+
 ### Misspellings
 
 By default, Searchkick handles misspelled queries by returning results with an [edit distance](http://en.wikipedia.org/wiki/Levenshtein_distance) of one. To turn off this feature, use:
@@ -254,7 +275,7 @@ Product.search "zuchini", misspellings: false # no zucchini
 You can also change the edit distance with:
 
 ```ruby
-Product.search "zucini", misspellings: {distance: 2} # zucchini
+Product.search "zucini", misspellings: {edit_distance: 2} # zucchini
 ```
 
 ### Indexing
@@ -303,7 +324,36 @@ end
 #### No need to reindex
 
 - App starts
-- Records are inserted, updated or deleted (syncs automatically)
+
+### Stay Synced
+
+There are three strategies for keeping the index synced with your database.
+
+1. Immediate (default)
+
+  Anytime a record is inserted, updated, or deleted
+
+2. Asynchronous
+
+  Use background jobs for better performance
+
+  ```ruby
+  class Product < ActiveRecord::Base
+    searchkick callbacks: :async
+  end
+  ```
+
+  Supports [Delayed Job](https://github.com/collectiveidea/delayed_job) only at the moment
+
+3. Manual
+
+  Turn off automatic syncing
+
+  ```ruby
+  class Product < ActiveRecord::Base
+    searchkick callbacks: false
+  end
+  ```
 
 ### Keep Getting Better
 
@@ -742,15 +792,7 @@ class Product < ActiveRecord::Base
 end
 ```
 
-Turn off callbacks permanently
-
-```ruby
-class Product < ActiveRecord::Base
-  searchkick callbacks: false
-end
-```
-
-or temporarily
+Turn off callbacks temporarily
 
 ```ruby
 Product.disable_search_callbacks # or use Searchkick.disable_callbacks for all models
@@ -794,25 +836,20 @@ class Product < ActiveRecord::Base
 end
 ```
 
-Asynchronous reindexing
+Reindex asynchronously
 
 ```ruby
 class Product < ActiveRecord::Base
   searchkick callbacks: false
 
-  # add the callbacks manually
-
-  # ActiveRecord - one callback
-  after_commit :reindex_async
-
-  # Mongoid - two callbacks
-  after_save :reindex_async
-  after_destroy :reindex_async
-
   def reindex_async
-    # delayed job
-    delay.reindex
+    # custom code to reindex
   end
+
+  after_commit :reindex_async
+  # or for Mongoid
+  # after_save :reindex_async
+  # after_destroy :reindex_async
 end
 ```
 

data/lib/searchkick.rb

@@ -7,6 +7,7 @@ require "searchkick/reindex"
 require "searchkick/results"
 require "searchkick/query"
 require "searchkick/similar"
+require "searchkick/reindex_job"
 require "searchkick/model"
 require "searchkick/tasks"
 require "searchkick/logging" if defined?(Rails)
@@ -17,9 +18,13 @@ module Searchkick
   class InvalidQueryError < Elasticsearch::Transport::Transport::Errors::BadRequest; end
 
   class << self
+    attr_accessor :callbacks
     attr_accessor :search_method_name
+    attr_accessor :wordnet_path
   end
+  self.callbacks = true
   self.search_method_name = :search
+  self.wordnet_path = "/var/lib/wn_s.pl"
 
   def self.client
     @client ||= Elasticsearch::Client.new(url: ENV["ELASTICSEARCH_URL"])
@@ -33,18 +38,16 @@ module Searchkick
     @server_version ||= client.info["version"]["number"]
   end
 
-  @callbacks = true
-
   def self.enable_callbacks
-    @callbacks = true
+    self.callbacks = true
   end
 
   def self.disable_callbacks
-    @callbacks = false
+    self.callbacks = false
   end
 
   def self.callbacks?
-    @callbacks
+    callbacks
   end
 end
 

data/lib/searchkick/logging.rb

@@ -11,10 +11,44 @@ module Searchkick
         execute_without_instrumentation
       end
     end
-
     alias_method_chain :execute, :instrumentation
   end
 
+  class Index
+    def store_with_instrumentation(record)
+      event = {
+        name: "#{record.searchkick_klass.name} Store",
+        id: search_id(record)
+      }
+      ActiveSupport::Notifications.instrument("request.searchkick", event) do
+        store_without_instrumentation(record)
+      end
+    end
+    alias_method_chain :store, :instrumentation
+
+    def remove_with_instrumentation(record)
+      event = {
+        name: "#{record.searchkick_klass.name} Remove",
+        id: search_id(record)
+      }
+      ActiveSupport::Notifications.instrument("request.searchkick", event) do
+        remove_without_instrumentation(record)
+      end
+    end
+    alias_method_chain :remove, :instrumentation
+
+    def import_with_instrumentation(records)
+      event = {
+        name: "#{records.first.searchkick_klass.name} Import",
+        count: records.size
+      }
+      ActiveSupport::Notifications.instrument("request.searchkick", event) do
+        import_without_instrumentation(records)
+      end
+    end
+    alias_method_chain :import, :instrumentation
+  end
+
   # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/log_subscriber.rb
   class LogSubscriber < ActiveSupport::LogSubscriber
     def self.runtime=(value)
@@ -43,6 +77,16 @@ module Searchkick
       host = Searchkick.client.transport.hosts.first
       debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map{|t| CGI.escape(t) }.join(",")}" : ""}/_search?pretty -d '#{payload[:query][:body].to_json}'"
     end
+
+    def request(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      debug "  #{color(name, YELLOW, true)}  #{payload.except(:name).to_json}"
+    end
   end
 
   # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/railties/controller_runtime.rb

data/lib/searchkick/model.rb

@@ -7,10 +7,12 @@ module Searchkick
       class_eval do
         cattr_reader :searchkick_options, :searchkick_env, :searchkick_klass
 
+        callbacks = options.has_key?(:callbacks) ? options[:callbacks] : true
+
         class_variable_set :@@searchkick_options, options.dup
         class_variable_set :@@searchkick_env, ENV["RACK_ENV"] || ENV["RAILS_ENV"] || "development"
         class_variable_set :@@searchkick_klass, self
-        class_variable_set :@@searchkick_callbacks, options[:callbacks] != false
+        class_variable_set :@@searchkick_callbacks, callbacks
         class_variable_set :@@searchkick_index, options[:index_name] || [options[:index_prefix], model_name.plural, searchkick_env].compact.join("_")
 
         def self.searchkick_index
@@ -30,11 +32,20 @@ module Searchkick
         extend Searchkick::Reindex
         include Searchkick::Similar
 
-        if respond_to?(:after_commit)
-          after_commit :reindex, if: proc{ self.class.search_callbacks? }
-        else
-          after_save :reindex, if: proc{ self.class.search_callbacks? }
-          after_destroy :reindex, if: proc{ self.class.search_callbacks? }
+        if callbacks == :async
+          def reindex_async
+            Delayed::Job.enqueue Searchkick::ReindexJob.new(self.class.name, id)
+          end
+        end
+
+        if callbacks
+          callback_name = callbacks == :async ? :reindex_async : :reindex
+          if respond_to?(:after_commit)
+            after_commit callback_name, if: proc{ self.class.search_callbacks? }
+          else
+            after_save callback_name, if: proc{ self.class.search_callbacks? }
+            after_destroy callback_name, if: proc{ self.class.search_callbacks? }
+          end
         end
 
         def self.enable_search_callbacks

data/lib/searchkick/query.rb

@@ -102,11 +102,12 @@ module Searchkick
                   shared_options.merge(boost: 10 * factor, analyzer: "searchkick_search"),
                   shared_options.merge(boost: 10 * factor, analyzer: "searchkick_search2")
                 ]
-                if options[:misspellings] != false
-                  distance = (options[:misspellings].is_a?(Hash) && options[:misspellings][:distance]) || 1
+                misspellings = options.has_key?(:misspellings) ? options[:misspellings] : options[:mispellings] # why not?
+                if misspellings != false
+                  edit_distance = (misspellings.is_a?(Hash) && (misspellings[:edit_distance] || misspellings[:distance])) || 1
                   qs.concat [
-                    shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search"),
-                    shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search2")
+                    shared_options.merge(fuzziness: edit_distance, max_expansions: 3, analyzer: "searchkick_search"),
+                    shared_options.merge(fuzziness: edit_distance, max_expansions: 3, analyzer: "searchkick_search2")
                   ]
                 end
               elsif field.end_with?(".exact")

data/lib/searchkick/reindex.rb

@@ -230,6 +230,17 @@ module Searchkick
           settings[:analysis][:analyzer][:default_index][:filter] << "searchkick_synonym"
         end
 
+        if options[:wordnet]
+          settings[:analysis][:filter][:searchkick_wordnet] = {
+            type: "synonym",
+            format: "wordnet",
+            synonyms_path: Searchkick.wordnet_path
+          }
+
+          settings[:analysis][:analyzer][:default_index][:filter].insert(4, "searchkick_wordnet")
+          settings[:analysis][:analyzer][:default_index][:filter] << "searchkick_wordnet"
+        end
+
         if options[:special_characters] == false
           settings[:analysis][:analyzer].each do |analyzer, analyzer_settings|
             analyzer_settings[:filter].reject!{|f| f == "asciifolding" }

data/lib/searchkick/reindex_job.rb

@@ -0,0 +1,28 @@
+module Searchkick
+  class ReindexJob
+
+    def initialize(klass, id)
+      @klass = klass
+      @id = id
+    end
+
+    def perform
+      model = @klass.constantize
+      record = model.find(@id) rescue nil # TODO fix lazy coding
+      index = model.searchkick_index
+      if !record or !record.should_index?
+        # hacky
+        record ||= model.new
+        record.id = @id
+        begin
+          index.remove record
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          # do nothing
+        end
+      else
+        index.store record
+      end
+    end
+
+  end
+end

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "0.8.1"
+  VERSION = "0.8.2"
 end

data/searchkick.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "elasticsearch", ">= 1"
   spec.add_dependency "hashie"
 
-  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "minitest", "~> 4.7"
+  spec.add_development_dependency "minitest"
 end

data/test/autocomplete_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestAutocomplete < Minitest::Unit::TestCase
+class TestAutocomplete < Minitest::Test
 
   def test_autocomplete
     store_names ["Hummus"]

data/test/boost_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestBoost < Minitest::Unit::TestCase
+class TestBoost < Minitest::Test
 
   # conversions
 

data/test/facets_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestFacets < Minitest::Unit::TestCase
+class TestFacets < Minitest::Test
 
   def setup
     super

data/test/highlight_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestHighlight < Minitest::Unit::TestCase
+class TestHighlight < Minitest::Test
 
   def test_basic
     store_names ["Two Door Cinema Club"]

data/test/index_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestIndex < Minitest::Unit::TestCase
+class TestIndex < Minitest::Test
 
   def test_clean_indices
     old_index = Searchkick::Index.new("products_test_20130801000000000")

data/test/inheritance_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestInheritance < Minitest::Unit::TestCase
+class TestInheritance < Minitest::Test
 
   def test_child_reindex
     store_names ["Max"], Cat

data/test/match_test.rb

@@ -2,7 +2,7 @@
 
 require_relative "test_helper"
 
-class TestMatch < Minitest::Unit::TestCase
+class TestMatch < Minitest::Test
 
   # exact
 

data/test/model_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestModel < Minitest::Unit::TestCase
+class TestModel < Minitest::Test
 
   def test_disable_callbacks_model
     store_names ["product a"]

data/test/query_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestQuery < Minitest::Unit::TestCase
+class TestQuery < Minitest::Test
 
   def test_basic
     store_names ["Milk", "Apple"]

data/test/reindex_job_test.rb

@@ -0,0 +1,33 @@
+require_relative "test_helper"
+
+class TestReindexJob < Minitest::Test
+
+  def setup
+    super
+    Searchkick.disable_callbacks
+  end
+
+  def teardown
+    Searchkick.enable_callbacks
+  end
+
+  def test_create
+    product = Product.create!(name: "Boom")
+    Product.searchkick_index.refresh
+    assert_search "*", []
+    Searchkick::ReindexJob.new("Product", product.id).perform
+    Product.searchkick_index.refresh
+    assert_search "*", ["Boom"]
+  end
+
+  def test_destroy
+    product = Product.create!(name: "Boom")
+    Product.reindex
+    assert_search "*", ["Boom"]
+    product.destroy
+    Searchkick::ReindexJob.new("Product", product.id).perform
+    Product.searchkick_index.refresh
+    assert_search "*", []
+  end
+
+end

data/test/should_index_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestShouldIndex < Minitest::Unit::TestCase
+class TestShouldIndex < Minitest::Test
 
   def test_basic
     store_names ["INDEX", "DO NOT INDEX"]

data/test/similar_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestSimilar < Minitest::Unit::TestCase
+class TestSimilar < Minitest::Test
 
   def test_similar
     store_names ["Annie's Naturals Organic Shiitake & Sesame Dressing"]

data/test/sql_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestSql < Minitest::Unit::TestCase
+class TestSql < Minitest::Test
 
   def test_limit
     store_names ["Product A", "Product B", "Product C", "Product D"]

data/test/suggest_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestSuggest < Minitest::Unit::TestCase
+class TestSuggest < Minitest::Test
 
   def test_basic
     store_names ["Great White Shark", "Hammerhead Shark", "Tiger Shark"]

data/test/synonyms_test.rb

@@ -1,6 +1,6 @@
 require_relative "test_helper"
 
-class TestSynonyms < Minitest::Unit::TestCase
+class TestSynonyms < Minitest::Test
 
   def test_bleach
     store_names ["Clorox Bleach", "Kroger Bleach"]
@@ -42,4 +42,9 @@ class TestSynonyms < Minitest::Unit::TestCase
     assert_search "scallions", ["Green Onions"]
   end
 
+  # def test_wordnet
+  #   store_names ["Creature", "Beast", "Dragon"], Animal
+  #   assert_search "animal", ["Creature", "Beast"], {}, Animal
+  # end
+
 end

data/test/test_helper.rb

@@ -6,6 +6,8 @@ require "logger"
 
 ENV["RACK_ENV"] = "test"
 
+Minitest::Test = Minitest::Unit::TestCase unless defined?(Minitest::Test)
+
 File.delete("elasticsearch.log") if File.exists?("elasticsearch.log")
 Searchkick.client.transport.logger = Logger.new("elasticsearch.log")
 
@@ -169,7 +171,11 @@ class Store
 end
 
 class Animal
-  searchkick autocomplete: [:name], suggest: [:name], index_name: -> { "#{self.name.tableize}-#{Date.today.year}" }
+  searchkick \
+    autocomplete: [:name],
+    suggest: [:name],
+    index_name: -> { "#{self.name.tableize}-#{Date.today.year}" }
+    # wordnet: true
 end
 
 Product.searchkick_index.delete if Product.searchkick_index.exists?
@@ -179,7 +185,7 @@ Product.reindex # run twice for both index paths
 Store.reindex
 Animal.reindex
 
-class Minitest::Unit::TestCase
+class Minitest::Test
 
   def setup
     Product.destroy_all

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-16 00:00:00.000000000 Z
+date: 2014-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -84,16 +84,16 @@ dependencies:
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '0'
 description: Intelligent search made easy
 email:
 - andrew@chartkick.com
@@ -117,8 +117,8 @@ files:
 - lib/searchkick/model.rb
 - lib/searchkick/query.rb
 - lib/searchkick/reindex.rb
+- lib/searchkick/reindex_job.rb
 - lib/searchkick/results.rb
-- lib/searchkick/search.rb
 - lib/searchkick/similar.rb
 - lib/searchkick/tasks.rb
 - lib/searchkick/version.rb
@@ -132,6 +132,7 @@ files:
 - test/match_test.rb
 - test/model_test.rb
 - test/query_test.rb
+- test/reindex_job_test.rb
 - test/should_index_test.rb
 - test/similar_test.rb
 - test/sql_test.rb
@@ -174,6 +175,7 @@ test_files:
 - test/match_test.rb
 - test/model_test.rb
 - test/query_test.rb
+- test/reindex_job_test.rb
 - test/should_index_test.rb
 - test/similar_test.rb
 - test/sql_test.rb

data/lib/searchkick/search.rb

@@ -1,14 +0,0 @@
-module Searchkick
-  module Search
-
-    def search(term = nil, options = {})
-      query = Searchkick::Query.new(self, term, options)
-      if options[:execute] == false
-        query
-      else
-        query.execute
-      end
-    end
-
-  end
-end