searchkick 0.8.5 → 0.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c50ef339d04f201523ff1f41aad11277d7168dab
-  data.tar.gz: d7f79bf282327a781e23475f64b7530dfac6beab
+  metadata.gz: 11adf1d76060b33e1a52d9a02f078d70e6b23396
+  data.tar.gz: 913a8ac5be964f35969ff71b59a10e5911d16680
 SHA512:
-  metadata.gz: 61072c05e3aa82903b699e2062dc0a2944f7fcec20545709d2b83f1f0a760778adffbc7cf7002815c56c2ba6f9484de13f35134c484842ebbf56785d549a6883
-  data.tar.gz: 5c95dd8df5ead5050d2b24175e01b6ad197d7f430f92554f2330cb3b0976360ab4182d956e62ae734ce38b77985eb84f6c0e7e95a0d147d18f061e0964ee61e6
+  metadata.gz: 3b99448bfc3469410428abd5af56ce31ba632d0ebd0b8a9e8990a17f553e776995915925d2e6f17e173dae7576429cfe51d93a865f182322d48e2a4f0b3984a2
+  data.tar.gz: 4322539cd2b8e07d5eb0db03fc6af6dff1cd9f9ede8ad986df9b6717a76b9f5f102c803710fdcb453392f29e1a35cd74c5f8964b60259e1c52acb2c364a45cfa

data/.travis.yml

@@ -5,7 +5,7 @@ services:
   - elasticsearch
   - mongodb
 before_install:
-  - wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.3.2.deb && sudo dpkg -i elasticsearch-1.3.2.deb && sudo service elasticsearch restart
+  - ./ci/before_install.sh
 script: bundle exec rake test
 before_script:
   - psql -c 'create database searchkick_test;' -U postgres
@@ -15,9 +15,14 @@ notifications:
     on_failure: change
 gemfile:
   - Gemfile
+  - gemfiles/activerecord41.gemfile
   - gemfiles/activerecord40.gemfile
   - gemfiles/activerecord32.gemfile
   - gemfiles/activerecord31.gemfile
   - gemfiles/mongoid2.gemfile
   - gemfiles/mongoid3.gemfile
   - gemfiles/mongoid4.gemfile
+matrix:
+  include:
+    - gemfile: gemfiles/nobrainer.gemfile
+      env: NOBRAINER=true

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.8.6
+
+- Added support for NoBrainer
+- Added `stem_conversions: false` option
+- Added support for multiple `boost_where` values on the same field
+- Added support for array of values for `boost_where`
+- Fixed suggestions with partial match boost
+- Fixed redefining existing instance methods in models
+
 ## 0.8.5
 
 - Added support for Elasticsearch 1.4

data/Gemfile

@@ -4,5 +4,4 @@ source 'https://rubygems.org'
 gemspec
 
 gem "sqlite3"
-gem "activerecord", "~> 4.1.0"
-gem "activejob_backport"
+gem "activerecord", "~> 4.2.0"

data/README.md

@@ -134,8 +134,9 @@ boost_by: {orders_count: {factor: 10}} # default factor is 1
 Boost matching documents
 
 ```ruby
-boost_where: {user_id: 1} # default factor is 1000
-boost_where: {user_id: {value: 1, factor: 100}}
+boost_where: {user_id: 1}
+boost_where: {user_id: {value: 1, factor: 100}} # default factor is 1000
+boost_where: {user_id: [{value: 1, factor: 100}, {value: 2, factor: 200}]}
 ```
 
 [Conversions](#keep-getting-better) are also a great way to boost.
@@ -355,6 +356,22 @@ There are three strategies for keeping the index synced with your database.
   end
   ```
 
+#### Associations
+
+Data is **not** automatically synced when an association is updated.  If this is desired, add a callback to reindex:
+
+```ruby
+class Image < ActiveRecord::Base
+  belongs_to :product
+
+  after_commit :reindex_product
+
+  def reindex_product
+    product.reindex # or reindex_async
+  end
+end
+```
+
 ### Keep Getting Better
 
 Searchkick uses conversion data to learn what users are looking for.  If a user searches for “ice cream” and adds Ben & Jerry’s Chunky Monkey to the cart (our conversion metric at Instacart), that item gets a little more weight for similar searches.
@@ -685,7 +702,7 @@ Product.searchkick_index.tokens("dieg", analyzer: "searchkick_word_search")
 # ["dieg"] - match!!
 ```
 
-See the [complete list of analyzers](lib/searchkick/reindex.rb#L86).
+See the [complete list of analyzers](lib/searchkick/index.rb#L209).
 
 ## Deployment
 
@@ -828,6 +845,8 @@ Reindex one record
 ```ruby
 product = Product.find 10
 product.reindex
+# or to reindex in the background
+product.reindex_async
 ```
 
 Remove old indices
@@ -902,28 +921,17 @@ class Product < ActiveRecord::Base
 end
 ```
 
-Make fields unsearchable but include in the source
+Create index without importing
 
 ```ruby
-class Product < ActiveRecord::Base
-  searchkick unsearchable: [:color]
-end
+Product.reindex(import: false)
 ```
 
-Reindex asynchronously
+Make fields unsearchable but include in the source
 
 ```ruby
 class Product < ActiveRecord::Base
-  searchkick callbacks: false
-
-  def reindex_async
-    # custom code to reindex
-  end
-
-  after_commit :reindex_async
-  # or for Mongoid
-  # after_save :reindex_async
-  # after_destroy :reindex_async
+  searchkick unsearchable: [:color]
 end
 ```
 
@@ -1011,12 +1019,12 @@ Thanks to Karel Minarik for [Elasticsearch Ruby](https://github.com/elasticsearc
 
 ## Roadmap
 
+- Semantic search features
 - Search multiple fields for different terms
 - Search across models
 - Search nested objects
 - Add section on testing
 - Much finer customization
-- More transparency into generated queries (for advanced use)
 
 ## Contributing
 
@@ -1029,6 +1037,9 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 
 To get started with development and testing:
 
-1. Clone the repo
-2. `bundle`
-3. `rake test`
+```sh
+git clone https://github.com/ankane/searchkick.git
+cd searchkick
+bundle install
+rake test
+```

data/ci/before_install.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.3.2.deb
+sudo dpkg -i elasticsearch-1.3.2.deb
+sudo service elasticsearch restart
+
+if [ -n "$NOBRAINER" ]; then
+  source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list
+  wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add -
+  sudo apt-get update -q
+  sudo apt-get install rethinkdb
+  sudo cp /etc/rethinkdb/default.conf.sample /etc/rethinkdb/instances.d/instance1.conf
+  sudo service rethinkdb restart
+fi

data/gemfiles/activerecord41.gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "sqlite3"
+gem "activerecord", "~> 4.1.0"
+gem "activejob_backport"

data/gemfiles/nobrainer.gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "nobrainer", ">= 0.19.0"

data/lib/searchkick.rb

@@ -3,10 +3,8 @@ require "elasticsearch"
 require "hashie"
 require "searchkick/version"
 require "searchkick/index"
-require "searchkick/reindex"
 require "searchkick/results"
 require "searchkick/query"
-require "searchkick/similar"
 require "searchkick/reindex_job"
 require "searchkick/model"
 require "searchkick/tasks"
@@ -30,11 +28,13 @@ module Searchkick
     attr_accessor :search_method_name
     attr_accessor :wordnet_path
     attr_accessor :timeout
+    attr_accessor :models
   end
   self.callbacks = true
   self.search_method_name = :search
   self.wordnet_path = "/var/lib/wn_s.pl"
   self.timeout = 10
+  self.models = []
 
   def self.client
     @client ||=

data/lib/searchkick/index.rb

@@ -1,9 +1,10 @@
 module Searchkick
   class Index
-    attr_reader :name
+    attr_reader :name, :options
 
-    def initialize(name)
+    def initialize(name, options = {})
       @name = name
+      @options = options
     end
 
     def create(options = {})
@@ -22,6 +23,23 @@ module Searchkick
       client.indices.refresh index: name
     end
 
+    def alias_exists?
+      client.indices.exists_alias name: name
+    end
+
+    def swap(new_name)
+      old_indices =
+        begin
+          client.indices.get_alias(name: name).keys
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          []
+        end
+      actions = old_indices.map{|old_name| {remove: {index: old_name, alias: name}} } + [{add: {index: new_name, alias: name}}]
+      client.indices.update_aliases body: {actions: actions}
+    end
+
+    # record based
+
     def store(record)
       client.index(
         index: name,
@@ -57,18 +75,399 @@ module Searchkick
       )["_source"]
     end
 
-    def klass_document_type(klass)
-      if klass.respond_to?(:document_type)
-        klass.document_type
+    def reindex_record(record)
+      if record.destroyed? or !record.should_index?
+        begin
+          remove(record)
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          # do nothing
+        end
       else
-        klass.model_name.to_s.underscore
+        store(record)
+      end
+    end
+
+    def reindex_record_async(record)
+      if defined?(Searchkick::ReindexV2Job)
+          Searchkick::ReindexV2Job.perform_later(record.class.name, record.id.to_s)
+      else
+        Delayed::Job.enqueue Searchkick::ReindexJob.new(record.class.name, record.id.to_s)
       end
     end
 
+    def similar_record(record, options = {})
+      like_text = retrieve(record).to_hash
+        .keep_if{|k,v| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
+        .values.compact.join(" ")
+
+      # TODO deep merge method
+      options[:where] ||= {}
+      options[:where][:_id] ||= {}
+      options[:where][:_id][:not] = record.id.to_s
+      options[:limit] ||= 10
+      options[:similar] = true
+
+      # TODO use index class instead of record class
+      search_model(record.class, like_text, options)
+    end
+
+    # search
+
+    def search_model(searchkick_klass, term = nil, options = {}, &block)
+      query = Searchkick::Query.new(searchkick_klass, term, options)
+      if block
+        block.call(query.body)
+      end
+      if options[:execute] == false
+        query
+      else
+        query.execute
+      end
+    end
+
+    # reindex
+
+    def create_index
+      index = Searchkick::Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
+      index.create(index_options)
+      index
+    end
+
+    # remove old indices that start w/ index_name
+    def clean_indices
+      all_indices = client.indices.get_aliases
+      indices = all_indices.select{|k, v| (v.empty? || v["aliases"].empty?) && k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
+      indices.each do |index|
+        Searchkick::Index.new(index).delete
+      end
+      indices
+    end
+
+    # https://gist.github.com/jarosan/3124884
+    # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
+    def reindex_scope(scope, options = {})
+      skip_import = options[:import] == false
+
+      clean_indices
+
+      index = create_index
+
+      # check if alias exists
+      if alias_exists?
+        # import before swap
+        index.import_scope(scope) unless skip_import
+
+        # get existing indices to remove
+        swap(index.name)
+        clean_indices
+      else
+        delete if exists?
+        swap(index.name)
+
+        # import after swap
+        index.import_scope(scope) unless skip_import
+      end
+
+      index.refresh
+
+      true
+    end
+
+    def import_scope(scope)
+      batch_size = @options[:batch_size] || 1000
+
+      # use scope for import
+      scope = scope.search_import if scope.respond_to?(:search_import)
+      if scope.respond_to?(:find_in_batches)
+        scope.find_in_batches batch_size: batch_size do |batch|
+          import batch.select{|item| item.should_index? }
+        end
+      else
+        # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
+        # use cursor for Mongoid
+        items = []
+        scope.all.each do |item|
+          items << item if item.should_index?
+          if items.length == batch_size
+            index.import items
+            items = []
+          end
+        end
+        import items
+      end
+    end
+
+    def index_options
+      options = @options
+
+      if options[:mappings] and !options[:merge_mappings]
+        settings = options[:settings] || {}
+        mappings = options[:mappings]
+      else
+        settings = {
+          analysis: {
+            analyzer: {
+              searchkick_keyword: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase"] + (options[:stem_conversions] == false ? [] : ["searchkick_stemmer"])
+              },
+              default_index: {
+                type: "custom",
+                tokenizer: "standard",
+                # synonym should come last, after stemming and shingle
+                # shingle must come before searchkick_stemmer
+                filter: ["standard", "lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"]
+              },
+              searchkick_search: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["standard", "lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"]
+              },
+              searchkick_search2: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["standard", "lowercase", "asciifolding", "searchkick_stemmer"]
+              },
+              # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
+              searchkick_autocomplete_index: {
+                type: "custom",
+                tokenizer: "searchkick_autocomplete_ngram",
+                filter: ["lowercase", "asciifolding"]
+              },
+              searchkick_autocomplete_search: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding"]
+              },
+              searchkick_word_search: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding"]
+              },
+              searchkick_suggest_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
+              },
+              searchkick_text_start_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+              },
+              searchkick_text_middle_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+              },
+              searchkick_text_end_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
+              },
+              searchkick_word_start_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+              },
+              searchkick_word_middle_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+              },
+              searchkick_word_end_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
+              }
+            },
+            filter: {
+              searchkick_index_shingle: {
+                type: "shingle",
+                token_separator: ""
+              },
+              # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7
+              searchkick_search_shingle: {
+                type: "shingle",
+                token_separator: "",
+                output_unigrams: false,
+                output_unigrams_if_no_shingles: true
+              },
+              searchkick_suggest_shingle: {
+                type: "shingle",
+                max_shingle_size: 5
+              },
+              searchkick_edge_ngram: {
+                type: "edgeNGram",
+                min_gram: 1,
+                max_gram: 50
+              },
+              searchkick_ngram: {
+                type: "nGram",
+                min_gram: 1,
+                max_gram: 50
+              },
+              searchkick_stemmer: {
+                type: "snowball",
+                language: options[:language] || "English"
+              }
+            },
+            tokenizer: {
+              searchkick_autocomplete_ngram: {
+                type: "edgeNGram",
+                min_gram: 1,
+                max_gram: 50
+              }
+            }
+          }
+        }
+
+        if Searchkick.env == "test"
+          settings.merge!(number_of_shards: 1, number_of_replicas: 0)
+        end
+
+        settings.deep_merge!(options[:settings] || {})
+
+        # synonyms
+        synonyms = options[:synonyms] || []
+        if synonyms.any?
+          settings[:analysis][:filter][:searchkick_synonym] = {
+            type: "synonym",
+            synonyms: synonyms.select{|s| s.size > 1 }.map{|s| s.join(",") }
+          }
+          # choosing a place for the synonym filter when stemming is not easy
+          # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
+          # TODO use a snowball stemmer on synonyms when creating the token filter
+
+          # http://elasticsearch-users.115913.n3.nabble.com/synonym-multi-words-search-td4030811.html
+          # I find the following approach effective if you are doing multi-word synonyms (synonym phrases):
+          # - Only apply the synonym expansion at index time
+          # - Don't have the synonym filter applied search
+          # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
+          settings[:analysis][:analyzer][:default_index][:filter].insert(4, "searchkick_synonym")
+          settings[:analysis][:analyzer][:default_index][:filter] << "searchkick_synonym"
+        end
+
+        if options[:wordnet]
+          settings[:analysis][:filter][:searchkick_wordnet] = {
+            type: "synonym",
+            format: "wordnet",
+            synonyms_path: Searchkick.wordnet_path
+          }
+
+          settings[:analysis][:analyzer][:default_index][:filter].insert(4, "searchkick_wordnet")
+          settings[:analysis][:analyzer][:default_index][:filter] << "searchkick_wordnet"
+        end
+
+        if options[:special_characters] == false
+          settings[:analysis][:analyzer].each do |analyzer, analyzer_settings|
+            analyzer_settings[:filter].reject!{|f| f == "asciifolding" }
+          end
+        end
+
+        mapping = {}
+
+        # conversions
+        if options[:conversions]
+          mapping[:conversions] = {
+            type: "nested",
+            properties: {
+              query: {type: "string", analyzer: "searchkick_keyword"},
+              count: {type: "integer"}
+            }
+          }
+        end
+
+        mapping_options = Hash[
+          [:autocomplete, :suggest, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight]
+            .map{|type| [type, (options[type] || []).map(&:to_s)] }
+        ]
+
+        mapping_options.values.flatten.uniq.each do |field|
+          field_mapping = {
+            type: "multi_field",
+            fields: {
+              field => {type: "string", index: "not_analyzed"},
+              "analyzed" => {type: "string", index: "analyzed"}
+              # term_vector: "with_positions_offsets" for fast / correct highlighting
+              # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-highlighting.html#_fast_vector_highlighter
+            }
+          }
+
+          mapping_options.except(:highlight).each do |type, fields|
+            if fields.include?(field)
+              field_mapping[:fields][type] = {type: "string", index: "analyzed", analyzer: "searchkick_#{type}_index"}
+            end
+          end
+
+          if mapping_options[:highlight].include?(field)
+            field_mapping[:fields]["analyzed"][:term_vector] = "with_positions_offsets"
+          end
+
+          mapping[field] = field_mapping
+        end
+
+        (options[:locations] || []).map(&:to_s).each do |field|
+          mapping[field] = {
+            type: "geo_point"
+          }
+        end
+
+        (options[:unsearchable] || []).map(&:to_s).each do |field|
+          mapping[field] = {
+            type: "string",
+            index: "no"
+          }
+        end
+
+        mappings = {
+          _default_: {
+            properties: mapping,
+            # https://gist.github.com/kimchy/2898285
+            dynamic_templates: [
+              {
+                string_template: {
+                  match: "*",
+                  match_mapping_type: "string",
+                  mapping: {
+                    # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
+                    type: "multi_field",
+                    fields: {
+                      # analyzed field must be the default field for include_in_all
+                      # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
+                      # however, we can include the not_analyzed field in _all
+                      # and the _all index analyzer will take care of it
+                      "{name}" => {type: "string", index: "not_analyzed"},
+                      "analyzed" => {type: "string", index: "analyzed"}
+                    }
+                  }
+                }
+              }
+            ]
+          }
+        }.deep_merge(options[:mappings] || {})
+      end
+
+      {
+        settings: settings,
+        mappings: mappings
+      }
+    end
+
+    # other
+
     def tokens(text, options = {})
       client.indices.analyze({text: text, index: name}.merge(options))["tokens"].map{|t| t["token"] }
     end
 
+    def klass_document_type(klass)
+      if klass.respond_to?(:document_type)
+        klass.document_type
+      else
+        klass.model_name.to_s.underscore
+      end
+    end
+
     protected
 
     def client