searchkick 0.5.3 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6b20e3f5f346c6ef75c257cf2b2aafa23884ee2b
-  data.tar.gz: f8c6e426340f0602037a3204e8d38a40d28cf430
+  metadata.gz: 633b6ec4127ea2638b8c05db6c1c1c0b536c4193
+  data.tar.gz: 69d65f9ecc57e55a954482e03a0998a1f10023dc
 SHA512:
-  metadata.gz: 210eecb919e7491f84acdc1f4f69f2c1bf898b808033ed31f8adc63cc4ac636d0e242cd9136cf1467842aafcd0e358b570b02e9eb0f2e1a7d158807c096e6b3d
-  data.tar.gz: b4907fb7bde4d2a7c13a68b26b84527f8deb00bd41da7216a0637f15f62923f39d0589151edbcd44b1bc0f1eec61e3ae8e5e0be1cc95e440f09b6a694d10f007
+  metadata.gz: 669dd3262ff71f967b6d0266851eca6f769befc5317289bb058781cf11d6e46af2e47bf9193a60fbc19117b04c5653551b4966bb349d6a9f88f79baa424b0116
+  data.tar.gz: 024fb4884e8a4d08e3a8954e3d2de0fce7bc7968311da0b64d6c9fe3f24e9b9c6488fa37d11c4c5f4674bdd5e60f72ac4da0ab8d5122c8c60fc07ef8542790c6

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.6.0
+
+- Moved to elasticsearch-ruby
+- Added support for modifying the query and viewing the response
+- Added support for page_entries_info method
+
 ## 0.5.3
 
 - Fixed bug w/ word_* queries

data/Gemfile

@@ -3,5 +3,7 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in searchkick.gemspec
 gemspec
 
+gem "sqlite3"
+gem "activerecord"
 # gem "activerecord", "~> 3.2.0"
 # gem "activerecord", "~> 3.1.0"

data/README.md

@@ -147,7 +147,7 @@ Product.search "fresh honey" # fresh AND honey
 To change this, use:
 
 ```ruby
-Product.search "fresh honey", partial: true # fresh OR honey
+Product.search "fresh honey", operator: "or" # fresh OR honey
 ```
 
 By default, results must match the entire word - `back` will not match `backpack`. You can change this behavior with:
@@ -496,7 +496,7 @@ And to search, use:
 ```ruby
 Animal.search "*"                   # all animals
 Dog.search "*"                      # just dogs
-Animal.search "*", type: [Dog, Cat] # just cats and dogs [master]
+Animal.search "*", type: [Dog, Cat] # just cats and dogs
 ```
 
 **Note:** The `suggest` option retrieves suggestions from the parent at the moment.
@@ -568,7 +568,13 @@ end
 And use the `query` option to search:
 
 ```ruby
-Product.search query: {match: {name: "milk"}}
+products = Product.search query: {match: {name: "milk"}}
+```
+
+View the response with:
+
+```ruby
+products.response
 ```
 
 To keep the mappings and settings generated by Searchkick, use:
@@ -579,6 +585,14 @@ class Product < ActiveRecord::Base
 end
 ```
 
+To modify the query generated by Searchkick, use:
+
+```ruby
+query = Product.search "2% Milk", execute: false
+query.body[:query] = {match_all: {}}
+products = query.execute
+```
+
 ## Reference
 
 Searchkick requires Elasticsearch `0.90.0` or higher.

data/gemfiles/mongoid4.gemfile

@@ -3,4 +3,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in searchkick.gemspec
 gemspec path: "../"
 
-gem "mongoid", github: "mongoid/mongoid"
+gem "mongoid", "4.0.0.beta1"

data/lib/searchkick.rb

@@ -1,14 +1,24 @@
-require "tire"
+require "active_model"
+require "patron"
+require "elasticsearch"
 require "searchkick/version"
+require "searchkick/index"
 require "searchkick/reindex"
 require "searchkick/results"
+require "searchkick/query"
 require "searchkick/search"
 require "searchkick/similar"
 require "searchkick/model"
 require "searchkick/tasks"
-require "searchkick/logger" if defined?(Rails)
+# TODO add logger
+# require "searchkick/logger" if defined?(Rails)
 
 module Searchkick
+
+  def self.client
+    @client ||= Elasticsearch::Client.new(url: ENV["ELASTICSEARCH_URL"])
+  end
+
   @callbacks = true
 
   def self.enable_callbacks

data/lib/searchkick/index.rb

@@ -0,0 +1,67 @@
+module Searchkick
+  class Index
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+    end
+
+    def create(options = {})
+      client.indices.create index: name, body: options
+    end
+
+    def delete
+      client.indices.delete index: name
+    end
+
+    def exists?
+      client.indices.exists index: name
+    end
+
+    def refresh
+      client.indices.refresh index: name
+    end
+
+    def store(record)
+      client.index(
+        index: name,
+        type: record.document_type,
+        id: record.id,
+        body: record.as_indexed_json
+      )
+    end
+
+    def remove(record)
+      client.delete(
+        index: name,
+        type: record.document_type,
+        id: record.id
+      )
+    end
+
+    def import(records)
+      if records.any?
+        client.bulk(
+          index: name,
+          type: records.first.document_type,
+          body: records.map{|r| data = r.as_indexed_json; {index: {_id: data["_id"] || data["id"] || r.id, data: data}} }
+        )
+      end
+    end
+
+    def retrieve(document_type, id)
+      client.get_source(
+        index: name,
+        type: document_type,
+        id: id
+      )
+    end
+
+    protected
+
+    def client
+      Searchkick.client
+    end
+
+  end
+end

data/lib/searchkick/model.rb

@@ -13,7 +13,7 @@ module Searchkick
         # set index name
         # TODO support proc
         index_name = options[:index_name] || [options[:index_prefix], model_name.plural, searchkick_env].compact.join("_")
-        class_variable_set :@@searchkick_index, Tire::Index.new(index_name)
+        class_variable_set :@@searchkick_index, Searchkick::Index.new(index_name)
 
         extend Searchkick::Search
         extend Searchkick::Reindex
@@ -45,7 +45,11 @@ module Searchkick
         def reindex
           index = self.class.searchkick_index
           if destroyed? or !should_index?
-            index.remove self
+            begin
+              index.remove self
+            rescue Elasticsearch::Transport::Transport::Errors::NotFound
+              # do nothing
+            end
           else
             index.store self
           end
@@ -55,7 +59,7 @@ module Searchkick
           respond_to?(:to_hash) ? to_hash : serializable_hash
         end
 
-        def to_indexed_json
+        def as_indexed_json
           source = search_data
 
           # stringify fields
@@ -115,7 +119,7 @@ module Searchkick
 
           # p search_data
 
-          source.to_json
+          source.as_json
         end
 
         # TODO remove

data/lib/searchkick/query.rb

@@ -0,0 +1,441 @@
+module Searchkick
+  class Query
+    attr_reader :klass, :term, :options
+    attr_accessor :body
+
+    def initialize(klass, term, options = {})
+      if term.is_a?(Hash)
+        options = term
+        term = nil
+      else
+        term = term.to_s
+      end
+
+      @klass = klass
+      @term = term
+      @options = options
+
+      fields =
+        if options[:fields]
+          if options[:autocomplete]
+            options[:fields].map{|f| "#{f}.autocomplete" }
+          else
+            options[:fields].map do |value|
+              k, v = value.is_a?(Hash) ? value.to_a.first : [value, :word]
+              "#{k}.#{v == :word ? "analyzed" : v}"
+            end
+          end
+        else
+          if options[:autocomplete]
+            (searchkick_options[:autocomplete] || []).map{|f| "#{f}.autocomplete" }
+          else
+            ["_all"]
+          end
+        end
+
+      operator = options[:operator] || (options[:partial] ? "or" : "and")
+
+      # pagination
+      page = [options[:page].to_i, 1].max
+      per_page = (options[:limit] || options[:per_page] || 100000).to_i
+      offset = options[:offset] || (page - 1) * per_page
+      index_name = options[:index_name] || searchkick_index.name
+
+      conversions_field = searchkick_options[:conversions]
+      personalize_field = searchkick_options[:personalize]
+
+      all = term == "*"
+
+      if options[:query]
+        payload = options[:query]
+      elsif options[:similar]
+        payload = {
+          more_like_this: {
+            fields: fields,
+            like_text: term,
+            min_doc_freq: 1,
+            min_term_freq: 1,
+            analyzer: "searchkick_search2"
+          }
+        }
+      elsif all
+        payload = {
+          match_all: {}
+        }
+      else
+        if options[:autocomplete]
+          payload = {
+            multi_match: {
+              fields: fields,
+              query: term,
+              analyzer: "searchkick_autocomplete_search"
+            }
+          }
+        else
+          queries = []
+          fields.each do |field|
+            if field == "_all" or field.end_with?(".analyzed")
+              shared_options = {
+                fields: [field],
+                query: term,
+                use_dis_max: false,
+                operator: operator,
+                cutoff_frequency: 0.001
+              }
+              queries.concat [
+                {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search")},
+                {multi_match: shared_options.merge(boost: 10, analyzer: "searchkick_search2")}
+              ]
+              if options[:misspellings] != false
+                distance = (options[:misspellings].is_a?(Hash) && options[:misspellings][:distance]) || 1
+                queries.concat [
+                  {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search")},
+                  {multi_match: shared_options.merge(fuzziness: distance, max_expansions: 3, analyzer: "searchkick_search2")}
+                ]
+              end
+            else
+              analyzer = field.match(/\.word_(start|middle|end)\z/) ? "searchkick_word_search" : "searchkick_autocomplete_search"
+              queries << {
+                multi_match: {
+                  fields: [field],
+                  query: term,
+                  analyzer: analyzer
+                }
+              }
+            end
+          end
+
+          payload = {
+            dis_max: {
+              queries: queries
+            }
+          }
+        end
+
+        if conversions_field and options[:conversions] != false
+          # wrap payload in a bool query
+          payload = {
+            bool: {
+              must: payload,
+              should: {
+                nested: {
+                  path: conversions_field,
+                  score_mode: "total",
+                  query: {
+                    custom_score: {
+                      query: {
+                        match: {
+                          query: term
+                        }
+                      },
+                      script: "doc['count'].value"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        end
+      end
+
+      custom_filters = []
+
+      if options[:boost]
+        custom_filters << {
+          filter: {
+            exists: {
+              field: options[:boost]
+            }
+          },
+          script: "log(doc['#{options[:boost]}'].value + 2.718281828)"
+        }
+      end
+
+      if options[:user_id] and personalize_field
+        custom_filters << {
+          filter: {
+            term: {
+              personalize_field => options[:user_id]
+            }
+          },
+          boost: 100
+        }
+      end
+
+      if options[:personalize]
+        custom_filters << {
+          filter: {
+            term: options[:personalize]
+          },
+          boost: 100
+        }
+      end
+
+      if custom_filters.any?
+        payload = {
+          custom_filters_score: {
+            query: payload,
+            filters: custom_filters,
+            score_mode: "total"
+          }
+        }
+      end
+
+      payload = {
+        query: payload,
+        size: per_page,
+        from: offset
+      }
+      payload[:explain] = options[:explain] if options[:explain]
+
+      # order
+      if options[:order]
+        order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
+        payload[:sort] = Hash[ order.map{|k, v| [k.to_s == "id" ? :_id : k, v] } ]
+      end
+
+      # filters
+      filters = where_filters(options[:where])
+      if filters.any?
+        payload[:filter] = {
+          and: filters
+        }
+      end
+
+      # facets
+      facet_limits = {}
+      if options[:facets]
+        facets = options[:facets] || {}
+        if facets.is_a?(Array) # convert to more advanced syntax
+          facets = Hash[ facets.map{|f| [f, {}] } ]
+        end
+
+        payload[:facets] = {}
+        facets.each do |field, facet_options|
+          # ask for extra facets due to
+          # https://github.com/elasticsearch/elasticsearch/issues/1305
+
+          if facet_options[:ranges]
+            payload[:facets][field] = {
+              range: {
+                field.to_sym => facet_options[:ranges]
+              }
+            }
+          else
+            payload[:facets][field] = {
+              terms: {
+                field: field,
+                size: facet_options[:limit] ? facet_options[:limit] + 150 : 100000
+              }
+            }
+          end
+
+          facet_limits[field] = facet_options[:limit] if facet_options[:limit]
+
+          # offset is not possible
+          # http://elasticsearch-users.115913.n3.nabble.com/Is-pagination-possible-in-termsStatsFacet-td3422943.html
+
+          facet_filters = where_filters(facet_options[:where])
+          if facet_filters.any?
+            payload[:facets][field][:facet_filter] = {
+              and: {
+                filters: facet_filters
+              }
+            }
+          end
+        end
+      end
+
+      # suggestions
+      if options[:suggest]
+        suggest_fields = (searchkick_options[:suggest] || []).map(&:to_s)
+        # intersection
+        suggest_fields = suggest_fields & options[:fields].map(&:to_s) if options[:fields]
+        if suggest_fields.any?
+          payload[:suggest] = {text: term}
+          suggest_fields.each do |field|
+            payload[:suggest][field] = {
+              phrase: {
+                field: "#{field}.suggest"
+              }
+            }
+          end
+        end
+      end
+
+      # highlight
+      if options[:highlight]
+        payload[:highlight] = {
+          fields: Hash[ fields.map{|f| [f, {}] } ]
+        }
+        if options[:highlight].is_a?(Hash) and tag = options[:highlight][:tag]
+          payload[:highlight][:pre_tags] = [tag]
+          payload[:highlight][:post_tags] = [tag.to_s.gsub(/\A</, "</")]
+        end
+      end
+
+      # model and eagar loading
+      load = options[:load].nil? ? true : options[:load]
+
+      # An empty array will cause only the _id and _type for each hit to be returned
+      # http://www.elasticsearch.org/guide/reference/api/search/fields/
+      payload[:fields] = [] if load
+
+      if options[:type] or klass != searchkick_klass
+        @type = [options[:type] || klass].flatten.map(&:document_type)
+      end
+
+      @body = payload
+      @facet_limits = facet_limits
+      @page = page
+      @per_page = per_page
+      @load = load
+    end
+
+    def searchkick_index
+      klass.searchkick_index
+    end
+
+    def searchkick_options
+      klass.searchkick_options
+    end
+
+    def searchkick_klass
+      klass.searchkick_klass
+    end
+
+    def document_type
+      klass.document_type
+    end
+
+    def execute
+      params = {
+        index: searchkick_index.name,
+        body: body
+      }
+      params.merge!(type: @type) if @type
+      begin
+        response = Searchkick.client.search(params)
+      rescue => e # TODO rescue type
+        status_code = e.message[1..3].to_i
+        if status_code == 404
+          raise "Index missing - run #{searchkick_klass.name}.reindex"
+        elsif status_code == 500 and (e.message.include?("IllegalArgumentException[minimumSimilarity >= 1]") or e.message.include?("No query registered for [multi_match]") or e.message.include?("[match] query does not support [cutoff_frequency]]"))
+          raise "Upgrade Elasticsearch to 0.90.0 or greater"
+        else
+          raise e
+        end
+      end
+
+      # apply facet limit in client due to
+      # https://github.com/elasticsearch/elasticsearch/issues/1305
+      @facet_limits.each do |field, limit|
+        field = field.to_s
+        facet = response["facets"][field]
+        response["facets"][field]["terms"] = facet["terms"].first(limit)
+        response["facets"][field]["other"] = facet["total"] - facet["terms"].sum{|term| term["count"] }
+      end
+
+      opts = {
+        page: @page,
+        per_page: @per_page,
+        load: @load,
+        includes: options[:include] || options[:includes]
+      }
+      Searchkick::Results.new(searchkick_klass, response, opts)
+    end
+
+    private
+
+    def where_filters(where)
+      filters = []
+      (where || {}).each do |field, value|
+        field = :_id if field.to_s == "id"
+
+        if field == :or
+          value.each do |or_clause|
+            filters << {or: or_clause.map{|or_statement| {and: where_filters(or_statement)} }}
+          end
+        else
+          # expand ranges
+          if value.is_a?(Range)
+            value = {gte: value.first, (value.exclude_end? ? :lt : :lte) => value.last}
+          end
+
+          if value.is_a?(Array)
+            value = {in: value}
+          end
+
+          if value.is_a?(Hash)
+            value.each do |op, op_value|
+              case op
+              when :within, :bottom_right
+                # do nothing
+              when :near
+                filters << {
+                  geo_distance: {
+                    field => op_value.map(&:to_f).reverse,
+                    distance: value[:within] || "50mi"
+                  }
+                }
+              when :top_left
+                filters << {
+                  geo_bounding_box: {
+                    field => {
+                      top_left: op_value.map(&:to_f).reverse,
+                      bottom_right: value[:bottom_right].map(&:to_f).reverse
+                    }
+                  }
+                }
+              when :not # not equal
+                filters << {not: term_filters(field, op_value)}
+              when :all
+                filters << {terms: {field => op_value, execution: "and"}}
+              when :in
+                filters << term_filters(field, op_value)
+              else
+                range_query =
+                  case op
+                  when :gt
+                    {from: op_value, include_lower: false}
+                  when :gte
+                    {from: op_value, include_lower: true}
+                  when :lt
+                    {to: op_value, include_upper: false}
+                  when :lte
+                    {to: op_value, include_upper: true}
+                  else
+                    raise "Unknown where operator"
+                  end
+                # issue 132
+                if existing = filters.find{ |f| f[:range] && f[:range][field] }
+                  existing[:range][field].merge!(range_query)
+                else
+                  filters << {range: {field => range_query}}
+                end
+              end
+            end
+          else
+            filters << term_filters(field, value)
+          end
+        end
+      end
+      filters
+    end
+
+    def term_filters(field, value)
+      if value.is_a?(Array) # in query
+        if value.any?
+          {or: value.map{|v| term_filters(field, v) }}
+        else
+          {terms: {field => value}} # match nothing
+        end
+      elsif value.nil?
+        {missing: {"field" => field, existence: true, null_value: true}}
+      else
+        {term: {field => value}}
+      end
+    end
+
+  end
+end