searchkick_bharthur 0.0.1

data/lib/searchkick/reindex_job.rb

@@ -0,0 +1,26 @@
+module Searchkick
+  class ReindexJob
+    def initialize(klass, id)
+      @klass = klass
+      @id = id
+    end
+
+    def perform
+      model = @klass.constantize
+      record = model.find(@id) rescue nil # TODO fix lazy coding
+      index = model.searchkick_index
+      if !record || !record.should_index?
+        # hacky
+        record ||= model.new
+        record.id = @id
+        begin
+          index.remove record
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          # do nothing
+        end
+      else
+        index.store record
+      end
+    end
+  end
+end

data/lib/searchkick/reindex_v2_job.rb

@@ -0,0 +1,23 @@
+module Searchkick
+  class ReindexV2Job < ActiveJob::Base
+    queue_as :searchkick
+
+    def perform(klass, id)
+      model = klass.constantize
+      record = model.find(id) rescue nil # TODO fix lazy coding
+      index = model.searchkick_index
+      if !record || !record.should_index?
+        # hacky
+        record ||= model.new
+        record.id = id
+        begin
+          index.remove record
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          # do nothing
+        end
+      else
+        index.store record
+      end
+    end
+  end
+end

data/lib/searchkick/results.rb

@@ -0,0 +1,211 @@
+require "forwardable"
+
+module Searchkick
+  class Results
+    include Enumerable
+    extend Forwardable
+
+    attr_reader :klass, :response, :options
+
+    def_delegators :results, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary
+
+    def initialize(klass, response, options = {})
+      @klass = klass
+      @response = response
+      @options = options
+    end
+
+    # experimental: may not make next release
+    def records
+      @records ||= results_query(klass, hits)
+    end
+
+    def results
+      @results ||= begin
+        if options[:load]
+          # results can have different types
+          results = {}
+
+          hits.group_by { |hit, _| hit["_type"] }.each do |type, grouped_hits|
+            results[type] = results_query(type.camelize.constantize, grouped_hits).to_a.index_by { |r| r.id.to_s }
+          end
+
+          # sort
+          hits.map do |hit|
+            results[hit["_type"]][hit["_id"].to_s]
+          end.compact
+        else
+          hits.map do |hit|
+            result =
+              if hit["_source"]
+                hit.except("_source").merge(hit["_source"])
+              elsif hit["fields"]
+                hit.except("fields").merge(hit["fields"])
+              else
+                hit
+              end
+
+            if hit["highlight"]
+              highlight = Hash[hit["highlight"].map { |k, v| [base_field(k), v.first] }]
+              options[:highlighted_fields].map { |k| base_field(k) }.each do |k|
+                result["highlighted_#{k}"] ||= (highlight[k] || result[k])
+              end
+            end
+
+            result["id"] ||= result["_id"] # needed for legacy reasons
+            Hashie::Mash.new(result)
+          end
+        end
+      end
+    end
+
+    def suggestions
+      if response["suggest"]
+        response["suggest"].values.flat_map { |v| v.first["options"] }.sort_by { |o| -o["score"] }.map { |o| o["text"] }.uniq
+      else
+        raise "Pass `suggest: true` to the search method for suggestions"
+      end
+    end
+
+    def each_with_hit(&block)
+      results.zip(hits).each(&block)
+    end
+
+    def with_details
+      each_with_hit.map do |model, hit|
+        details = {}
+        if hit["highlight"]
+          details[:highlight] = Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, v.first] }]
+        end
+        [model, details]
+      end
+    end
+
+    def facets
+      response["facets"]
+    end
+
+    def aggregations
+      response["aggregations"]
+    end
+
+    def aggs
+      @aggs ||= begin
+        if aggregations
+          aggregations.dup.each do |field, filtered_agg|
+            buckets = filtered_agg[field]
+            # move the buckets one level above into the field hash
+            if buckets
+              filtered_agg.delete(field)
+              filtered_agg.merge!(buckets)
+            end
+          end
+        end
+      end
+    end
+
+    def took
+      response["took"]
+    end
+
+    def error
+      response["error"]
+    end
+
+    def model_name
+      klass.model_name
+    end
+
+    def entry_name
+      model_name.human.downcase
+    end
+
+    def total_count
+      response["hits"]["total"]
+    end
+    alias_method :total_entries, :total_count
+
+    def current_page
+      options[:page]
+    end
+
+    def per_page
+      options[:per_page]
+    end
+    alias_method :limit_value, :per_page
+
+    def padding
+      options[:padding]
+    end
+
+    def total_pages
+      (total_count / per_page.to_f).ceil
+    end
+    alias_method :num_pages, :total_pages
+
+    def offset_value
+      (current_page - 1) * per_page + padding
+    end
+    alias_method :offset, :offset_value
+
+    def previous_page
+      current_page > 1 ? (current_page - 1) : nil
+    end
+    alias_method :prev_page, :previous_page
+
+    def next_page
+      current_page < total_pages ? (current_page + 1) : nil
+    end
+
+    def first_page?
+      previous_page.nil?
+    end
+
+    def last_page?
+      next_page.nil?
+    end
+
+    def out_of_range?
+      current_page > total_pages
+    end
+
+    def hits
+      @response["hits"]["hits"]
+    end
+
+    private
+
+    def results_query(records, hits)
+      ids = hits.map { |hit| hit["_id"] }
+
+      if options[:includes]
+        records =
+          if defined?(NoBrainer::Document) && records < NoBrainer::Document
+            records.preload(options[:includes])
+          else
+            records.includes(options[:includes])
+          end
+      end
+
+      if records.respond_to?(:primary_key) && records.primary_key
+        # ActiveRecord
+        records.where(records.primary_key => ids)
+      elsif records.respond_to?(:all) && records.all.respond_to?(:for_ids)
+        # Mongoid 2
+        records.all.for_ids(ids)
+      elsif records.respond_to?(:queryable)
+        # Mongoid 3+
+        records.queryable.for_ids(ids)
+      elsif records.respond_to?(:unscoped) && records.all.respond_to?(:preload)
+        # Nobrainer
+        records.unscoped.where(:id.in => ids)
+      else
+        raise "Not sure how to load records"
+      end
+    end
+
+    def base_field(k)
+      k.sub(/\.(analyzed|word_start|word_middle|word_end|text_start|text_middle|text_end|exact)\z/, "")
+    end
+  end
+end

data/lib/searchkick/tasks.rb

@@ -0,0 +1,33 @@
+require "rake"
+
+namespace :searchkick do
+  desc "reindex model"
+  task reindex: :environment do
+    if ENV["CLASS"]
+      klass = ENV["CLASS"].constantize rescue nil
+      if klass
+        klass.reindex
+      else
+        abort "Could not find class: #{ENV['CLASS']}"
+      end
+    else
+      abort "USAGE: rake searchkick:reindex CLASS=Product"
+    end
+  end
+
+  if defined?(Rails)
+
+    namespace :reindex do
+      desc "reindex all models"
+      task all: :environment do
+        Rails.application.eager_load!
+        Searchkick.models.each do |model|
+          puts "Reindexing #{model.name}..."
+          model.reindex
+        end
+        puts "Reindex complete"
+      end
+    end
+
+  end
+end

data/lib/searchkick/version.rb

@@ -0,0 +1,3 @@
+module Searchkick
+  VERSION = "0.0.1"
+end

data/lib/searchkick.rb

@@ -0,0 +1,159 @@
+require "active_model"
+require "elasticsearch"
+require "hashie"
+require "searchkick/version"
+require "searchkick/index"
+require "searchkick/results"
+require "searchkick/query"
+require "searchkick/reindex_job"
+require "searchkick/model"
+require "searchkick/tasks"
+require "searchkick/middleware"
+require "searchkick/logging" if defined?(ActiveSupport::Notifications)
+
+# background jobs
+begin
+  require "active_job"
+rescue LoadError
+  # do nothing
+end
+require "searchkick/reindex_v2_job" if defined?(ActiveJob)
+
+module Searchkick
+  class Error < StandardError; end
+  class MissingIndexError < Error; end
+  class UnsupportedVersionError < Error; end
+  class InvalidQueryError < Elasticsearch::Transport::Transport::Errors::BadRequest; end
+  class DangerousOperation < Error; end
+  class ImportError < Error; end
+
+  class << self
+    attr_accessor :search_method_name, :wordnet_path, :timeout, :models
+    attr_writer :client, :env, :search_timeout
+  end
+  self.search_method_name = :search
+  self.wordnet_path = "/var/lib/wn_s.pl"
+  self.timeout = 10
+  self.models = []
+
+  def self.client
+    @client ||=
+      Elasticsearch::Client.new(
+        url: ENV["ELASTICSEARCH_URL"],
+        transport_options: {request: {timeout: timeout}}
+      ) do |f|
+        f.use Searchkick::Middleware
+      end
+  end
+
+  def self.env
+    @env ||= ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+  end
+
+  def self.search_timeout
+    @search_timeout || timeout
+  end
+
+  def self.server_version
+    @server_version ||= client.info["version"]["number"]
+  end
+
+  def self.server_below?(version)
+    Gem::Version.new(server_version.sub("-", ".")) < Gem::Version.new(version.sub("-", "."))
+  end
+
+  def self.enable_callbacks
+    self.callbacks_value = nil
+  end
+
+  def self.disable_callbacks
+    self.callbacks_value = false
+  end
+
+  def self.callbacks?
+    Thread.current[:searchkick_callbacks_enabled].nil? || Thread.current[:searchkick_callbacks_enabled]
+  end
+
+  def self.callbacks(value)
+    if block_given?
+      previous_value = callbacks_value
+      begin
+        self.callbacks_value = value
+        yield
+        perform_bulk if callbacks_value == :bulk
+      ensure
+        self.callbacks_value = previous_value
+      end
+    else
+      self.callbacks_value = value
+    end
+  end
+
+  # private
+  def self.queue_items(items)
+    queued_items.concat(items)
+    perform_bulk unless callbacks_value == :bulk
+  end
+
+  # private
+  def self.perform_bulk
+    items = queued_items
+    clear_queued_items
+    perform_items(items)
+  end
+
+  # private
+  def self.perform_items(items)
+    if items.any?
+      response = client.bulk(body: items)
+      if response["errors"]
+        first_item = response["items"].first
+        raise Searchkick::ImportError, (first_item["index"] || first_item["delete"])["error"]
+      end
+    end
+  end
+
+  # private
+  def self.queued_items
+    Thread.current[:searchkick_queued_items] ||= []
+  end
+
+  # private
+  def self.clear_queued_items
+    Thread.current[:searchkick_queued_items] = []
+  end
+
+  # private
+  def self.callbacks_value
+    Thread.current[:searchkick_callbacks_enabled]
+  end
+
+  # private
+  def self.callbacks_value=(value)
+    Thread.current[:searchkick_callbacks_enabled] = value
+  end
+
+  def self.search(term = nil, options = {}, &block)
+    query = Searchkick::Query.new(nil, term, options)
+    block.call(query.body) if block
+    if options[:execute] == false
+      query
+    else
+      query.execute
+    end
+  end
+
+  def self.multi_search(queries)
+    if queries.any?
+      responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
+      queries.each_with_index do |query, i|
+        query.handle_response(responses[i])
+      end
+    end
+    nil
+  end
+end
+
+# TODO find better ActiveModel hook
+ActiveModel::Callbacks.send(:include, Searchkick::Model)
+ActiveRecord::Base.send(:extend, Searchkick::Model) if defined?(ActiveRecord)

data/searchkick.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "searchkick/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "searchkick_bharthur"
+  spec.version       = Searchkick::VERSION
+  spec.authors       = ["Andrew Kane", "Shiv Bharthur (Customization)"]
+  spec.email         = ["andrew@chartkick.com", "shiv.bharthur@gmail.com"]
+  spec.description   = "Intelligent search made easy"
+  spec.summary       = "Searchkick learns what your users are looking for. As more people search, it gets smarter and the results get better. It’s friendly for developers - and magical for your users."
+  spec.homepage      = "https://github.com/ankane/searchkick"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activemodel"
+  spec.add_dependency "elasticsearch", ">= 1"
+  spec.add_dependency "hashie"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+end

data/test/aggs_test.rb

@@ -0,0 +1,115 @@
+require_relative "test_helper"
+
+class AggsTest < Minitest::Test
+  def setup
+    super
+    store [
+      {name: "Product Show", latitude: 37.7833, longitude: 12.4167, store_id: 1, in_stock: true, color: "blue", price: 21, created_at: 2.days.ago},
+      {name: "Product Hide", latitude: 29.4167, longitude: -98.5000, store_id: 2, in_stock: false, color: "green", price: 25, created_at: 2.days.from_now},
+      {name: "Product B", latitude: 43.9333, longitude: -122.4667, store_id: 2, in_stock: false, color: "red", price: 5},
+      {name: "Foo", latitude: 43.9333, longitude: 12.4667, store_id: 3, in_stock: false, color: "yellow", price: 15}
+    ]
+  end
+
+  def test_basic
+    assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: [:store_id])
+  end
+
+  def test_where
+    assert_equal ({1 => 1}), store_agg(aggs: {store_id: {where: {in_stock: true}}})
+  end
+
+  def test_order
+    agg = Product.search("Product", aggs: {color: {order: {"_term" => "desc"}}}).aggs["color"]
+    assert_equal %w(red green blue), agg["buckets"].map { |b| b["key"] }
+  end
+
+  def test_field
+    assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: {store_id: {}})
+    assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: {store_id: {field: "store_id"}})
+    assert_equal ({1 => 1, 2 => 2}), store_agg({aggs: {store_id_new: {field: "store_id"}}}, "store_id_new")
+  end
+
+  def test_min_doc_count
+    assert_equal ({2 => 2}), store_agg(aggs: {store_id: {min_doc_count: 2}})
+  end
+
+  def test_no_aggs
+    assert_nil Product.search("*").aggs
+  end
+
+  def test_limit
+    agg = Product.search("Product", aggs: {store_id: {limit: 1}}).aggs["store_id"]
+    assert_equal 1, agg["buckets"].size
+    # assert_equal 3, agg["doc_count"]
+    assert_equal(1, agg["sum_other_doc_count"]) unless Searchkick.server_below?("1.4.0")
+  end
+
+  def test_ranges
+    price_ranges = [{to: 10}, {from: 10, to: 20}, {from: 20}]
+    agg = Product.search("Product", aggs: {price: {ranges: price_ranges}}).aggs["price"]
+
+    assert_equal 3, agg["buckets"].size
+    assert_equal 10.0, agg["buckets"][0]["to"]
+    assert_equal 20.0, agg["buckets"][2]["from"]
+    assert_equal 1, agg["buckets"][0]["doc_count"]
+    assert_equal 0, agg["buckets"][1]["doc_count"]
+    assert_equal 2, agg["buckets"][2]["doc_count"]
+  end
+
+  def test_date_ranges
+    ranges = [{to: 1.day.ago}, {from: 1.day.ago, to: 1.day.from_now}, {from: 1.day.from_now}]
+    agg = Product.search("Product", aggs: {created_at: {date_ranges: ranges}}).aggs["created_at"]
+
+    assert_equal 1, agg["buckets"][0]["doc_count"]
+    assert_equal 1, agg["buckets"][1]["doc_count"]
+    assert_equal 1, agg["buckets"][2]["doc_count"]
+  end
+
+  def test_query_where
+    assert_equal ({1 => 1}), store_agg(where: {in_stock: true}, aggs: [:store_id])
+  end
+
+  def test_two_wheres
+    assert_equal ({2 => 1}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false}}})
+  end
+
+  def test_where_override
+    assert_equal ({}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false, color: "blue"}}})
+    assert_equal ({2 => 1}), store_agg(where: {color: "blue"}, aggs: {store_id: {where: {in_stock: false, color: "red"}}})
+  end
+
+  def test_skip
+    assert_equal ({1 => 1, 2 => 2}), store_agg(where: {store_id: 2}, aggs: [:store_id])
+  end
+
+  def test_skip_complex
+    assert_equal ({1 => 1, 2 => 1}), store_agg(where: {store_id: 2, price: {gt: 5}}, aggs: [:store_id])
+  end
+
+  def test_multiple
+    assert_equal ({"store_id" => {1 => 1, 2 => 2}, "color" => {"blue" => 1, "green" => 1, "red" => 1}}), store_multiple_aggs(aggs: [:store_id, :color])
+  end
+
+  def test_smart_aggs_false
+    assert_equal ({2 => 2}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false}}}, smart_aggs: false)
+    assert_equal ({2 => 2}), store_agg(where: {color: "blue"}, aggs: {store_id: {where: {in_stock: false}}}, smart_aggs: false)
+  end
+
+  protected
+
+  def buckets_as_hash(agg)
+    Hash[agg["buckets"].map { |v| [v["key"], v["doc_count"]] }]
+  end
+
+  def store_agg(options, agg_key = "store_id")
+    buckets = Product.search("Product", options).aggs[agg_key]
+    buckets_as_hash(buckets)
+  end
+
+  def store_multiple_aggs(options)
+    Hash[Product.search("Product", options).aggs.map do |field, filtered_agg|
+      [field, buckets_as_hash(filtered_agg)]
+    end]
+  end
+end

data/test/autocomplete_test.rb

@@ -0,0 +1,65 @@
+require_relative "test_helper"
+
+class AutocompleteTest < Minitest::Test
+  def test_autocomplete
+    store_names ["Hummus"]
+    assert_search "hum", ["Hummus"], autocomplete: true
+  end
+
+  def test_autocomplete_two_words
+    store_names ["Organic Hummus"]
+    assert_search "hum", [], autocomplete: true
+  end
+
+  def test_autocomplete_fields
+    store_names ["Hummus"]
+    assert_search "hum", ["Hummus"], autocomplete: true, fields: [:name]
+  end
+
+  def test_text_start
+    store_names ["Where in the World is Carmen San Diego"]
+    assert_search "where in the world is", ["Where in the World is Carmen San Diego"], fields: [{name: :text_start}]
+    assert_search "in the world", [], fields: [{name: :text_start}]
+  end
+
+  def test_text_middle
+    store_names ["Where in the World is Carmen San Diego"]
+    assert_search "where in the world is", ["Where in the World is Carmen San Diego"], fields: [{name: :text_middle}]
+    assert_search "n the wor", ["Where in the World is Carmen San Diego"], fields: [{name: :text_middle}]
+    assert_search "men san diego", ["Where in the World is Carmen San Diego"], fields: [{name: :text_middle}]
+    assert_search "world carmen", [], fields: [{name: :text_middle}]
+  end
+
+  def test_text_end
+    store_names ["Where in the World is Carmen San Diego"]
+    assert_search "men san diego", ["Where in the World is Carmen San Diego"], fields: [{name: :text_end}]
+    assert_search "carmen san", [], fields: [{name: :text_end}]
+  end
+
+  def test_word_start
+    store_names ["Where in the World is Carmen San Diego"]
+    assert_search "car san wor", ["Where in the World is Carmen San Diego"], fields: [{name: :word_start}]
+  end
+
+  def test_word_middle
+    store_names ["Where in the World is Carmen San Diego"]
+    assert_search "orl", ["Where in the World is Carmen San Diego"], fields: [{name: :word_middle}]
+  end
+
+  def test_word_end
+    store_names ["Where in the World is Carmen San Diego"]
+    assert_search "rld men ego", ["Where in the World is Carmen San Diego"], fields: [{name: :word_end}]
+  end
+
+  def test_word_start_multiple_words
+    store_names ["Dark Grey", "Dark Blue"]
+    assert_search "dark grey", ["Dark Grey"], fields: [{name: :word_start}]
+  end
+
+  # TODO find a better place
+
+  def test_exact
+    store_names ["hi@example.org"]
+    assert_search "hi@example.org", ["hi@example.org"], fields: [{name: :exact}]
+  end
+end