searchkick-hooopo 2.3.0

data/lib/searchkick/reindex_queue.rb

@@ -0,0 +1,38 @@
+module Searchkick
+  class ReindexQueue
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+
+      raise Searchkick::Error, "Searchkick.redis not set" unless Searchkick.redis
+    end
+
+    def push(record_id)
+      Searchkick.with_redis { |r| r.lpush(redis_key, record_id) }
+    end
+
+    # TODO use reliable queuing
+    def reserve(limit: 1000)
+      record_ids = Set.new
+      while record_ids.size < limit && record_id = Searchkick.with_redis { |r| r.rpop(redis_key) }
+        record_ids << record_id
+      end
+      record_ids.to_a
+    end
+
+    def clear
+      Searchkick.with_redis { |r| r.del(redis_key) }
+    end
+
+    def length
+      Searchkick.with_redis { |r| r.llen(redis_key) }
+    end
+
+    private
+
+    def redis_key
+      "searchkick:reindex_queue:#{name}"
+    end
+  end
+end

data/lib/searchkick/reindex_v2_job.rb

@@ -0,0 +1,39 @@
+module Searchkick
+  class ReindexV2Job < ActiveJob::Base
+    RECORD_NOT_FOUND_CLASSES = [
+      "ActiveRecord::RecordNotFound",
+      "Mongoid::Errors::DocumentNotFound",
+      "NoBrainer::Error::DocumentNotFound",
+      "Cequel::Record::RecordNotFound"
+    ]
+
+    queue_as { Searchkick.queue_name }
+
+    def perform(klass, id)
+      model = klass.constantize
+      record =
+        begin
+          model.find(id)
+        rescue => e
+          # check by name rather than rescue directly so we don't need
+          # to determine which classes are defined
+          raise e unless RECORD_NOT_FOUND_CLASSES.include?(e.class.name)
+          nil
+        end
+
+      index = model.searchkick_index
+      if !record || !record.should_index?
+        # hacky
+        record ||= model.new
+        record.id = id
+        begin
+          index.remove record
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          # do nothing
+        end
+      else
+        index.store record
+      end
+    end
+  end
+end

data/lib/searchkick/results.rb

@@ -0,0 +1,216 @@
+require "forwardable"
+
+module Searchkick
+  class Results
+    include Enumerable
+    extend Forwardable
+
+    attr_reader :klass, :response, :options
+
+    def_delegators :results, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary
+
+    def initialize(klass, response, options = {})
+      @klass = klass
+      @response = response
+      @options = options
+    end
+
+    # experimental: may not make next release
+    def records
+      @records ||= results_query(klass, hits)
+    end
+
+    def results
+      @results ||= begin
+        if options[:load]
+          # results can have different types
+          results = {}
+
+          hits.group_by { |hit, _| hit["_type"] }.each do |type, grouped_hits|
+            results[type] = results_query(type.camelize.constantize, grouped_hits).to_a.index_by { |r| r.id.to_s }
+          end
+
+          # sort
+          hits.map do |hit|
+            result = results[hit["_type"]][hit["_id"].to_s]
+            if result && !(options[:load].is_a?(Hash) && options[:load][:dumpable])
+              unless result.respond_to?(:search_hit)
+                result.define_singleton_method(:search_hit) do
+                  hit
+                end
+              end
+
+              if hit["highlight"] && !result.respond_to?(:search_highlights)
+                highlights = Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, v.first] }]
+                result.define_singleton_method(:search_highlights) do
+                  highlights
+                end
+              end
+            end
+            result
+          end.compact
+        else
+          hits.map do |hit|
+            result =
+              if hit["_source"]
+                hit.except("_source").merge(hit["_source"])
+              elsif hit["fields"]
+                hit.except("fields").merge(hit["fields"])
+              else
+                hit
+              end
+
+            if hit["highlight"]
+              highlight = Hash[hit["highlight"].map { |k, v| [base_field(k), v.first] }]
+              options[:highlighted_fields].map { |k| base_field(k) }.each do |k|
+                result["highlighted_#{k}"] ||= (highlight[k] || result[k])
+              end
+            end
+
+            result["id"] ||= result["_id"] # needed for legacy reasons
+            Hashie::Mash.new(result)
+          end
+        end
+      end
+    end
+
+    def suggestions
+      if response["suggest"]
+        response["suggest"].values.flat_map { |v| v.first["options"] }.sort_by { |o| -o["score"] }.map { |o| o["text"] }.uniq
+      else
+        raise "Pass `suggest: true` to the search method for suggestions"
+      end
+    end
+
+    def each_with_hit(&block)
+      results.zip(hits).each(&block)
+    end
+
+    def with_details
+      each_with_hit.map do |model, hit|
+        details = {}
+        if hit["highlight"]
+          details[:highlight] = Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, v.first] }]
+        end
+        [model, details]
+      end
+    end
+
+    def aggregations
+      response["aggregations"]
+    end
+
+    def aggs
+      @aggs ||= begin
+        if aggregations
+          aggregations.dup.each do |field, filtered_agg|
+            buckets = filtered_agg[field]
+            # move the buckets one level above into the field hash
+            if buckets
+              filtered_agg.delete(field)
+              filtered_agg.merge!(buckets)
+            end
+          end
+        end
+      end
+    end
+
+    def took
+      response["took"]
+    end
+
+    def error
+      response["error"]
+    end
+
+    def model_name
+      klass.model_name
+    end
+
+    def entry_name
+      model_name.human.downcase
+    end
+
+    def total_count
+      response["hits"]["total"]
+    end
+    alias_method :total_entries, :total_count
+
+    def current_page
+      options[:page]
+    end
+
+    def per_page
+      options[:per_page]
+    end
+    alias_method :limit_value, :per_page
+
+    def padding
+      options[:padding]
+    end
+
+    def total_pages
+      (total_count / per_page.to_f).ceil
+    end
+    alias_method :num_pages, :total_pages
+
+    def offset_value
+      (current_page - 1) * per_page + padding
+    end
+    alias_method :offset, :offset_value
+
+    def previous_page
+      current_page > 1 ? (current_page - 1) : nil
+    end
+    alias_method :prev_page, :previous_page
+
+    def next_page
+      current_page < total_pages ? (current_page + 1) : nil
+    end
+
+    def first_page?
+      previous_page.nil?
+    end
+
+    def last_page?
+      next_page.nil?
+    end
+
+    def out_of_range?
+      current_page > total_pages
+    end
+
+    def hits
+      @response["hits"]["hits"]
+    end
+
+    def misspellings?
+      @options[:misspellings]
+    end
+
+    private
+
+    def results_query(records, hits)
+      ids = hits.map { |hit| hit["_id"] }
+
+      if options[:includes]
+        records =
+          if defined?(NoBrainer::Document) && records < NoBrainer::Document
+            if Gem.loaded_specs["nobrainer"].version >= Gem::Version.new("0.21")
+              records.eager_load(options[:includes])
+            else
+              records.preload(options[:includes])
+            end
+          else
+            records.includes(options[:includes])
+          end
+      end
+
+      Searchkick.load_records(records, ids)
+    end
+
+    def base_field(k)
+      k.sub(/\.(analyzed|word_start|word_middle|word_end|text_start|text_middle|text_end|exact)\z/, "")
+    end
+  end
+end

data/lib/searchkick/tasks.rb

@@ -0,0 +1,33 @@
+require "rake"
+
+namespace :searchkick do
+  desc "reindex model"
+  task reindex: :environment do
+    if ENV["CLASS"]
+      klass = ENV["CLASS"].constantize rescue nil
+      if klass
+        klass.reindex
+      else
+        abort "Could not find class: #{ENV['CLASS']}"
+      end
+    else
+      abort "USAGE: rake searchkick:reindex CLASS=Product"
+    end
+  end
+
+  if defined?(Rails)
+
+    namespace :reindex do
+      desc "reindex all models"
+      task all: :environment do
+        Rails.application.eager_load!
+        Searchkick.models.each do |model|
+          puts "Reindexing #{model.name}..."
+          model.reindex
+        end
+        puts "Reindex complete"
+      end
+    end
+
+  end
+end

data/lib/searchkick/version.rb

@@ -0,0 +1,3 @@
+module Searchkick
+  VERSION = "2.3.0"
+end

data/lib/searchkick.rb

@@ -0,0 +1,215 @@
+require "active_model"
+require "elasticsearch"
+require "hashie"
+require "searchkick/version"
+require "searchkick/index_options"
+require "searchkick/index"
+require "searchkick/indexer"
+require "searchkick/reindex_queue"
+require "searchkick/results"
+require "searchkick/query"
+require "searchkick/model"
+require "searchkick/tasks"
+require "searchkick/middleware"
+require "searchkick/logging" if defined?(ActiveSupport::Notifications)
+require "active_support/core_ext/hash/deep_merge"
+
+# background jobs
+begin
+  require "active_job"
+rescue LoadError
+  # do nothing
+end
+if defined?(ActiveJob)
+  require "searchkick/bulk_reindex_job"
+  require "searchkick/process_queue_job"
+  require "searchkick/process_batch_job"
+  require "searchkick/reindex_v2_job"
+end
+
+module Searchkick
+  class Error < StandardError; end
+  class MissingIndexError < Error; end
+  class UnsupportedVersionError < Error; end
+  class InvalidQueryError < Elasticsearch::Transport::Transport::Errors::BadRequest; end
+  class DangerousOperation < Error; end
+  class ImportError < Error; end
+
+  class << self
+    attr_accessor :search_method_name, :wordnet_path, :timeout, :models, :client_options, :redis, :index_suffix, :queue_name, :searchkick_search_analyzer, :searchkick_search2_analyzer
+    attr_writer :client, :env, :search_timeout
+    attr_reader :aws_credentials
+  end
+  self.search_method_name = :search
+  self.wordnet_path = "/var/lib/wn_s.pl"
+  self.timeout = 10
+  self.models = []
+  self.client_options = {}
+  self.queue_name = :searchkick
+  self.searchkick_search_analyzer = 'searchkick_search'
+  self.searchkick_search2_analyzer = 'searchkick_search2'
+
+  def self.client
+    @client ||= begin
+      require "typhoeus/adapters/faraday" if defined?(Typhoeus)
+
+      Elasticsearch::Client.new({
+        url: ENV["ELASTICSEARCH_URL"],
+        transport_options: {request: {timeout: timeout}, headers: {content_type: "application/json"}}
+      }.deep_merge(client_options)) do |f|
+        f.use Searchkick::Middleware
+        f.request :aws_signers_v4, {
+          credentials: Aws::Credentials.new(aws_credentials[:access_key_id], aws_credentials[:secret_access_key]),
+          service_name: "es",
+          region: aws_credentials[:region] || "us-east-1"
+        } if aws_credentials
+      end
+    end
+  end
+
+  def self.env
+    @env ||= ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+  end
+
+  def self.search_timeout
+    @search_timeout || timeout
+  end
+
+  def self.server_version
+    @server_version ||= client.info["version"]["number"]
+  end
+
+  def self.server_below?(version)
+    Gem::Version.new(server_version.sub("-", ".")) < Gem::Version.new(version.sub("-", "."))
+  end
+
+  def self.search(term = "*", **options, &block)
+    klass = options[:model]
+
+    # TODO add in next major version
+    # if !klass
+    #   index_name = Array(options[:index_name])
+    #   if index_name.size == 1 && index_name.first.respond_to?(:searchkick_index)
+    #     klass = index_name.first
+    #   end
+    # end
+
+    query = Searchkick::Query.new(klass, term, options.except(:model))
+    block.call(query.body) if block
+    if options[:execute] == false
+      query
+    else
+      query.execute
+    end
+  end
+
+  def self.multi_search(queries)
+    if queries.any?
+      responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
+      queries.each_with_index do |query, i|
+        query.handle_response(responses[i])
+      end
+    end
+    queries
+  end
+
+  # callbacks
+
+  def self.enable_callbacks
+    self.callbacks_value = nil
+  end
+
+  def self.disable_callbacks
+    self.callbacks_value = false
+  end
+
+  def self.callbacks?
+    Thread.current[:searchkick_callbacks_enabled].nil? || Thread.current[:searchkick_callbacks_enabled]
+  end
+
+  def self.callbacks(value)
+    if block_given?
+      previous_value = callbacks_value
+      begin
+        self.callbacks_value = value
+        yield
+        indexer.perform if callbacks_value == :bulk
+      ensure
+        self.callbacks_value = previous_value
+      end
+    else
+      self.callbacks_value = value
+    end
+  end
+
+  def self.aws_credentials=(creds)
+    require "faraday_middleware/aws_signers_v4"
+    @aws_credentials = creds
+    @client = nil # reset client
+  end
+
+  def self.reindex_status(index_name)
+    if redis
+      batches_left = Searchkick::Index.new(index_name).batches_left
+      {
+        completed: batches_left == 0,
+        batches_left: batches_left
+      }
+    end
+  end
+
+  def self.with_redis
+    if redis
+      if redis.respond_to?(:with)
+        redis.with do |r|
+          yield r
+        end
+      else
+        yield redis
+      end
+    end
+  end
+
+  # private
+  def self.load_records(records, ids)
+    records =
+      if records.respond_to?(:primary_key)
+        # ActiveRecord
+        records.where(records.primary_key => ids) if records.primary_key
+      elsif records.respond_to?(:queryable)
+        # Mongoid 3+
+        records.queryable.for_ids(ids)
+      elsif records.respond_to?(:unscoped) && :id.respond_to?(:in)
+        # Nobrainer
+        records.unscoped.where(:id.in => ids)
+      elsif records.respond_to?(:key_column_names)
+        records.where(records.key_column_names.first => ids)
+      end
+
+    raise Searchkick::Error, "Not sure how to load records" if !records
+
+    records
+  end
+
+  # private
+  def self.indexer
+    Thread.current[:searchkick_indexer] ||= Searchkick::Indexer.new
+  end
+
+  # private
+  def self.callbacks_value
+    Thread.current[:searchkick_callbacks_enabled]
+  end
+
+  # private
+  def self.callbacks_value=(value)
+    Thread.current[:searchkick_callbacks_enabled] = value
+  end
+end
+
+# TODO find better ActiveModel hook
+ActiveModel::Callbacks.send(:include, Searchkick::Model)
+
+ActiveSupport.on_load(:active_record) do
+  extend Searchkick::Model
+end

data/searchkick.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "searchkick/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "searchkick-hooopo"
+  spec.version       = Searchkick::VERSION
+  spec.authors       = ["Andrew Kane"]
+  spec.email         = ["andrew@chartkick.com"]
+  spec.description   = "Intelligent search made easy"
+  spec.summary       = "Searchkick learns what your users are looking for. As more people search, it gets smarter and the results get better. It’s friendly for developers - and magical for your users."
+  spec.homepage      = "https://github.com/ankane/searchkick"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features|benchmark)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activemodel", ">= 4.1"
+  spec.add_dependency "elasticsearch", ">= 1"
+  spec.add_dependency "hashie"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+end