searchkick-sinneduy 0.9.0

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+task default: :test
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+end

data/ci/before_install.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.3.2.deb
+sudo dpkg -i elasticsearch-1.3.2.deb
+sudo service elasticsearch restart
+
+if [ -n "$NOBRAINER" ]; then
+  source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list
+  wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add -
+  sudo apt-get update -q
+  sudo apt-get install rethinkdb
+  sudo cp /etc/rethinkdb/default.conf.sample /etc/rethinkdb/instances.d/instance1.conf
+  sudo service rethinkdb restart
+fi

data/gemfiles/activerecord31.gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "sqlite3"
+gem "activerecord", "~> 3.1.0"

data/gemfiles/activerecord32.gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "sqlite3"
+gem "activerecord", "~> 3.2.0"

data/gemfiles/activerecord40.gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "sqlite3"
+gem "activerecord", "~> 4.0.0"
+gem "activejob_backport"

data/gemfiles/activerecord41.gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "sqlite3"
+gem "activerecord", "~> 4.1.0"
+gem "activejob_backport"

data/gemfiles/mongoid2.gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "mongoid", "~> 2"
+gem "bson_ext"

data/gemfiles/mongoid3.gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "mongoid", "~> 3.1.0"

data/gemfiles/mongoid4.gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "mongoid", "~> 4.0.0"
+gem "activejob_backport"

data/gemfiles/nobrainer.gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "nobrainer", "0.22.0"

data/lib/searchkick.rb

@@ -0,0 +1,72 @@
+require "active_model"
+require "elasticsearch"
+require "hashie"
+require "searchkick/version"
+require "searchkick/index"
+require "searchkick/results"
+require "searchkick/query"
+require "searchkick/reindex_job"
+require "searchkick/model"
+require "searchkick/tasks"
+require "searchkick/logging" if defined?(Rails)
+
+# background jobs
+begin
+  require "active_job"
+rescue LoadError
+  # do nothing
+end
+require "searchkick/reindex_v2_job" if defined?(ActiveJob)
+
+module Searchkick
+  class MissingIndexError < StandardError; end
+  class UnsupportedVersionError < StandardError; end
+  class InvalidQueryError < Elasticsearch::Transport::Transport::Errors::BadRequest; end
+
+  class << self
+    attr_accessor :search_method_name
+    attr_accessor :wordnet_path
+    attr_accessor :timeout
+    attr_accessor :models
+  end
+  self.search_method_name = :search
+  self.wordnet_path = "/var/lib/wn_s.pl"
+  self.timeout = 10
+  self.models = []
+
+  def self.client
+    @client ||=
+      Elasticsearch::Client.new(
+        url: ENV["ELASTICSEARCH_URL"],
+        transport_options: {request: {timeout: timeout}}
+      )
+  end
+
+  class << self
+    attr_writer :client
+  end
+
+  def self.server_version
+    @server_version ||= client.info["version"]["number"]
+  end
+
+  def self.enable_callbacks
+    Thread.current[:searchkick_callbacks_enabled] = true
+  end
+
+  def self.disable_callbacks
+    Thread.current[:searchkick_callbacks_enabled] = false
+  end
+
+  def self.callbacks?
+    Thread.current[:searchkick_callbacks_enabled].nil? || Thread.current[:searchkick_callbacks_enabled]
+  end
+
+  def self.env
+    @env ||= ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+  end
+end
+
+# TODO find better ActiveModel hook
+ActiveModel::Callbacks.send(:include, Searchkick::Model)
+ActiveRecord::Base.send(:extend, Searchkick::Model) if defined?(ActiveRecord)

data/lib/searchkick/index.rb

@@ -0,0 +1,550 @@
+module Searchkick
+  class Index
+    attr_reader :name, :options
+
+    def initialize(name, options = {})
+      @name = name
+      @options = options
+    end
+
+    def create(options = {})
+      client.indices.create index: name, body: options
+    end
+
+    def delete
+      client.indices.delete index: name
+    end
+
+    def exists?
+      client.indices.exists index: name
+    end
+
+    def refresh
+      client.indices.refresh index: name
+    end
+
+    def alias_exists?
+      client.indices.exists_alias name: name
+    end
+
+    def swap(new_name)
+      old_indices =
+        begin
+          client.indices.get_alias(name: name).keys
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          []
+        end
+      actions = old_indices.map { |old_name| {remove: {index: old_name, alias: name}} } + [{add: {index: new_name, alias: name}}]
+      client.indices.update_aliases body: {actions: actions}
+    end
+
+    # record based
+
+    def store(record)
+      client.index(
+        index: name,
+        type: document_type(record),
+        id: search_id(record),
+        body: search_data(record)
+      )
+    end
+
+    def remove(record)
+      id = search_id(record)
+      unless id.blank?
+        client.delete(
+          index: name,
+          type: document_type(record),
+          id: id
+        )
+      end
+    end
+
+    def import(records)
+      records.group_by { |r| document_type(r) }.each do |type, batch|
+        client.bulk(
+          index: name,
+          type: type,
+          body: batch.map { |r| {index: {_id: search_id(r), data: search_data(r)}} }
+        )
+      end
+    end
+
+    def retrieve(record)
+      client.get(
+        index: name,
+        type: document_type(record),
+        id: search_id(record)
+      )["_source"]
+    end
+
+    def reindex_record(record)
+      if record.destroyed? || !record.should_index?
+        begin
+          remove(record)
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          # do nothing
+        end
+      else
+        store(record)
+      end
+    end
+
+    def reindex_record_async(record)
+      if defined?(Searchkick::ReindexV2Job)
+        Searchkick::ReindexV2Job.perform_later(record.class.name, record.id.to_s)
+      else
+        Delayed::Job.enqueue Searchkick::ReindexJob.new(record.class.name, record.id.to_s)
+      end
+    end
+
+    def similar_record(record, options = {})
+      like_text = retrieve(record).to_hash
+        .keep_if { |k, v| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
+        .values.compact.join(" ")
+
+      # TODO deep merge method
+      options[:where] ||= {}
+      options[:where][:_id] ||= {}
+      options[:where][:_id][:not] = record.id.to_s
+      options[:limit] ||= 10
+      options[:similar] = true
+
+      # TODO use index class instead of record class
+      search_model(record.class, like_text, options)
+    end
+
+    # search
+
+    def search_model(searchkick_klass, term = nil, options = {}, &block)
+      query = Searchkick::Query.new(searchkick_klass, term, options)
+      if block
+        block.call(query.body)
+      end
+      if options[:execute] == false
+        query
+      else
+        query.execute
+      end
+    end
+
+    # reindex
+
+    def create_index
+      index = Searchkick::Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
+      index.create(index_options)
+      index
+    end
+
+    # remove old indices that start w/ index_name
+    def clean_indices
+      all_indices = client.indices.get_aliases
+      indices = all_indices.select { |k, v| (v.empty? || v["aliases"].empty?) && k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
+      indices.each do |index|
+        Searchkick::Index.new(index).delete
+      end
+      indices
+    end
+
+    # https://gist.github.com/jarosan/3124884
+    # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
+    def reindex_scope(scope, options = {})
+      skip_import = options[:import] == false
+
+      clean_indices
+
+      index = create_index
+
+      # check if alias exists
+      if alias_exists?
+        # import before swap
+        index.import_scope(scope) unless skip_import
+
+        # get existing indices to remove
+        swap(index.name)
+        clean_indices
+      else
+        delete if exists?
+        swap(index.name)
+
+        # import after swap
+        index.import_scope(scope) unless skip_import
+      end
+
+      index.refresh
+
+      true
+    end
+
+    def import_scope(scope)
+      batch_size = @options[:batch_size] || 1000
+
+      # use scope for import
+      scope = scope.search_import if scope.respond_to?(:search_import)
+      if scope.respond_to?(:find_in_batches)
+        scope.find_in_batches batch_size: batch_size do |batch|
+          import batch.select(&:should_index?)
+        end
+      else
+        # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
+        # use cursor for Mongoid
+        items = []
+        scope.all.each do |item|
+          items << item if item.should_index?
+          if items.length == batch_size
+            import items
+            items = []
+          end
+        end
+        import items
+      end
+    end
+
+    def index_options
+      options = @options
+
+      if options[:mappings] && !options[:merge_mappings]
+        settings = options[:settings] || {}
+        mappings = options[:mappings]
+      else
+        settings = {
+          analysis: {
+            analyzer: {
+              searchkick_keyword: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase"] + (options[:stem_conversions] == false ? [] : ["searchkick_stemmer"])
+              },
+              default_index: {
+                type: "custom",
+                tokenizer: "standard",
+                # synonym should come last, after stemming and shingle
+                # shingle must come before searchkick_stemmer
+                filter: ["standard", "lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"]
+              },
+              searchkick_search: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["standard", "lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"]
+              },
+              searchkick_search2: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["standard", "lowercase", "asciifolding", "searchkick_stemmer"]
+              },
+              # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
+              searchkick_autocomplete_index: {
+                type: "custom",
+                tokenizer: "searchkick_autocomplete_ngram",
+                filter: ["lowercase", "asciifolding"]
+              },
+              searchkick_autocomplete_search: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding"]
+              },
+              searchkick_word_search: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding"]
+              },
+              searchkick_suggest_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
+              },
+              searchkick_text_start_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+              },
+              searchkick_text_middle_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+              },
+              searchkick_text_end_index: {
+                type: "custom",
+                tokenizer: "keyword",
+                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
+              },
+              searchkick_word_start_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
+              },
+              searchkick_word_middle_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "searchkick_ngram"]
+              },
+              searchkick_word_end_index: {
+                type: "custom",
+                tokenizer: "standard",
+                filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
+              }
+            },
+            filter: {
+              searchkick_index_shingle: {
+                type: "shingle",
+                token_separator: ""
+              },
+              # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7
+              searchkick_search_shingle: {
+                type: "shingle",
+                token_separator: "",
+                output_unigrams: false,
+                output_unigrams_if_no_shingles: true
+              },
+              searchkick_suggest_shingle: {
+                type: "shingle",
+                max_shingle_size: 5
+              },
+              searchkick_edge_ngram: {
+                type: "edgeNGram",
+                min_gram: 1,
+                max_gram: 50
+              },
+              searchkick_ngram: {
+                type: "nGram",
+                min_gram: 1,
+                max_gram: 50
+              },
+              searchkick_stemmer: {
+                type: "snowball",
+                language: options[:language] || "English"
+              }
+            },
+            tokenizer: {
+              searchkick_autocomplete_ngram: {
+                type: "edgeNGram",
+                min_gram: 1,
+                max_gram: 50
+              }
+            }
+          }
+        }
+
+        if Searchkick.env == "test"
+          settings.merge!(number_of_shards: 1, number_of_replicas: 0)
+        end
+
+        settings.deep_merge!(options[:settings] || {})
+
+        # synonyms
+        synonyms = options[:synonyms] || []
+        if synonyms.any?
+          settings[:analysis][:filter][:searchkick_synonym] = {
+            type: "synonym",
+            synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.join(",") }
+          }
+          # choosing a place for the synonym filter when stemming is not easy
+          # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
+          # TODO use a snowball stemmer on synonyms when creating the token filter
+
+          # http://elasticsearch-users.115913.n3.nabble.com/synonym-multi-words-search-td4030811.html
+          # I find the following approach effective if you are doing multi-word synonyms (synonym phrases):
+          # - Only apply the synonym expansion at index time
+          # - Don't have the synonym filter applied search
+          # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
+          settings[:analysis][:analyzer][:default_index][:filter].insert(4, "searchkick_synonym")
+          settings[:analysis][:analyzer][:default_index][:filter] << "searchkick_synonym"
+        end
+
+        if options[:wordnet]
+          settings[:analysis][:filter][:searchkick_wordnet] = {
+            type: "synonym",
+            format: "wordnet",
+            synonyms_path: Searchkick.wordnet_path
+          }
+
+          settings[:analysis][:analyzer][:default_index][:filter].insert(4, "searchkick_wordnet")
+          settings[:analysis][:analyzer][:default_index][:filter] << "searchkick_wordnet"
+        end
+
+        if options[:special_characters] == false
+          settings[:analysis][:analyzer].each do |analyzer, analyzer_settings|
+            analyzer_settings[:filter].reject! { |f| f == "asciifolding" }
+          end
+        end
+
+        mapping = {}
+
+        # conversions
+        if options[:conversions]
+          mapping[:conversions] = {
+            type: "nested",
+            properties: {
+              query: {type: "string", analyzer: "searchkick_keyword"},
+              count: {type: "integer"}
+            }
+          }
+        end
+
+        mapping_options = Hash[
+          [:autocomplete, :suggest, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight]
+            .map { |type| [type, (options[type] || []).map(&:to_s)] }
+        ]
+
+        mapping_options.values.flatten.uniq.each do |field|
+          field_mapping = {
+            type: "multi_field",
+            fields: {
+              field => {type: "string", index: "not_analyzed"},
+              "analyzed" => {type: "string", index: "analyzed"}
+              # term_vector: "with_positions_offsets" for fast / correct highlighting
+              # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-highlighting.html#_fast_vector_highlighter
+            }
+          }
+
+          mapping_options.except(:highlight).each do |type, fields|
+            if fields.include?(field)
+              field_mapping[:fields][type] = {type: "string", index: "analyzed", analyzer: "searchkick_#{type}_index"}
+            end
+          end
+
+          if mapping_options[:highlight].include?(field)
+            field_mapping[:fields]["analyzed"][:term_vector] = "with_positions_offsets"
+          end
+
+          mapping[field] = field_mapping
+        end
+
+        (options[:locations] || []).map(&:to_s).each do |field|
+          mapping[field] = {
+            type: "geo_point"
+          }
+        end
+
+        (options[:unsearchable] || []).map(&:to_s).each do |field|
+          mapping[field] = {
+            type: "string",
+            index: "no"
+          }
+        end
+
+        routing = {}
+        if options[:routing]
+          routing = {required: true, path: options[:routing].to_s}
+        end
+
+        mappings = {
+          _default_: {
+            properties: mapping,
+            _routing: routing,
+            # https://gist.github.com/kimchy/2898285
+            dynamic_templates: [
+              {
+                string_template: {
+                  match: "*",
+                  match_mapping_type: "string",
+                  mapping: {
+                    # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
+                    type: "multi_field",
+                    fields: {
+                      # analyzed field must be the default field for include_in_all
+                      # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
+                      # however, we can include the not_analyzed field in _all
+                      # and the _all index analyzer will take care of it
+                      "{name}" => {type: "string", index: "not_analyzed"},
+                      "analyzed" => {type: "string", index: "analyzed"}
+                    }
+                  }
+                }
+              }
+            ]
+          }
+        }.deep_merge(options[:mappings] || {})
+      end
+
+      {
+        settings: settings,
+        mappings: mappings
+      }
+    end
+
+    # other
+
+    def tokens(text, options = {})
+      client.indices.analyze({text: text, index: name}.merge(options))["tokens"].map { |t| t["token"] }
+    end
+
+    def klass_document_type(klass)
+      if klass.respond_to?(:document_type)
+        klass.document_type
+      else
+        klass.model_name.to_s.underscore
+      end
+    end
+
+    protected
+
+    def client
+      Searchkick.client
+    end
+
+    def document_type(record)
+      klass_document_type(record.class)
+    end
+
+    def search_id(record)
+      record.id.is_a?(Numeric) ? record.id : record.id.to_s
+    end
+
+    def search_data(record)
+      source = record.search_data
+      options = record.class.searchkick_options
+
+      # stringify fields
+      # remove _id since search_id is used instead
+      source = source.inject({}) { |memo, (k, v)| memo[k.to_s] = v; memo }.except("_id")
+
+      # conversions
+      conversions_field = options[:conversions]
+      if conversions_field && source[conversions_field]
+        source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} }
+      end
+
+      # hack to prevent generator field doesn't exist error
+      (options[:suggest] || []).map(&:to_s).each do |field|
+        source[field] = nil unless source[field]
+      end
+
+      # locations
+      (options[:locations] || []).map(&:to_s).each do |field|
+        if source[field]
+          if source[field].first.is_a?(Array) # array of arrays
+            source[field] = source[field].map { |a| a.map(&:to_f).reverse }
+          else
+            source[field] = source[field].map(&:to_f).reverse
+          end
+        end
+      end
+
+      cast_big_decimal(source)
+
+      source.as_json
+    end
+
+    # change all BigDecimal values to floats due to
+    # https://github.com/rails/rails/issues/6033
+    # possible loss of precision :/
+    def cast_big_decimal(obj)
+      case obj
+      when BigDecimal
+        obj.to_f
+      when Hash
+        obj.each do |k, v|
+          obj[k] = cast_big_decimal(v)
+        end
+      when Enumerable
+        obj.map do |v|
+          cast_big_decimal(v)
+        end
+      else
+        obj
+      end
+    end
+
+  end
+end