searchkick 2.5.0 → 3.0.0

data/lib/searchkick/record_data.rb

@@ -0,0 +1,133 @@
+module Searchkick
+  class RecordData
+    EXCLUDED_ATTRIBUTES = ["id", :id]
+    TYPE_KEYS = ["type", :type]
+
+    attr_reader :index, :record
+
+    def initialize(index, record)
+      @index = index
+      @record = record
+    end
+
+    def index_data
+      data = record_data
+      data[:data] = search_data
+      {index: data}
+    end
+
+    def update_data(method_name)
+      data = record_data
+      data[:data] = {doc: search_data(method_name)}
+      {update: data}
+    end
+
+    def delete_data
+      {delete: record_data}
+    end
+
+    def search_id
+      id = record.respond_to?(:search_document_id) ? record.search_document_id : record.id
+      id.is_a?(Numeric) ? id : id.to_s
+    end
+
+    def document_type(ignore_type = false)
+      index.klass_document_type(record.class, ignore_type)
+    end
+
+    private
+
+    def record_data
+      data = {
+        _index: index.name,
+        _id: search_id,
+        _type: document_type
+      }
+      data[:_routing] = record.search_routing if record.respond_to?(:search_routing)
+      data
+    end
+
+    def search_data(method_name = nil)
+      partial_reindex = !method_name.nil?
+
+      # remove _id since search_id is used instead
+      source = record.send(method_name || :search_data)
+      EXCLUDED_ATTRIBUTES.each do |attr|
+        raise Searchkick::Error, "Cannot index a field with name: #{attr}" if source[attr]
+      end
+
+      # conversions
+      index.conversions_fields.each do |conversions_field|
+        if source[conversions_field]
+          source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} }
+        end
+      end
+
+      # hack to prevent generator field doesn't exist error
+      if !partial_reindex
+        index.suggest_fields.each do |field|
+          if !source[field] && !source[field.to_sym]
+            source[field] = nil
+          end
+        end
+      end
+
+      # locations
+      index.locations_fields.each do |field|
+        if source[field]
+          if !source[field].is_a?(Hash) && (source[field].first.is_a?(Array) || source[field].first.is_a?(Hash))
+            # multiple locations
+            source[field] = source[field].map { |a| location_value(a) }
+          else
+            source[field] = location_value(source[field])
+          end
+        end
+      end
+
+      if index.options[:inheritance]
+        if !TYPE_KEYS.any? { |tk| source.key?(tk) }
+          source[:type] = document_type(true)
+        end
+      end
+
+      cast_big_decimal(source)
+
+      source
+    end
+
+    def location_value(value)
+      if value.is_a?(Array)
+        value.map(&:to_f).reverse
+      elsif value.is_a?(Hash)
+        {lat: value[:lat].to_f, lon: value[:lon].to_f}
+      else
+        value
+      end
+    end
+
+    # change all BigDecimal values to floats due to
+    # https://github.com/rails/rails/issues/6033
+    # possible loss of precision :/
+    def cast_big_decimal(obj)
+      case obj
+      when BigDecimal
+        obj.to_f
+      when Hash
+        obj.each do |k, v|
+          # performance
+          if v.is_a?(BigDecimal)
+            obj[k] = v.to_f
+          elsif v.is_a?(Enumerable) ||
+            obj[k] = cast_big_decimal(v)
+          end
+        end
+      when Enumerable
+        obj.map do |v|
+          cast_big_decimal(v)
+        end
+      else
+        obj
+      end
+    end
+  end
+end

data/lib/searchkick/record_indexer.rb

@@ -0,0 +1,55 @@
+module Searchkick
+  class RecordIndexer
+    attr_reader :record, :index
+
+    def initialize(record)
+      @record = record
+      @index = record.class.searchkick_index
+    end
+
+    def reindex(method_name = nil, refresh: false, mode: nil)
+      unless [true, nil, :async, :queue].include?(mode)
+        raise ArgumentError, "Invalid value for mode"
+      end
+
+      mode ||= Searchkick.callbacks_value || index.options[:callbacks] || true
+
+      case mode
+      when :queue
+        if method_name
+          raise Searchkick::Error, "Partial reindex not supported with queue option"
+        end
+
+        index.reindex_queue.push(record.id.to_s)
+      when :async
+        unless defined?(ActiveJob)
+          raise Searchkick::Error, "Active Job not found"
+        end
+
+        Searchkick::ReindexV2Job.perform_later(record.class.name, record.id.to_s, method_name)
+      else # bulk, true
+        reindex_record(method_name)
+
+        index.refresh if refresh
+      end
+    end
+
+    private
+
+    def reindex_record(method_name)
+      if record.destroyed? || !record.persisted? || !record.should_index?
+        begin
+          index.remove(record)
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          # do nothing
+        end
+      else
+        if method_name
+          index.update_record(record, method_name)
+        else
+          index.store(record)
+        end
+      end
+    end
+  end
+end

data/lib/searchkick/reindex_queue.rb

@@ -15,7 +15,7 @@ module Searchkick
     # TODO use reliable queuing
     def reserve(limit: 1000)
       record_ids = Set.new
-      while record_ids.size < limit && record_id = Searchkick.with_redis { |r| r.rpop(redis_key) }
+      while record_ids.size < limit && (record_id = Searchkick.with_redis { |r| r.rpop(redis_key) })
         record_ids << record_id
       end
       record_ids.to_a

data/lib/searchkick/reindex_v2_job.rb

@@ -9,11 +9,15 @@ module Searchkick
 
     queue_as { Searchkick.queue_name }
 
-    def perform(klass, id)
+    def perform(klass, id, method_name = nil)
       model = klass.constantize
       record =
         begin
-          model.find(id)
+          if model.respond_to?(:unscoped)
+            model.unscoped.find(id)
+          else
+            model.find(id)
+          end
         rescue => e
           # check by name rather than rescue directly so we don't need
           # to determine which classes are defined
@@ -21,19 +25,12 @@ module Searchkick
           nil
         end
 
-      index = model.searchkick_index
-      if !record || !record.should_index?
-        # hacky
-        record ||= model.new
+      unless record
+        record = model.new
         record.id = id
-        begin
-          index.remove record
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          # do nothing
-        end
-      else
-        index.store record
       end
+
+      RecordIndexer.new(record).reindex(method_name, mode: true)
     end
   end
 end

data/lib/searchkick/results.rb

@@ -15,11 +15,6 @@ module Searchkick
       @options = options
     end
 
-    # experimental: may not make next release
-    def records
-      @records ||= results_query(klass, hits)
-    end
-
     def results
       @results ||= begin
         if options[:load]
@@ -34,12 +29,6 @@ module Searchkick
           hits.map do |hit|
             result = results[hit["_type"]][hit["_id"].to_s]
             if result && !(options[:load].is_a?(Hash) && options[:load][:dumpable])
-              unless result.respond_to?(:search_hit)
-                result.define_singleton_method(:search_hit) do
-                  hit
-                end
-              end
-
               if hit["highlight"] && !result.respond_to?(:search_highlights)
                 highlights = Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, v.first] }]
                 result.define_singleton_method(:search_highlights) do
@@ -84,20 +73,6 @@ module Searchkick
       end
     end
 
-    def each_with_hit(&block)
-      results.zip(hits).each(&block)
-    end
-
-    def with_details
-      each_with_hit.map do |model, hit|
-        details = {}
-        if hit["highlight"]
-          details[:highlight] = Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, v.first] }]
-        end
-        [model, details]
-      end
-    end
-
     def aggregations
       response["aggregations"]
     end
@@ -196,6 +171,20 @@ module Searchkick
       end
     end
 
+    def with_hit
+      results.zip(hits)
+    end
+
+    def highlights(multiple: false)
+      hits.map do |hit|
+        Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, multiple ? v : v.first] }]
+      end
+    end
+
+    def with_highlights(multiple: false)
+      results.zip(highlights(multiple: multiple))
+    end
+
     def misspellings?
       @options[:misspellings]
     end

data/lib/searchkick/tasks.rb

@@ -1,5 +1,3 @@
-require "rake"
-
 namespace :searchkick do
   desc "reindex model"
   task reindex: :environment do
@@ -16,7 +14,6 @@ namespace :searchkick do
   end
 
   if defined?(Rails)
-
     namespace :reindex do
       desc "reindex all models"
       task all: :environment do
@@ -28,6 +25,5 @@ namespace :searchkick do
         puts "Reindex complete"
       end
     end
-
   end
 end

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "2.5.0"
+  VERSION = "3.0.0"
 end

data/searchkick.gemspec

@@ -18,11 +18,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features|benchmark)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activemodel", ">= 4.1"
-  spec.add_dependency "elasticsearch", ">= 1"
+  spec.add_dependency "activemodel", ">= 4.2"
+  spec.add_dependency "elasticsearch", ">= 5"
   spec.add_dependency "hashie"
 
   spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
+  spec.add_development_dependency "rake"
 end

data/test/boost_test.rb

@@ -40,9 +40,9 @@ class BoostTest < Minitest::Test
     assert_order "speaker", ["Speaker D", "Speaker C", "Speaker B", "Speaker A"], {conversions: "conversions_a", conversions_term: "speaker_1"}, Speaker
   end
 
-  def test_conversions_stemmed
+  def test_conversions_case
     store [
-      {name: "Tomato A", conversions: {"tomato" => 1, "tomatos" => 1, "Tomatoes" => 1}},
+      {name: "Tomato A", conversions: {"tomato" => 1, "TOMATO" => 1, "tOmAtO" => 1}},
       {name: "Tomato B", conversions: {"tomato" => 2}}
     ]
     assert_order "tomato", ["Tomato A", "Tomato B"]
@@ -122,13 +122,7 @@ class BoostTest < Minitest::Test
       {name: "Tomato B", orders_count: 10},
     ]
 
-    if elasticsearch_below50?
-      assert_raises(ArgumentError) do
-        assert_order "tomato", ["Tomato A", "Tomato B"], boost_by: {orders_count: {missing: 100}}
-      end
-    else
-      assert_order "tomato", ["Tomato A", "Tomato B"], boost_by: {orders_count: {missing: 100}}
-    end
+    assert_order "tomato", ["Tomato A", "Tomato B"], boost_by: {orders_count: {missing: 100}}
   end
 
   def test_boost_by_boost_mode_multiply

data/test/geo_shape_test.rb

@@ -116,9 +116,6 @@ class GeoShapeTest < Minitest::Test
   end
 
   def test_search_math
-    # TODO find out why this is failing
-    skip unless elasticsearch_below60?
-
     assert_search "witch", ["Region A"], {
       where: {
         territory: {
@@ -145,7 +142,6 @@ class GeoShapeTest < Minitest::Test
   end
 
   def test_contains
-    skip if elasticsearch_below22?
     assert_search "*", ["Region C"], {
       where: {
         territory: {

data/test/highlight_test.rb

@@ -3,29 +3,34 @@ require_relative "test_helper"
 class HighlightTest < Minitest::Test
   def test_basic
     store_names ["Two Door Cinema Club"]
-    assert_equal "Two Door <em>Cinema</em> Club", Product.search("cinema", fields: [:name], highlight: true).first.search_highlights[:name]
+    assert_equal "Two Door <em>Cinema</em> Club", Product.search("cinema", highlight: true).highlights.first[:name]
   end
 
   def test_tag
     store_names ["Two Door Cinema Club"]
-    assert_equal "Two Door <strong>Cinema</strong> Club", Product.search("cinema", fields: [:name], highlight: {tag: "<strong>"}).first.search_highlights[:name]
+    assert_equal "Two Door <strong>Cinema</strong> Club", Product.search("cinema", highlight: {tag: "<strong>"}).highlights.first[:name]
   end
 
   def test_tag_class
     store_names ["Two Door Cinema Club"]
-    assert_equal "Two Door <strong class='classy'>Cinema</strong> Club", Product.search("cinema", fields: [:name], highlight: {tag: "<strong class='classy'>"}).first.search_highlights[:name]
+    assert_equal "Two Door <strong class='classy'>Cinema</strong> Club", Product.search("cinema", highlight: {tag: "<strong class='classy'>"}).highlights.first[:name]
+  end
+
+  def test_very_long
+    store_names [("Two Door Cinema Club " * 100).strip]
+    assert_equal ("Two Door <em>Cinema</em> Club " * 100).strip, Product.search("cinema", highlight: true).highlights.first[:name]
   end
 
   def test_multiple_fields
     store [{name: "Two Door Cinema Club", color: "Cinema Orange"}]
-    highlights = Product.search("cinema", fields: [:name, :color], highlight: true).first.search_highlights
+    highlights = Product.search("cinema", fields: [:name, :color], highlight: true).highlights.first
     assert_equal "Two Door <em>Cinema</em> Club", highlights[:name]
     assert_equal "<em>Cinema</em> Orange", highlights[:color]
   end
 
   def test_fields
     store [{name: "Two Door Cinema Club", color: "Cinema Orange"}]
-    highlights = Product.search("cinema", fields: [:name, :color], highlight: {fields: [:name]}).first.search_highlights
+    highlights = Product.search("cinema", fields: [:name, :color], highlight: {fields: [:name]}).highlights.first
     assert_equal "Two Door <em>Cinema</em> Club", highlights[:name]
     assert_nil highlights[:color]
   end
@@ -33,22 +38,22 @@ class HighlightTest < Minitest::Test
   def test_field_options
     store_names ["Two Door Cinema Club are a Northern Irish indie rock band"]
     fragment_size = ENV["MATCH"] == "word_start" ? 26 : 21
-    assert_equal "Two Door <em>Cinema</em> Club are", Product.search("cinema", fields: [:name], highlight: {fields: {name: {fragment_size: fragment_size}}}).first.search_highlights[:name]
+    assert_equal "Two Door <em>Cinema</em> Club are", Product.search("cinema", highlight: {fields: {name: {fragment_size: fragment_size}}}).highlights.first[:name]
   end
 
   def test_multiple_words
     store_names ["Hello World Hello"]
-    assert_equal "<em>Hello</em> World <em>Hello</em>", Product.search("hello", fields: [:name], highlight: true).first.search_highlights[:name]
+    assert_equal "<em>Hello</em> World <em>Hello</em>", Product.search("hello", highlight: true).highlights.first[:name]
   end
 
   def test_encoder
     store_names ["<b>Hello</b>"]
-    assert_equal "&lt;b&gt;<em>Hello</em>&lt;&#x2F;b&gt;", Product.search("hello", fields: [:name], highlight: {encoder: "html"}, misspellings: false).first.search_highlights[:name]
+    assert_equal "&lt;b&gt;<em>Hello</em>&lt;&#x2F;b&gt;", Product.search("hello", highlight: {encoder: "html"}, misspellings: false).highlights.first[:name]
   end
 
   def test_word_middle
     store_names ["Two Door Cinema Club"]
-    assert_equal "Two Door <em>Cinema</em> Club", Product.search("ine", fields: [:name], match: :word_middle, highlight: true).first.search_highlights[:name]
+    assert_equal "Two Door <em>Cinema</em> Club", Product.search("ine", match: :word_middle, highlight: true).highlights.first[:name]
   end
 
   def test_body
@@ -68,11 +73,22 @@ class HighlightTest < Minitest::Test
         }
       }
     }
-    assert_equal "Two Door <strong>Cinema</strong> Club", Product.search(body: body).first.search_highlights[:"name.analyzed"]
+    assert_equal "Two Door <strong>Cinema</strong> Club", Product.search(body: body).highlights.first[:"name.analyzed"]
+  end
+
+  def test_multiple_highlights
+    store_names ["Two Door Cinema Club Some Other Words And Much More Doors Cinema Club"]
+    highlights = Product.search("cinema", highlight: {fragment_size: 20}).highlights(multiple: true).first[:name]
+    assert highlights.is_a?(Array)
+    assert_equal highlights.count, 2
+    refute_equal highlights.first, highlights.last
+    highlights.each do |highlight|
+      assert highlight.include?("<em>Cinema</em>")
+    end
   end
 
-  def test_legacy
+  def test_search_highlights_method
     store_names ["Two Door Cinema Club"]
-    assert_equal "Two Door <em>Cinema</em> Club", Product.search("cinema", fields: [:name], highlight: true).with_details.first[1][:highlight][:name]
+    assert_equal "Two Door <em>Cinema</em> Club", Product.search("cinema", highlight: true).first.search_highlights[:name]
   end
 end