searchkick 2.3.2 → 4.4.1

data/lib/searchkick/railtie.rb

@@ -0,0 +1,7 @@
+module Searckick
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load "tasks/searchkick.rake"
+    end
+  end
+end

data/lib/searchkick/record_data.rb

@@ -0,0 +1,128 @@
+module Searchkick
+  class RecordData
+    TYPE_KEYS = ["type", :type]
+
+    attr_reader :index, :record
+
+    def initialize(index, record)
+      @index = index
+      @record = record
+    end
+
+    def index_data
+      data = record_data
+      data[:data] = search_data
+      {index: data}
+    end
+
+    def update_data(method_name)
+      data = record_data
+      data[:data] = {doc: search_data(method_name)}
+      {update: data}
+    end
+
+    def delete_data
+      {delete: record_data}
+    end
+
+    def search_id
+      id = record.respond_to?(:search_document_id) ? record.search_document_id : record.id
+      id.is_a?(Numeric) ? id : id.to_s
+    end
+
+    def document_type(ignore_type = false)
+      index.klass_document_type(record.class, ignore_type)
+    end
+
+    def record_data
+      data = {
+        _index: index.name,
+        _id: search_id
+      }
+      data[:_type] = document_type if Searchkick.server_below7?
+      data[:routing] = record.search_routing if record.respond_to?(:search_routing)
+      data
+    end
+
+    private
+
+    def search_data(method_name = nil)
+      partial_reindex = !method_name.nil?
+
+      source = record.send(method_name || :search_data)
+
+      # conversions
+      index.conversions_fields.each do |conversions_field|
+        if source[conversions_field]
+          source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} }
+        end
+      end
+
+      # hack to prevent generator field doesn't exist error
+      if !partial_reindex
+        index.suggest_fields.each do |field|
+          if !source.key?(field) && !source.key?(field.to_sym)
+            source[field] = nil
+          end
+        end
+      end
+
+      # locations
+      index.locations_fields.each do |field|
+        if source[field]
+          if !source[field].is_a?(Hash) && (source[field].first.is_a?(Array) || source[field].first.is_a?(Hash))
+            # multiple locations
+            source[field] = source[field].map { |a| location_value(a) }
+          else
+            source[field] = location_value(source[field])
+          end
+        end
+      end
+
+      if index.options[:inheritance]
+        if !TYPE_KEYS.any? { |tk| source.key?(tk) }
+          source[:type] = document_type(true)
+        end
+      end
+
+      cast_big_decimal(source)
+
+      source
+    end
+
+    def location_value(value)
+      if value.is_a?(Array)
+        value.map(&:to_f).reverse
+      elsif value.is_a?(Hash)
+        {lat: value[:lat].to_f, lon: value[:lon].to_f}
+      else
+        value
+      end
+    end
+
+    # change all BigDecimal values to floats due to
+    # https://github.com/rails/rails/issues/6033
+    # possible loss of precision :/
+    def cast_big_decimal(obj)
+      case obj
+      when BigDecimal
+        obj.to_f
+      when Hash
+        obj.each do |k, v|
+          # performance
+          if v.is_a?(BigDecimal)
+            obj[k] = v.to_f
+          elsif v.is_a?(Enumerable)
+            obj[k] = cast_big_decimal(v)
+          end
+        end
+      when Enumerable
+        obj.map do |v|
+          cast_big_decimal(v)
+        end
+      else
+        obj
+      end
+    end
+  end
+end

data/lib/searchkick/record_indexer.rb

@@ -0,0 +1,79 @@
+module Searchkick
+  class RecordIndexer
+    attr_reader :record, :index
+
+    def initialize(record)
+      @record = record
+      @index = record.class.searchkick_index
+    end
+
+    def reindex(method_name = nil, refresh: false, mode: nil)
+      unless [:inline, true, nil, :async, :queue].include?(mode)
+        raise ArgumentError, "Invalid value for mode"
+      end
+
+      mode ||= Searchkick.callbacks_value || index.options[:callbacks] || true
+
+      case mode
+      when :queue
+        if method_name
+          raise Searchkick::Error, "Partial reindex not supported with queue option"
+        end
+
+        # always pass routing in case record is deleted
+        # before the queue job runs
+        if record.respond_to?(:search_routing)
+          routing = record.search_routing
+        end
+
+        # escape pipe with double pipe
+        value = queue_escape(record.id.to_s)
+        value = "#{value}|#{queue_escape(routing)}" if routing
+        index.reindex_queue.push(value)
+      when :async
+        unless defined?(ActiveJob)
+          raise Searchkick::Error, "Active Job not found"
+        end
+
+        # always pass routing in case record is deleted
+        # before the async job runs
+        if record.respond_to?(:search_routing)
+          routing = record.search_routing
+        end
+
+        Searchkick::ReindexV2Job.perform_later(
+          record.class.name,
+          record.id.to_s,
+          method_name ? method_name.to_s : nil,
+          routing: routing
+        )
+      else # bulk, inline/true/nil
+        reindex_record(method_name)
+
+        index.refresh if refresh
+      end
+    end
+
+    private
+
+    def queue_escape(value)
+      value.gsub("|", "||")
+    end
+
+    def reindex_record(method_name)
+      if record.destroyed? || !record.persisted? || !record.should_index?
+        begin
+          index.remove(record)
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          # do nothing
+        end
+      else
+        if method_name
+          index.update_record(record, method_name)
+        else
+          index.store(record)
+        end
+      end
+    end
+  end
+end

data/lib/searchkick/reindex_queue.rb

@@ -15,7 +15,7 @@ module Searchkick
     # TODO use reliable queuing
     def reserve(limit: 1000)
       record_ids = Set.new
-      while record_ids.size < limit && record_id = Searchkick.with_redis { |r| r.rpop(redis_key) }
+      while record_ids.size < limit && (record_id = Searchkick.with_redis { |r| r.rpop(redis_key) })
         record_ids << record_id
       end
       record_ids.to_a

data/lib/searchkick/reindex_v2_job.rb

@@ -9,11 +9,15 @@ module Searchkick
 
     queue_as { Searchkick.queue_name }
 
-    def perform(klass, id)
+    def perform(klass, id, method_name = nil, routing: nil)
       model = klass.constantize
       record =
         begin
-          model.find(id)
+          if model.respond_to?(:unscoped)
+            model.unscoped.find(id)
+          else
+            model.find(id)
+          end
         rescue => e
           # check by name rather than rescue directly so we don't need
           # to determine which classes are defined
@@ -21,19 +25,17 @@ module Searchkick
           nil
         end
 
-      index = model.searchkick_index
-      if !record || !record.should_index?
-        # hacky
-        record ||= model.new
+      unless record
+        record = model.new
         record.id = id
-        begin
-          index.remove record
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          # do nothing
+        if routing
+          record.define_singleton_method(:search_routing) do
+            routing
+          end
         end
-      else
-        index.store record
       end
+
+      RecordIndexer.new(record).reindex(method_name, mode: :inline)
     end
   end
 end

data/lib/searchkick/results.rb

@@ -15,40 +15,58 @@ module Searchkick
       @options = options
     end
 
-    # experimental: may not make next release
-    def records
-      @records ||= results_query(klass, hits)
+    def results
+      @results ||= with_hit.map(&:first)
     end
 
-    def results
-      @results ||= begin
+    # TODO return enumerator like with_score
+    def with_hit
+      @with_hit ||= begin
         if options[:load]
           # results can have different types
           results = {}
 
-          hits.group_by { |hit, _| hit["_type"] }.each do |type, grouped_hits|
-            results[type] = results_query(type.camelize.constantize, grouped_hits).to_a.index_by { |r| r.id.to_s }
+          hits.group_by { |hit, _| hit["_index"] }.each do |index, grouped_hits|
+            klasses =
+              if @klass
+                [@klass]
+              else
+                index_alias = index.split("_")[0..-2].join("_")
+                Array((options[:index_mapping] || {})[index_alias])
+              end
+            raise Searchkick::Error, "Unknown model for index: #{index}" unless klasses.any?
+
+            results[index] = {}
+            klasses.each do |klass|
+              results[index].merge!(results_query(klass, grouped_hits).to_a.index_by { |r| r.id.to_s })
+            end
           end
 
-          # sort
-          hits.map do |hit|
-            result = results[hit["_type"]][hit["_id"].to_s]
-            if result && !(options[:load].is_a?(Hash) && options[:load][:dumpable])
-              unless result.respond_to?(:search_hit)
-                result.define_singleton_method(:search_hit) do
-                  hit
-                end
-              end
+          missing_ids = []
 
-              if hit["highlight"] && !result.respond_to?(:search_highlights)
-                highlights = Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, v.first] }]
-                result.define_singleton_method(:search_highlights) do
-                  highlights
+          # sort
+          results =
+            hits.map do |hit|
+              result = results[hit["_index"]][hit["_id"].to_s]
+              if result && !(options[:load].is_a?(Hash) && options[:load][:dumpable])
+                if (hit["highlight"] || options[:highlight]) && !result.respond_to?(:search_highlights)
+                  highlights = hit_highlights(hit)
+                  result.define_singleton_method(:search_highlights) do
+                    highlights
+                  end
                 end
               end
+              [result, hit]
+            end.select do |result, hit|
+              missing_ids << hit["_id"] unless result
+              result
             end
-            result
-          end.compact
+
+          if missing_ids.any?
+            Searchkick.warn("Records in search index do not exist in database: #{missing_ids.join(", ")}")
+          end
+
+          results
         else
           hits.map do |hit|
             result =
@@ -60,15 +78,15 @@ module Searchkick
                 hit
               end
 
-            if hit["highlight"]
-              highlight = Hash[hit["highlight"].map { |k, v| [base_field(k), v.first] }]
+            if hit["highlight"] || options[:highlight]
+              highlight = Hash[hit["highlight"].to_a.map { |k, v| [base_field(k), v.first] }]
               options[:highlighted_fields].map { |k| base_field(k) }.each do |k|
                 result["highlighted_#{k}"] ||= (highlight[k] || result[k])
               end
             end
 
             result["id"] ||= result["_id"] # needed for legacy reasons
-            Hashie::Mash.new(result)
+            [HashWrapper.new(result), hit]
           end
         end
       end
@@ -77,25 +95,13 @@ module Searchkick
     def suggestions
       if response["suggest"]
         response["suggest"].values.flat_map { |v| v.first["options"] }.sort_by { |o| -o["score"] }.map { |o| o["text"] }.uniq
+      elsif options[:suggest] || options[:term] == "*" # TODO remove 2nd term
+        []
       else
         raise "Pass `suggest: true` to the search method for suggestions"
       end
     end
 
-    def each_with_hit(&block)
-      results.zip(hits).each(&block)
-    end
-
-    def with_details
-      each_with_hit.map do |model, hit|
-        details = {}
-        if hit["highlight"]
-          details[:highlight] = Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, v.first] }]
-        end
-        [model, details]
-      end
-    end
-
     def aggregations
       response["aggregations"]
     end
@@ -138,7 +144,13 @@ module Searchkick
     end
 
     def total_count
-      response["hits"]["total"]
+      if options[:total_entries]
+        options[:total_entries]
+      elsif response["hits"]["total"].is_a?(Hash)
+        response["hits"]["total"]["value"]
+      else
+        response["hits"]["total"]
+      end
     end
     alias_method :total_entries, :total_count
 
@@ -187,13 +199,83 @@ module Searchkick
     end
 
     def hits
-      @response["hits"]["hits"]
+      if error
+        raise Searchkick::Error, "Query error - use the error method to view it"
+      else
+        @response["hits"]["hits"]
+      end
+    end
+
+    def highlights(multiple: false)
+      hits.map do |hit|
+        hit_highlights(hit, multiple: multiple)
+      end
+    end
+
+    # TODO return enumerator like with_score
+    def with_highlights(multiple: false)
+      with_hit.map do |result, hit|
+        [result, hit_highlights(hit, multiple: multiple)]
+      end
+    end
+
+    def with_score
+      return enum_for(:with_score) unless block_given?
+
+      with_hit.each do |result, hit|
+        yield result, hit["_score"]
+      end
     end
 
     def misspellings?
       @options[:misspellings]
     end
 
+    def scroll_id
+      @response["_scroll_id"]
+    end
+
+    def scroll
+      raise Searchkick::Error, "Pass `scroll` option to the search method for scrolling" unless scroll_id
+
+      if block_given?
+        records = self
+        while records.any?
+          yield records
+          records = records.scroll
+        end
+
+        records.clear_scroll
+      else
+        params = {
+          scroll: options[:scroll],
+          scroll_id: scroll_id
+        }
+
+        begin
+          # TODO Active Support notifications for this scroll call
+          Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+          if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
+            raise Searchkick::Error, "Scroll id has expired"
+          else
+            raise e
+          end
+        end
+      end
+    end
+
+    def clear_scroll
+      begin
+        # try to clear scroll
+        # not required as scroll will expire
+        # but there is a cost to open scrolls
+        Searchkick.client.clear_scroll(scroll_id: scroll_id)
+      rescue Elasticsearch::Transport::Transport::Error
+        # do nothing
+      end
+    end
+
     private
 
     def results_query(records, hits)
@@ -215,6 +297,10 @@ module Searchkick
           end
       end
 
+      if options[:scope_results]
+        records = options[:scope_results].call(records)
+      end
+
       Searchkick.load_records(records, ids)
     end
 
@@ -231,5 +317,13 @@ module Searchkick
     def base_field(k)
       k.sub(/\.(analyzed|word_start|word_middle|word_end|text_start|text_middle|text_end|exact)\z/, "")
     end
+
+    def hit_highlights(hit, multiple: false)
+      if hit["highlight"]
+        Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, multiple ? v : v.first] }]
+      else
+        {}
+      end
+    end
   end
 end

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "2.3.2"
+  VERSION = "4.4.1"
 end