searchkick 4.6.3 → 5.2.2

data/lib/searchkick/relation_indexer.rb

@@ -0,0 +1,155 @@
+module Searchkick
+  class RelationIndexer
+    attr_reader :index
+
+    def initialize(index)
+      @index = index
+    end
+
+    def reindex(relation, mode:, method_name: nil, full: false, resume: false, scope: nil)
+      # apply scopes
+      if scope
+        relation = relation.send(scope)
+      elsif relation.respond_to?(:search_import)
+        relation = relation.search_import
+      end
+
+      # remove unneeded loading for async and queue
+      if mode == :async || mode == :queue
+        if relation.respond_to?(:primary_key)
+          relation = relation.except(:includes, :preload)
+          unless mode == :queue && relation.klass.method_defined?(:search_routing)
+            relation = relation.except(:select).select(relation.primary_key)
+          end
+        elsif relation.respond_to?(:only)
+          unless mode == :queue && relation.klass.method_defined?(:search_routing)
+            relation = relation.only(:_id)
+          end
+        end
+      end
+
+      if mode == :async && full
+        return full_reindex_async(relation)
+      end
+
+      relation = resume_relation(relation) if resume
+
+      reindex_options = {
+        mode: mode,
+        method_name: method_name,
+        full: full
+      }
+      record_indexer = RecordIndexer.new(index)
+
+      in_batches(relation) do |items|
+        record_indexer.reindex(items, **reindex_options)
+      end
+    end
+
+    def batches_left
+      Searchkick.with_redis { |r| r.call("SCARD", batches_key) }
+    end
+
+    def batch_completed(batch_id)
+      Searchkick.with_redis { |r| r.call("SREM", batches_key, [batch_id]) }
+    end
+
+    private
+
+    def resume_relation(relation)
+      if relation.respond_to?(:primary_key)
+        # use total docs instead of max id since there's not a great way
+        # to get the max _id without scripting since it's a string
+        where = relation.arel_table[relation.primary_key].gt(index.total_docs)
+        relation = relation.where(where)
+      else
+        raise Error, "Resume not supported for Mongoid"
+      end
+    end
+
+    def in_batches(relation)
+      if relation.respond_to?(:find_in_batches)
+        klass = relation.klass
+        # remove order to prevent possible warnings
+        relation.except(:order).find_in_batches(batch_size: batch_size) do |batch|
+          # prevent scope from affecting search_data as well as inline jobs
+          # Active Record runs relation calls in scoping block
+          # https://github.com/rails/rails/blob/main/activerecord/lib/active_record/relation/delegation.rb
+          # note: we could probably just call klass.current_scope = nil
+          # anywhere in reindex method (after initial all call),
+          # but this is more cautious
+          previous_scope = klass.current_scope(true)
+          if previous_scope
+            begin
+              klass.current_scope = nil
+              yield batch
+            ensure
+              klass.current_scope = previous_scope
+            end
+          else
+            yield batch
+          end
+        end
+      else
+        klass = relation.klass
+        each_batch(relation, batch_size: batch_size) do |batch|
+          # prevent scope from affecting search_data as well as inline jobs
+          # note: Model.with_scope doesn't always restore scope, so use custom logic
+          previous_scope = Mongoid::Threaded.current_scope(klass)
+          if previous_scope
+            begin
+              Mongoid::Threaded.set_current_scope(nil, klass)
+              yield batch
+            ensure
+              Mongoid::Threaded.set_current_scope(previous_scope, klass)
+            end
+          else
+            yield batch
+          end
+        end
+      end
+    end
+
+    def each_batch(relation, batch_size:)
+      # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
+      # use cursor for Mongoid
+      items = []
+      relation.all.each do |item|
+        items << item
+        if items.length == batch_size
+          yield items
+          items = []
+        end
+      end
+      yield items if items.any?
+    end
+
+    def batch_size
+      @batch_size ||= index.options[:batch_size] || 1000
+    end
+
+    def full_reindex_async(relation)
+      batch_id = 1
+      class_name = relation.searchkick_options[:class_name]
+
+      in_batches(relation) do |items|
+        batch_job(class_name, batch_id, items.map(&:id))
+        batch_id += 1
+      end
+    end
+
+    def batch_job(class_name, batch_id, record_ids)
+      Searchkick.with_redis { |r| r.call("SADD", batches_key, [batch_id]) }
+      Searchkick::BulkReindexJob.perform_later(
+        class_name: class_name,
+        index_name: index.name,
+        batch_id: batch_id,
+        record_ids: record_ids.map { |v| v.instance_of?(Integer) ? v : v.to_s }
+      )
+    end
+
+    def batches_key
+      "searchkick:reindex:#{index.name}:batches"
+    end
+  end
+end

data/lib/searchkick/results.rb

@@ -1,10 +1,9 @@
-require "forwardable"
-
 module Searchkick
   class Results
     include Enumerable
     extend Forwardable
 
+    # TODO remove klass and options in 6.0
     attr_reader :klass, :response, :options
 
     def_delegators :results, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary
@@ -15,17 +14,16 @@ module Searchkick
       @options = options
     end
 
+    # TODO make private in 6.0
     def results
       @results ||= with_hit.map(&:first)
     end
 
-    # TODO return enumerator like with_score
     def with_hit
-      @with_hit ||= begin
-        if missing_records.any?
-          Searchkick.warn("Records in search index do not exist in database: #{missing_records.map { |v| v[:id] }.join(", ")}")
-        end
-        with_hit_and_missing_records[0]
+      return enum_for(:with_hit) unless block_given?
+
+      build_hits.each do |result|
+        yield result
       end
     end
 
@@ -145,7 +143,7 @@ module Searchkick
 
     def hits
       if error
-        raise Searchkick::Error, "Query error - use the error method to view it"
+        raise Error, "Query error - use the error method to view it"
       else
         @response["hits"]["hits"]
       end
@@ -157,10 +155,11 @@ module Searchkick
       end
     end
 
-    # TODO return enumerator like with_score
     def with_highlights(multiple: false)
-      with_hit.map do |result, hit|
-        [result, hit_highlights(hit, multiple: multiple)]
+      return enum_for(:with_highlights, multiple: multiple) unless block_given?
+
+      with_hit.each do |result, hit|
+        yield result, hit_highlights(hit, multiple: multiple)
       end
     end
 
@@ -181,7 +180,7 @@ module Searchkick
     end
 
     def scroll
-      raise Searchkick::Error, "Pass `scroll` option to the search method for scrolling" unless scroll_id
+      raise Error, "Pass `scroll` option to the search method for scrolling" unless scroll_id
 
       if block_given?
         records = self
@@ -194,10 +193,10 @@ module Searchkick
       else
         begin
           # TODO Active Support notifications for this scroll call
-          Searchkick::Results.new(@klass, Searchkick.client.scroll(scroll: options[:scroll], body: {scroll_id: scroll_id}), @options)
+          Results.new(@klass, Searchkick.client.scroll(scroll: options[:scroll], body: {scroll_id: scroll_id}), @options)
         rescue => e
           if Searchkick.not_found_error?(e) && e.message =~ /search_context_missing_exception/i
-            raise Searchkick::Error, "Scroll id has expired"
+            raise Error, "Scroll id has expired"
           else
             raise e
           end
@@ -235,7 +234,7 @@ module Searchkick
                 index_alias = index.split("_")[0..-2].join("_")
                 Array((options[:index_mapping] || {})[index_alias])
               end
-            raise Searchkick::Error, "Unknown model for index: #{index}. Pass the `models` option to the search method." unless models.any?
+            raise Error, "Unknown model for index: #{index}. Pass the `models` option to the search method." unless models.any?
             index_models[index] = models
           end
 
@@ -287,7 +286,7 @@ module Searchkick
                 end
 
               if hit["highlight"] || options[:highlight]
-                highlight = Hash[hit["highlight"].to_a.map { |k, v| [base_field(k), v.first] }]
+                highlight = hit["highlight"].to_a.to_h { |k, v| [base_field(k), v.first] }
                 options[:highlighted_fields].map { |k| base_field(k) }.each do |k|
                   result["highlighted_#{k}"] ||= (highlight[k] || result[k])
                 end
@@ -302,23 +301,25 @@ module Searchkick
       end
     end
 
+    def build_hits
+      @build_hits ||= begin
+        if missing_records.any?
+          Searchkick.warn("Records in search index do not exist in database: #{missing_records.map { |v| "#{v[:model].model_name} #{v[:id]}" }.join(", ")}")
+        end
+        with_hit_and_missing_records[0]
+      end
+    end
+
     def results_query(records, hits)
+      records = Searchkick.scope(records)
+
       ids = hits.map { |hit| hit["_id"] }
       if options[:includes] || options[:model_includes]
         included_relations = []
         combine_includes(included_relations, options[:includes])
         combine_includes(included_relations, options[:model_includes][records]) if options[:model_includes]
 
-        records =
-          if defined?(NoBrainer::Document) && records < NoBrainer::Document
-            if Gem.loaded_specs["nobrainer"].version >= Gem::Version.new("0.21")
-              records.eager_load(included_relations)
-            else
-              records.preload(included_relations)
-            end
-          else
-            records.includes(included_relations)
-          end
+        records = records.includes(included_relations)
       end
 
       if options[:scope_results]
@@ -344,7 +345,7 @@ module Searchkick
 
     def hit_highlights(hit, multiple: false)
       if hit["highlight"]
-        Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, multiple ? v : v.first] }]
+        hit["highlight"].to_h { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, multiple ? v : v.first] }
       else
         {}
       end

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "4.6.3"
+  VERSION = "5.2.2"
 end

data/lib/searchkick/where.rb

@@ -0,0 +1,11 @@
+module Searchkick
+  class Where
+    def initialize(relation)
+      @relation = relation
+    end
+
+    def not(value)
+      @relation.where(_not: value)
+    end
+  end
+end