searchkick 2.5.0 → 3.0.0

data/Rakefile

@@ -14,7 +14,3 @@ Rake::TestTask.new do |t|
   t.pattern = "test/**/*_test.rb"
   t.warning = false
 end
-
-task :benchmark do
-  require_relative "benchmark/benchmark"
-end

data/benchmark/Gemfile

@@ -3,9 +3,9 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in searchkick.gemspec
 gemspec path: "../"
 
-# gem "sqlite3"
+gem "sqlite3"
 gem "pg"
-gem "activerecord", "~> 5.0.0"
+gem "activerecord", "~> 5.1.0"
 gem "activerecord-import"
 gem "activejob"
 gem "redis"
@@ -21,3 +21,4 @@ gem "allocation_stats"
 gem "get_process_mem"
 gem "memory_profiler"
 gem "allocation_tracer"
+gem "benchmark-ips"

data/benchmark/{benchmark.rb → index.rb}

@@ -6,7 +6,7 @@ require "active_support/notifications"
 
 ActiveSupport::Notifications.subscribe "request.searchkick" do |*args|
   event = ActiveSupport::Notifications::Event.new(*args)
-  p event.duration
+  puts "Import: #{event.duration.round}ms"
 end
 
 ActiveJob::Base.queue_adapter = :sidekiq
@@ -33,55 +33,57 @@ class Product < ActiveRecord::Base
   end
 end
 
-total_docs = 100000
+if ENV["SETUP"]
+  total_docs = 100000
 
-# ActiveRecord::Migration.create_table :products, force: :cascade do |t|
-#   t.string :name
-#   t.string :color
-#   t.integer :store_id
-# end
+  ActiveRecord::Migration.create_table :products, force: :cascade do |t|
+    t.string :name
+    t.string :color
+    t.integer :store_id
+  end
 
-# Product.import ["name", "color", "store_id"], total_docs.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] }
+  Product.import ["name", "color", "store_id"], total_docs.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] }
 
-puts "Imported"
+  puts "Imported"
+end
 
 result = nil
 report = nil
 stats = nil
 
-# p GetProcessMem.new.mb
-
 Product.searchkick_index.delete rescue nil
 
+GC.start
+GC.disable
+start_mem = GetProcessMem.new.mb
+
 time =
   Benchmark.realtime do
     # result = RubyProf.profile do
     # report = MemoryProfiler.report do
     # stats = AllocationStats.trace do
-    reindex = Product.reindex(async: true)
-    p reindex
+    reindex = Product.reindex #(async: true)
+    # p reindex
     # end
 
-    60.times do |i|
-      if reindex.is_a?(Hash)
-        docs = Searchkick::Index.new(reindex[:index_name]).total_docs
-      else
-        docs = Product.searchkick_index.total_docs
-      end
-      puts "#{i}: #{docs}"
-      if docs == total_docs
-        break
-      end
-      p Searchkick.reindex_status(reindex[:index_name]) if reindex.is_a?(Hash)
-      sleep(1)
-      # Product.searchkick_index.refresh
-    end
+    # 60.times do |i|
+    #   if reindex.is_a?(Hash)
+    #     docs = Searchkick::Index.new(reindex[:index_name]).total_docs
+    #   else
+    #     docs = Product.searchkick_index.total_docs
+    #   end
+    #   puts "#{i}: #{docs}"
+    #   if docs == total_docs
+    #     break
+    #   end
+    #   p Searchkick.reindex_status(reindex[:index_name]) if reindex.is_a?(Hash)
+    #   sleep(1)
+    #   # Product.searchkick_index.refresh
+    # end
   end
 
-# p GetProcessMem.new.mb
-
-puts time.round(1)
-
+puts
+puts "Time: #{time.round(1)}s"
 
 if result
   printer = RubyProf::GraphPrinter.new(result)

data/benchmark/search.rb

@@ -0,0 +1,48 @@
+require "bundler/setup"
+Bundler.require(:default)
+require "active_record"
+require "benchmark/ips"
+
+ActiveRecord::Base.default_timezone = :utc
+ActiveRecord::Base.time_zone_aware_attributes = true
+ActiveRecord::Base.establish_connection adapter: "sqlite3", database: "/tmp/searchkick"
+
+class Product < ActiveRecord::Base
+  searchkick batch_size: 1000
+
+  def search_data
+    {
+      name: name,
+      color: color,
+      store_id: store_id
+    }
+  end
+end
+
+if ENV["SETUP"]
+  total_docs = 1000000
+
+  ActiveRecord::Migration.create_table :products, force: :cascade do |t|
+    t.string :name
+    t.string :color
+    t.integer :store_id
+  end
+
+  Product.import ["name", "color", "store_id"], total_docs.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] }
+
+  puts "Imported"
+
+  Product.reindex
+
+  puts "Reindexed"
+end
+
+query = Product.search("product", fields: [:name], where: {color: "red", store_id: 5}, limit: 10000, load: false, execute: false)
+
+require "pp"
+pp query.body.as_json
+puts
+
+Benchmark.ips do |x|
+  x.report { query.dup.execute }
+end

data/docs/Searchkick-3-Upgrade.md

@@ -0,0 +1,57 @@
+# Searchkick 3 Upgrade
+
+## Before You Upgrade
+
+Searchkick 3 no longer uses types, since they are deprecated in Elasticsearch 6.
+
+If you use inheritance, add to your parent model:
+
+```ruby
+class Animal < ApplicationRecord
+  searchkick inheritance: true
+end
+```
+
+And do a full reindex before upgrading.
+
+## Upgrading
+
+Update your Gemfile:
+
+```ruby
+gem 'searchkick', '~> 3'
+```
+
+And run:
+
+```sh
+bundle update searchkick
+```
+
+We recommend you don’t stem conversions anymore, so conversions for `pepper` don’t affect `peppers`, but if you want to keep the old behavior, use:
+
+```ruby
+Searchkick.model_options = {
+  stem_conversions: true
+}
+```
+
+Searchkick 3 disables the `_all` field by default, since Elasticsearch 6 removes the ability to reindex with it. If you’re on Elasticsearch 5 and still need it, add to your model:
+
+```ruby
+class Product < ApplicationRecord
+  searchkick _all: true
+end
+```
+
+If you use `record.reindex_async` or `record.reindex(async: true)`, replace it with:
+
+```ruby
+record.reindex(mode: :async)
+```
+
+If you use `log: true` with `boost_by`, replace it with `modifier: "ln2p"`.
+
+If you use the `body` option and have warnings about incompatible options, remove them, as they now throw an `ArgumentError`.
+
+Check out the [changelog](https://github.com/ankane/searchkick/blob/master/CHANGELOG.md) for the full list of changes.

data/lib/searchkick.rb

@@ -1,20 +1,37 @@
 require "active_model"
+require "active_support/core_ext/hash/deep_merge"
 require "elasticsearch"
 require "hashie"
-require "searchkick/version"
-require "searchkick/index_options"
+
+require "searchkick/bulk_indexer"
 require "searchkick/index"
 require "searchkick/indexer"
-require "searchkick/reindex_queue"
 require "searchkick/hash_wrapper"
-require "searchkick/results"
-require "searchkick/query"
-require "searchkick/multi_search"
-require "searchkick/model"
-require "searchkick/tasks"
 require "searchkick/middleware"
+require "searchkick/model"
+require "searchkick/multi_search"
+require "searchkick/query"
+require "searchkick/reindex_queue"
+require "searchkick/record_data"
+require "searchkick/record_indexer"
+require "searchkick/results"
+require "searchkick/version"
+
 require "searchkick/logging" if defined?(ActiveSupport::Notifications)
-require "active_support/core_ext/hash/deep_merge"
+
+begin
+  require "rake"
+rescue LoadError
+  # do nothing
+end
+require "searchkick/tasks" if defined?(Rake)
+
+begin
+  require "rake"
+rescue LoadError
+  # do nothing
+end
+require "searchkick/tasks" if defined?(Rake)
 
 # background jobs
 begin
@@ -24,8 +41,8 @@ rescue LoadError
 end
 if defined?(ActiveJob)
   require "searchkick/bulk_reindex_job"
-  require "searchkick/process_queue_job"
   require "searchkick/process_batch_job"
+  require "searchkick/process_queue_job"
   require "searchkick/reindex_v2_job"
 end
 
@@ -38,7 +55,7 @@ module Searchkick
   class ImportError < Error; end
 
   class << self
-    attr_accessor :search_method_name, :wordnet_path, :timeout, :models, :client_options, :redis, :index_prefix, :index_suffix, :queue_name
+    attr_accessor :search_method_name, :wordnet_path, :timeout, :models, :client_options, :redis, :index_prefix, :index_suffix, :queue_name, :model_options
     attr_writer :client, :env, :search_timeout
     attr_reader :aws_credentials
   end
@@ -48,6 +65,7 @@ module Searchkick
   self.models = []
   self.client_options = {}
   self.queue_name = :searchkick
+  self.model_options = {}
 
   def self.client
     @client ||= begin
@@ -80,18 +98,18 @@ module Searchkick
     Gem::Version.new(server_version.sub("-", ".")) < Gem::Version.new(version.sub("-", "."))
   end
 
-  def self.search(term = "*", **options, &block)
-    klass = options[:model]
+  def self.search(term = "*", model: nil, **options, &block)
+    klass = model
 
-    # TODO add in next major version
-    # if !klass
-    #   index_name = Array(options[:index_name])
-    #   if index_name.size == 1 && index_name.first.respond_to?(:searchkick_index)
-    #     klass = index_name.first
-    #   end
-    # end
+    # make Searchkick.search(index_name: [Product]) and Product.search equivalent
+    unless klass
+      index_name = Array(options[:index_name])
+      if index_name.size == 1 && index_name.first.respond_to?(:searchkick_index)
+        klass = index_name.first
+      end
+    end
 
-    query = Searchkick::Query.new(klass, term, options.except(:model))
+    query = Searchkick::Query.new(klass, term, options)
     block.call(query.body) if block
     if options[:execute] == false
       query
@@ -100,8 +118,8 @@ module Searchkick
     end
   end
 
-  def self.multi_search(queries, retry_misspellings: false)
-    Searchkick::MultiSearch.new(queries, retry_misspellings: retry_misspellings).perform
+  def self.multi_search(queries)
+    Searchkick::MultiSearch.new(queries).perform
   end
 
   # callbacks
@@ -114,8 +132,12 @@ module Searchkick
     self.callbacks_value = false
   end
 
-  def self.callbacks?
-    Thread.current[:searchkick_callbacks_enabled].nil? || Thread.current[:searchkick_callbacks_enabled]
+  def self.callbacks?(default: true)
+    if callbacks_value.nil?
+      default
+    else
+      callbacks_value != false
+    end
   end
 
   def self.callbacks(value)
@@ -123,8 +145,9 @@ module Searchkick
       previous_value = callbacks_value
       begin
         self.callbacks_value = value
-        yield
+        result = yield
         indexer.perform if callbacks_value == :bulk
+        result
       ensure
         self.callbacks_value = previous_value
       end
@@ -223,7 +246,7 @@ module Searchkick
 end
 
 # TODO find better ActiveModel hook
-ActiveModel::Callbacks.send(:include, Searchkick::Model)
+ActiveModel::Callbacks.include(Searchkick::Model)
 
 ActiveSupport.on_load(:active_record) do
   extend Searchkick::Model

data/lib/searchkick/bulk_indexer.rb

@@ -0,0 +1,168 @@
+module Searchkick
+  class BulkIndexer
+    attr_reader :index
+
+    def initialize(index)
+      @index = index
+    end
+
+    def import_scope(scope, resume: false, method_name: nil, async: false, batch: false, batch_id: nil, full: false)
+      # use scope for import
+      scope = scope.search_import if scope.respond_to?(:search_import)
+
+      if batch
+        import_or_update scope.to_a, method_name, async
+        Searchkick.with_redis { |r| r.srem(batches_key, batch_id) } if batch_id
+      elsif full && async
+        full_reindex_async(scope)
+      elsif scope.respond_to?(:find_in_batches)
+        if resume
+          # use total docs instead of max id since there's not a great way
+          # to get the max _id without scripting since it's a string
+
+          # TODO use primary key and prefix with table name
+          scope = scope.where("id > ?", total_docs)
+        end
+
+        scope = scope.select("id").except(:includes, :preload) if async
+
+        scope.find_in_batches batch_size: batch_size do |items|
+          import_or_update items, method_name, async
+        end
+      else
+        each_batch(scope) do |items|
+          import_or_update items, method_name, async
+        end
+      end
+    end
+
+    def bulk_index(records)
+      Searchkick.indexer.queue(records.map { |r| RecordData.new(index, r).index_data })
+    end
+
+    def bulk_delete(records)
+      Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(index, r).delete_data })
+    end
+
+    def bulk_update(records, method_name)
+      Searchkick.indexer.queue(records.map { |r| RecordData.new(index, r).update_data(method_name) })
+    end
+
+    private
+
+    def import_or_update(records, method_name, async)
+      if records.any?
+        if async
+          Searchkick::BulkReindexJob.perform_later(
+            class_name: records.first.class.name,
+            record_ids: records.map(&:id),
+            index_name: index.name,
+            method_name: method_name ? method_name.to_s : nil
+          )
+        else
+          records = records.select(&:should_index?)
+          if records.any?
+            with_retries do
+              # call out to index for ActiveSupport notifications
+              if method_name
+                index.bulk_update(records, method_name)
+              else
+                index.bulk_index(records)
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def full_reindex_async(scope)
+      if scope.respond_to?(:primary_key)
+        # TODO expire Redis key
+        primary_key = scope.primary_key
+
+        starting_id =
+          begin
+            scope.minimum(primary_key)
+          rescue ActiveRecord::StatementInvalid
+            false
+          end
+
+        if starting_id.nil?
+          # no records, do nothing
+        elsif starting_id.is_a?(Numeric)
+          max_id = scope.maximum(primary_key)
+          batches_count = ((max_id - starting_id + 1) / batch_size.to_f).ceil
+
+          batches_count.times do |i|
+            batch_id = i + 1
+            min_id = starting_id + (i * batch_size)
+            bulk_reindex_job scope, batch_id, min_id: min_id, max_id: min_id + batch_size - 1
+          end
+        else
+          scope.find_in_batches(batch_size: batch_size).each_with_index do |batch, i|
+            batch_id = i + 1
+
+            bulk_reindex_job scope, batch_id, record_ids: batch.map { |record| record.id.to_s }
+          end
+        end
+      else
+        batch_id = 1
+        # TODO remove any eager loading
+        scope = scope.only(:_id) if scope.respond_to?(:only)
+        each_batch(scope) do |items|
+          bulk_reindex_job scope, batch_id, record_ids: items.map { |i| i.id.to_s }
+          batch_id += 1
+        end
+      end
+    end
+
+    def each_batch(scope)
+      # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
+      # use cursor for Mongoid
+      items = []
+      scope.all.each do |item|
+        items << item
+        if items.length == batch_size
+          yield items
+          items = []
+        end
+      end
+      yield items if items.any?
+    end
+
+    def bulk_reindex_job(scope, batch_id, options)
+      Searchkick::BulkReindexJob.perform_later({
+        class_name: scope.model_name.name,
+        index_name: index.name,
+        batch_id: batch_id
+      }.merge(options))
+      Searchkick.with_redis { |r| r.sadd(batches_key, batch_id) }
+    end
+
+    def with_retries
+      retries = 0
+
+      begin
+        yield
+      rescue Faraday::ClientError => e
+        if retries < 1
+          retries += 1
+          retry
+        end
+        raise e
+      end
+    end
+
+    def batches_left
+      Searchkick.with_redis { |r| r.scard(batches_key) }
+    end
+
+    def batches_key
+      "searchkick:reindex:#{index.name}:batches"
+    end
+
+    def batch_size
+      @batch_size ||= index.options[:batch_size] || 1000
+    end
+  end
+end