searchkick_evichat 0.0.2

data/Rakefile

@@ -0,0 +1,16 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+begin
+  require "parallel_tests/tasks"
+  require "shellwords"
+rescue LoadError
+  # do nothing
+end
+
+task default: :test
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+  t.warning = false
+end

data/benchmark/Gemfile

@@ -0,0 +1,24 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "sqlite3"
+gem "pg"
+gem "activerecord", "~> 5.1.0"
+gem "activerecord-import"
+gem "activejob"
+gem "redis"
+gem "sidekiq"
+
+# performance
+gem "typhoeus"
+gem "oj"
+
+# profiling
+gem "ruby-prof"
+gem "allocation_stats"
+gem "get_process_mem"
+gem "memory_profiler"
+gem "allocation_tracer"
+gem "benchmark-ips"

data/benchmark/index.rb

@@ -0,0 +1,99 @@
+require "bundler/setup"
+Bundler.require(:default)
+require "active_record"
+require "benchmark"
+require "active_support/notifications"
+
+ActiveSupport::Notifications.subscribe "request.searchkick" do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  puts "Import: #{event.duration.round}ms"
+end
+
+ActiveJob::Base.queue_adapter = :sidekiq
+
+Searchkick.redis = Redis.new
+
+ActiveRecord::Base.default_timezone = :utc
+ActiveRecord::Base.time_zone_aware_attributes = true
+# ActiveRecord::Base.establish_connection adapter: "sqlite3", database: "/tmp/searchkick"
+ActiveRecord::Base.establish_connection "postgresql://localhost/searchkick_demo_development"
+# ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+ActiveJob::Base.logger = nil
+
+class Product < ActiveRecord::Base
+  searchkick batch_size: 1000
+
+  def search_data
+    {
+      name: name,
+      color: color,
+      store_id: store_id
+    }
+  end
+end
+
+if ENV["SETUP"]
+  total_docs = 100000
+
+  ActiveRecord::Migration.create_table :products, force: :cascade do |t|
+    t.string :name
+    t.string :color
+    t.integer :store_id
+  end
+
+  Product.import ["name", "color", "store_id"], total_docs.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] }
+
+  puts "Imported"
+end
+
+result = nil
+report = nil
+stats = nil
+
+Product.searchkick_index.delete rescue nil
+
+GC.start
+GC.disable
+start_mem = GetProcessMem.new.mb
+
+time =
+  Benchmark.realtime do
+    # result = RubyProf.profile do
+    # report = MemoryProfiler.report do
+    # stats = AllocationStats.trace do
+    reindex = Product.reindex #(async: true)
+    # p reindex
+    # end
+
+    # 60.times do |i|
+    #   if reindex.is_a?(Hash)
+    #     docs = Searchkick::Index.new(reindex[:index_name]).total_docs
+    #   else
+    #     docs = Product.searchkick_index.total_docs
+    #   end
+    #   puts "#{i}: #{docs}"
+    #   if docs == total_docs
+    #     break
+    #   end
+    #   p Searchkick.reindex_status(reindex[:index_name]) if reindex.is_a?(Hash)
+    #   sleep(1)
+    #   # Product.searchkick_index.refresh
+    # end
+  end
+
+puts
+puts "Time: #{time.round(1)}s"
+
+if result
+  printer = RubyProf::GraphPrinter.new(result)
+  printer.print(STDOUT, min_percent: 5)
+end
+
+if report
+  puts report.pretty_print
+end
+
+if stats
+  puts result.allocations(alias_paths: true).group_by(:sourcefile, :class).to_text
+end

data/benchmark/search.rb

@@ -0,0 +1,48 @@
+require "bundler/setup"
+Bundler.require(:default)
+require "active_record"
+require "benchmark/ips"
+
+ActiveRecord::Base.default_timezone = :utc
+ActiveRecord::Base.time_zone_aware_attributes = true
+ActiveRecord::Base.establish_connection adapter: "sqlite3", database: "/tmp/searchkick"
+
+class Product < ActiveRecord::Base
+  searchkick batch_size: 1000
+
+  def search_data
+    {
+      name: name,
+      color: color,
+      store_id: store_id
+    }
+  end
+end
+
+if ENV["SETUP"]
+  total_docs = 1000000
+
+  ActiveRecord::Migration.create_table :products, force: :cascade do |t|
+    t.string :name
+    t.string :color
+    t.integer :store_id
+  end
+
+  Product.import ["name", "color", "store_id"], total_docs.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] }
+
+  puts "Imported"
+
+  Product.reindex
+
+  puts "Reindexed"
+end
+
+query = Product.search("product", fields: [:name], where: {color: "red", store_id: 5}, limit: 10000, load: false, execute: false)
+
+require "pp"
+pp query.body.as_json
+puts
+
+Benchmark.ips do |x|
+  x.report { query.dup.execute }
+end

data/docs/Searchkick-3-Upgrade.md

@@ -0,0 +1,57 @@
+# Searchkick 3 Upgrade
+
+## Before You Upgrade
+
+Searchkick 3 no longer uses types, since they are deprecated in Elasticsearch 6.
+
+If you use inheritance, add to your parent model:
+
+```ruby
+class Animal < ApplicationRecord
+  searchkick inheritance: true
+end
+```
+
+And do a full reindex before upgrading.
+
+## Upgrading
+
+Update your Gemfile:
+
+```ruby
+gem 'searchkick', '~> 3'
+```
+
+And run:
+
+```sh
+bundle update searchkick
+```
+
+We recommend you don’t stem conversions anymore, so conversions for `pepper` don’t affect `peppers`, but if you want to keep the old behavior, use:
+
+```ruby
+Searchkick.model_options = {
+  stem_conversions: true
+}
+```
+
+Searchkick 3 disables the `_all` field by default, since Elasticsearch 6 removes the ability to reindex with it. If you’re on Elasticsearch 5 and still need it, add to your model:
+
+```ruby
+class Product < ApplicationRecord
+  searchkick _all: true
+end
+```
+
+If you use `record.reindex_async` or `record.reindex(async: true)`, replace it with:
+
+```ruby
+record.reindex(mode: :async)
+```
+
+If you use `log: true` with `boost_by`, replace it with `modifier: "ln2p"`.
+
+If you use the `body` option and have warnings about incompatible options, remove them, as they now throw an `ArgumentError`.
+
+Check out the [changelog](https://github.com/ankane/searchkick/blob/master/CHANGELOG.md) for the full list of changes.

data/lib/searchkick.rb

@@ -0,0 +1,255 @@
+require "active_model"
+require "active_support/core_ext/hash/deep_merge"
+require "elasticsearch"
+require "hashie"
+
+require "searchkick/bulk_indexer"
+require "searchkick/index"
+require "searchkick/indexer"
+require "searchkick/hash_wrapper"
+require "searchkick/middleware"
+require "searchkick/model"
+require "searchkick/multi_search"
+require "searchkick/query"
+require "searchkick/reindex_queue"
+require "searchkick/record_data"
+require "searchkick/record_indexer"
+require "searchkick/results"
+require "searchkick/version"
+
+require "searchkick/logging" if defined?(ActiveSupport::Notifications)
+
+begin
+  require "rake"
+rescue LoadError
+  # do nothing
+end
+require "searchkick/tasks" if defined?(Rake)
+
+begin
+  require "rake"
+rescue LoadError
+  # do nothing
+end
+require "searchkick/tasks" if defined?(Rake)
+
+# background jobs
+begin
+  require "active_job"
+rescue LoadError
+  # do nothing
+end
+if defined?(ActiveJob)
+  require "searchkick/bulk_reindex_job"
+  require "searchkick/process_batch_job"
+  require "searchkick/process_queue_job"
+  require "searchkick/reindex_v2_job"
+end
+
+module Searchkick
+  class Error < StandardError; end
+  class MissingIndexError < Error; end
+  class UnsupportedVersionError < Error; end
+  class InvalidQueryError < Elasticsearch::Transport::Transport::Errors::BadRequest; end
+  class DangerousOperation < Error; end
+  class ImportError < Error; end
+
+  class << self
+    attr_accessor :search_method_name, :wordnet_path, :timeout, :models, :client_options, :redis, :index_prefix, :index_suffix, :queue_name, :model_options
+    attr_writer :client, :env, :search_timeout
+    attr_reader :aws_credentials
+  end
+  self.search_method_name = :search
+  self.wordnet_path = "/var/lib/wn_s.pl"
+  self.timeout = 10
+  self.models = []
+  self.client_options = {}
+  self.queue_name = :searchkick
+  self.model_options = {}
+
+  def self.client
+    @client ||= begin
+      require "typhoeus/adapters/faraday" if defined?(Typhoeus)
+
+      Elasticsearch::Client.new({
+        url: ENV["ELASTICSEARCH_URL"],
+        transport_options: {request: {timeout: timeout}, headers: {content_type: "application/json"}},
+        retry_on_failure: 2
+      }.deep_merge(client_options)) do |f|
+        f.use Searchkick::Middleware
+        f.request signer_middleware_key, signer_middleware_aws_params if aws_credentials
+      end
+    end
+  end
+
+  def self.env
+    @env ||= ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+  end
+
+  def self.search_timeout
+    @search_timeout || timeout
+  end
+
+  def self.server_version
+    @server_version ||= client.info["version"]["number"]
+  end
+
+  def self.server_below?(version)
+    Gem::Version.new(server_version.split("-")[0]) < Gem::Version.new(version.split("-")[0])
+  end
+
+  def self.search(term = "*", model: nil, **options, &block)
+    options = options.dup
+    klass = model
+
+    # make Searchkick.search(index_name: [Product]) and Product.search equivalent
+    unless klass
+      index_name = Array(options[:index_name])
+      if index_name.size == 1 && index_name.first.respond_to?(:searchkick_index)
+        klass = index_name.first
+        options.delete(:index_name)
+      end
+    end
+
+    query = Searchkick::Query.new(klass, term, options)
+    block.call(query.body) if block
+    if options[:execute] == false
+      query
+    else
+      query.execute
+    end
+  end
+
+  def self.multi_search(queries)
+    Searchkick::MultiSearch.new(queries).perform
+  end
+
+  # callbacks
+
+  def self.enable_callbacks
+    self.callbacks_value = nil
+  end
+
+  def self.disable_callbacks
+    self.callbacks_value = false
+  end
+
+  def self.callbacks?(default: true)
+    if callbacks_value.nil?
+      default
+    else
+      callbacks_value != false
+    end
+  end
+
+  def self.callbacks(value)
+    if block_given?
+      previous_value = callbacks_value
+      begin
+        self.callbacks_value = value
+        result = yield
+        indexer.perform if callbacks_value == :bulk
+        result
+      ensure
+        self.callbacks_value = previous_value
+      end
+    else
+      self.callbacks_value = value
+    end
+  end
+
+  def self.aws_credentials=(creds)
+    begin
+      require "faraday_middleware/aws_signers_v4"
+    rescue LoadError
+      require "faraday_middleware/aws_sigv4"
+    end
+    @aws_credentials = creds
+    @client = nil # reset client
+  end
+
+  def self.reindex_status(index_name)
+    if redis
+      batches_left = Searchkick::Index.new(index_name).batches_left
+      {
+        completed: batches_left == 0,
+        batches_left: batches_left
+      }
+    else
+      raise Searchkick::Error, "Redis not configured"
+    end
+  end
+
+  def self.with_redis
+    if redis
+      if redis.respond_to?(:with)
+        redis.with do |r|
+          yield r
+        end
+      else
+        yield redis
+      end
+    end
+  end
+
+  # private
+  def self.load_records(records, ids)
+    records =
+      if records.respond_to?(:primary_key)
+        # ActiveRecord
+        records.where(records.primary_key => ids) if records.primary_key
+      elsif records.respond_to?(:queryable)
+        # Mongoid 3+
+        records.queryable.for_ids(ids)
+      elsif records.respond_to?(:unscoped) && :id.respond_to?(:in)
+        # Nobrainer
+        records.unscoped.where(:id.in => ids)
+      elsif records.respond_to?(:key_column_names)
+        records.where(records.key_column_names.first => ids)
+      end
+
+    raise Searchkick::Error, "Not sure how to load records" if !records
+
+    records
+  end
+
+  # private
+  def self.indexer
+    Thread.current[:searchkick_indexer] ||= Searchkick::Indexer.new
+  end
+
+  # private
+  def self.callbacks_value
+    Thread.current[:searchkick_callbacks_enabled]
+  end
+
+  # private
+  def self.callbacks_value=(value)
+    Thread.current[:searchkick_callbacks_enabled] = value
+  end
+
+  # private
+  def self.signer_middleware_key
+    defined?(FaradayMiddleware::AwsSignersV4) ? :aws_signers_v4 : :aws_sigv4
+  end
+
+  # private
+  def self.signer_middleware_aws_params
+    if signer_middleware_key == :aws_sigv4
+      {service: "es", region: "us-east-1"}.merge(aws_credentials)
+    else
+      {
+        credentials: aws_credentials[:credentials] || Aws::Credentials.new(aws_credentials[:access_key_id], aws_credentials[:secret_access_key]),
+        service_name: "es",
+        region: aws_credentials[:region] || "us-east-1"
+      }
+    end
+  end
+end
+
+# TODO find better ActiveModel hook
+ActiveModel::Callbacks.include(Searchkick::Model)
+
+ActiveSupport.on_load(:active_record) do
+  extend Searchkick::Model
+end