searchkick 2.3.2 → 4.4.1

data/benchmark/Gemfile

@@ -1,23 +0,0 @@
-source "https://rubygems.org"
-
-# Specify your gem's dependencies in searchkick.gemspec
-gemspec path: "../"
-
-# gem "sqlite3"
-gem "pg"
-gem "activerecord", "~> 5.0.0"
-gem "activerecord-import"
-gem "activejob"
-gem "redis"
-gem "sidekiq"
-
-# performance
-gem "typhoeus"
-gem "oj"
-
-# profiling
-gem "ruby-prof"
-gem "allocation_stats"
-gem "get_process_mem"
-gem "memory_profiler"
-gem "allocation_tracer"

data/benchmark/benchmark.rb

@@ -1,97 +0,0 @@
-require "bundler/setup"
-Bundler.require(:default)
-require "active_record"
-require "benchmark"
-require "active_support/notifications"
-
-ActiveSupport::Notifications.subscribe "request.searchkick" do |*args|
-  event = ActiveSupport::Notifications::Event.new(*args)
-  p event.duration
-end
-
-ActiveJob::Base.queue_adapter = :sidekiq
-
-Searchkick.redis = Redis.new
-
-ActiveRecord::Base.default_timezone = :utc
-ActiveRecord::Base.time_zone_aware_attributes = true
-# ActiveRecord::Base.establish_connection adapter: "sqlite3", database: "/tmp/searchkick"
-ActiveRecord::Base.establish_connection "postgresql://localhost/searchkick_demo_development"
-# ActiveRecord::Base.logger = Logger.new(STDOUT)
-
-ActiveJob::Base.logger = nil
-
-class Product < ActiveRecord::Base
-  searchkick batch_size: 1000
-
-  def search_data
-    {
-      name: name,
-      color: color,
-      store_id: store_id
-    }
-  end
-end
-
-total_docs = 100000
-
-# ActiveRecord::Migration.create_table :products, force: :cascade do |t|
-#   t.string :name
-#   t.string :color
-#   t.integer :store_id
-# end
-
-# Product.import ["name", "color", "store_id"], total_docs.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] }
-
-puts "Imported"
-
-result = nil
-report = nil
-stats = nil
-
-# p GetProcessMem.new.mb
-
-Product.searchkick_index.delete rescue nil
-
-time =
-  Benchmark.realtime do
-    # result = RubyProf.profile do
-    # report = MemoryProfiler.report do
-    # stats = AllocationStats.trace do
-    reindex = Product.reindex(async: true)
-    p reindex
-    # end
-
-    60.times do |i|
-      if reindex.is_a?(Hash)
-        docs = Searchkick::Index.new(reindex[:index_name]).total_docs
-      else
-        docs = Product.searchkick_index.total_docs
-      end
-      puts "#{i}: #{docs}"
-      if docs == total_docs
-        break
-      end
-      p Searchkick.reindex_status(reindex[:index_name]) if reindex.is_a?(Hash)
-      sleep(1)
-      # Product.searchkick_index.refresh
-    end
-  end
-
-# p GetProcessMem.new.mb
-
-puts time.round(1)
-
-
-if result
-  printer = RubyProf::GraphPrinter.new(result)
-  printer.print(STDOUT, min_percent: 5)
-end
-
-if report
-  puts report.pretty_print
-end
-
-if stats
-  puts result.allocations(alias_paths: true).group_by(:sourcefile, :class).to_text
-end

data/lib/searchkick/tasks.rb

@@ -1,33 +0,0 @@
-require "rake"
-
-namespace :searchkick do
-  desc "reindex model"
-  task reindex: :environment do
-    if ENV["CLASS"]
-      klass = ENV["CLASS"].constantize rescue nil
-      if klass
-        klass.reindex
-      else
-        abort "Could not find class: #{ENV['CLASS']}"
-      end
-    else
-      abort "USAGE: rake searchkick:reindex CLASS=Product"
-    end
-  end
-
-  if defined?(Rails)
-
-    namespace :reindex do
-      desc "reindex all models"
-      task all: :environment do
-        Rails.application.eager_load!
-        Searchkick.models.each do |model|
-          puts "Reindexing #{model.name}..."
-          model.reindex
-        end
-        puts "Reindex complete"
-      end
-    end
-
-  end
-end

data/searchkick.gemspec

@@ -1,28 +0,0 @@
-# coding: utf-8
-lib = File.expand_path("../lib", __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "searchkick/version"
-
-Gem::Specification.new do |spec|
-  spec.name          = "searchkick"
-  spec.version       = Searchkick::VERSION
-  spec.authors       = ["Andrew Kane"]
-  spec.email         = ["andrew@chartkick.com"]
-  spec.description   = "Intelligent search made easy"
-  spec.summary       = "Searchkick learns what your users are looking for. As more people search, it gets smarter and the results get better. It’s friendly for developers - and magical for your users."
-  spec.homepage      = "https://github.com/ankane/searchkick"
-  spec.license       = "MIT"
-
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features|benchmark)/})
-  spec.require_paths = ["lib"]
-
-  spec.add_dependency "activemodel", ">= 4.1"
-  spec.add_dependency "elasticsearch", ">= 1"
-  spec.add_dependency "hashie"
-
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "minitest"
-end

data/test/aggs_test.rb

@@ -1,197 +0,0 @@
-require_relative "test_helper"
-
-class AggsTest < Minitest::Test
-  def setup
-    super
-    store [
-      {name: "Product Show", latitude: 37.7833, longitude: 12.4167, store_id: 1, in_stock: true, color: "blue", price: 21, created_at: 2.days.ago},
-      {name: "Product Hide", latitude: 29.4167, longitude: -98.5000, store_id: 2, in_stock: false, color: "green", price: 25, created_at: 2.days.from_now},
-      {name: "Product B", latitude: 43.9333, longitude: -122.4667, store_id: 2, in_stock: false, color: "red", price: 5, created_at: Time.now},
-      {name: "Foo", latitude: 43.9333, longitude: 12.4667, store_id: 3, in_stock: false, color: "yellow", price: 15, created_at: Time.now}
-    ]
-  end
-
-  def test_basic
-    assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: [:store_id])
-  end
-
-  def test_where
-    assert_equal ({1 => 1}), store_agg(aggs: {store_id: {where: {in_stock: true}}})
-  end
-
-  def test_order
-    agg = Product.search("Product", aggs: {color: {order: {"_term" => "desc"}}}).aggs["color"]
-    assert_equal %w(red green blue), agg["buckets"].map { |b| b["key"] }
-  end
-
-  def test_field
-    assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: {store_id: {}})
-    assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: {store_id: {field: "store_id"}})
-    assert_equal ({1 => 1, 2 => 2}), store_agg({aggs: {store_id_new: {field: "store_id"}}}, "store_id_new")
-  end
-
-  def test_min_doc_count
-    assert_equal ({2 => 2}), store_agg(aggs: {store_id: {min_doc_count: 2}})
-  end
-
-  def test_no_aggs
-    assert_nil Product.search("*").aggs
-  end
-
-  def test_limit
-    agg = Product.search("Product", aggs: {store_id: {limit: 1}}).aggs["store_id"]
-    assert_equal 1, agg["buckets"].size
-    # assert_equal 3, agg["doc_count"]
-    assert_equal(1, agg["sum_other_doc_count"])
-  end
-
-  def test_ranges
-    price_ranges = [{to: 10}, {from: 10, to: 20}, {from: 20}]
-    agg = Product.search("Product", aggs: {price: {ranges: price_ranges}}).aggs["price"]
-
-    assert_equal 3, agg["buckets"].size
-    assert_equal 10.0, agg["buckets"][0]["to"]
-    assert_equal 20.0, agg["buckets"][2]["from"]
-    assert_equal 1, agg["buckets"][0]["doc_count"]
-    assert_equal 0, agg["buckets"][1]["doc_count"]
-    assert_equal 2, agg["buckets"][2]["doc_count"]
-  end
-
-  def test_date_ranges
-    ranges = [{to: 1.day.ago}, {from: 1.day.ago, to: 1.day.from_now}, {from: 1.day.from_now}]
-    agg = Product.search("Product", aggs: {created_at: {date_ranges: ranges}}).aggs["created_at"]
-
-    assert_equal 1, agg["buckets"][0]["doc_count"]
-    assert_equal 1, agg["buckets"][1]["doc_count"]
-    assert_equal 1, agg["buckets"][2]["doc_count"]
-  end
-
-  def test_query_where
-    assert_equal ({1 => 1}), store_agg(where: {in_stock: true}, aggs: [:store_id])
-  end
-
-  def test_two_wheres
-    assert_equal ({2 => 1}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false}}})
-  end
-
-  def test_where_override
-    assert_equal ({}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false, color: "blue"}}})
-    assert_equal ({2 => 1}), store_agg(where: {color: "blue"}, aggs: {store_id: {where: {in_stock: false, color: "red"}}})
-  end
-
-  def test_skip
-    assert_equal ({1 => 1, 2 => 2}), store_agg(where: {store_id: 2}, aggs: [:store_id])
-  end
-
-  def test_skip_complex
-    assert_equal ({1 => 1, 2 => 1}), store_agg(where: {store_id: 2, price: {gt: 5}}, aggs: [:store_id])
-  end
-
-  def test_multiple
-    assert_equal ({"store_id" => {1 => 1, 2 => 2}, "color" => {"blue" => 1, "green" => 1, "red" => 1}}), store_multiple_aggs(aggs: [:store_id, :color])
-  end
-
-  def test_smart_aggs_false
-    assert_equal ({2 => 2}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false}}}, smart_aggs: false)
-    assert_equal ({2 => 2}), store_agg(where: {color: "blue"}, aggs: {store_id: {where: {in_stock: false}}}, smart_aggs: false)
-  end
-
-  def test_aggs_group_by_date
-    store [{name: "Old Product", created_at: 3.years.ago}]
-    products =
-      Product.search("Product", {
-        where: {
-          created_at: {lt: Time.now}
-        },
-        aggs: {
-          products_per_year: {
-            date_histogram: {
-              field: :created_at,
-              interval: :year
-            }
-          }
-        }
-      })
-
-    assert_equal 4, products.aggs["products_per_year"]["buckets"].size
-  end
-
-  def test_aggs_avg
-    products =
-      Product.search("*", {
-        aggs: {
-          avg_price: {
-            avg: {
-              field: :price
-            }
-          }
-        }
-      })
-    assert_equal 16.5, products.aggs["avg_price"]["value"]
-  end
-
-  def test_aggs_cardinality
-    products =
-      Product.search("*", {
-        aggs: {
-          total_stores: {
-            cardinality: {
-              field: :store_id
-            }
-          }
-        }
-      })
-    assert_equal 3, products.aggs["total_stores"]["value"]
-  end
-
-  def test_aggs_min_max
-    products =
-      Product.search("*", {
-        aggs: {
-          min_price: {
-            min: {
-              field: :price
-            }
-          },
-          max_price: {
-            max: {
-              field: :price
-            }
-          }
-        }
-      })
-    assert_equal 5, products.aggs["min_price"]["value"]
-    assert_equal 25, products.aggs["max_price"]["value"]
-  end
-
-  def test_aggs_sum
-    products =
-      Product.search("*", {
-        aggs: {
-          sum_price: {
-            sum: {
-              field: :price
-            }
-          }
-        }
-      })
-    assert_equal 66, products.aggs["sum_price"]["value"]
-  end
-
-  protected
-
-  def buckets_as_hash(agg)
-    Hash[agg["buckets"].map { |v| [v["key"], v["doc_count"]] }]
-  end
-
-  def store_agg(options, agg_key = "store_id")
-    buckets = Product.search("Product", options).aggs[agg_key]
-    buckets_as_hash(buckets)
-  end
-
-  def store_multiple_aggs(options)
-    Hash[Product.search("Product", options).aggs.map do |field, filtered_agg|
-      [field, buckets_as_hash(filtered_agg)]
-    end]
-  end
-end

data/test/autocomplete_test.rb

@@ -1,75 +0,0 @@
-require_relative "test_helper"
-
-class AutocompleteTest < Minitest::Test
-  def test_autocomplete
-    store_names ["Hummus"]
-    assert_search "hum", ["Hummus"], match: :text_start
-  end
-
-  def test_autocomplete_two_words
-    store_names ["Organic Hummus"]
-    assert_search "hum", [], match: :text_start
-  end
-
-  def test_autocomplete_fields
-    store_names ["Hummus"]
-    assert_search "hum", ["Hummus"], match: :text_start, fields: [:name]
-  end
-
-  def test_text_start
-    store_names ["Where in the World is Carmen San Diego"]
-    assert_search "where in the world is", ["Where in the World is Carmen San Diego"], fields: [{name: :text_start}]
-    assert_search "in the world", [], fields: [{name: :text_start}]
-  end
-
-  def test_text_middle
-    store_names ["Where in the World is Carmen San Diego"]
-    assert_search "where in the world is", ["Where in the World is Carmen San Diego"], fields: [{name: :text_middle}]
-    assert_search "n the wor", ["Where in the World is Carmen San Diego"], fields: [{name: :text_middle}]
-    assert_search "men san diego", ["Where in the World is Carmen San Diego"], fields: [{name: :text_middle}]
-    assert_search "world carmen", [], fields: [{name: :text_middle}]
-  end
-
-  def test_text_end
-    store_names ["Where in the World is Carmen San Diego"]
-    assert_search "men san diego", ["Where in the World is Carmen San Diego"], fields: [{name: :text_end}]
-    assert_search "carmen san", [], fields: [{name: :text_end}]
-  end
-
-  def test_word_start
-    store_names ["Where in the World is Carmen San Diego"]
-    assert_search "car san wor", ["Where in the World is Carmen San Diego"], fields: [{name: :word_start}]
-  end
-
-  def test_word_middle
-    store_names ["Where in the World is Carmen San Diego"]
-    assert_search "orl", ["Where in the World is Carmen San Diego"], fields: [{name: :word_middle}]
-  end
-
-  def test_word_end
-    store_names ["Where in the World is Carmen San Diego"]
-    assert_search "rld men ego", ["Where in the World is Carmen San Diego"], fields: [{name: :word_end}]
-  end
-
-  def test_word_start_multiple_words
-    store_names ["Dark Grey", "Dark Blue"]
-    assert_search "dark grey", ["Dark Grey"], fields: [{name: :word_start}]
-  end
-
-  def test_word_start_exact
-    store_names ["Back Scratcher", "Backpack"]
-    assert_order "back", ["Back Scratcher", "Backpack"], fields: [{name: :word_start}]
-  end
-
-  def test_word_start_exact_martin
-    store_names ["Martina", "Martin"]
-    assert_order "martin", ["Martin", "Martina"], fields: [{name: :word_start}]
-  end
-
-  # TODO find a better place
-
-  def test_exact
-    store_names ["hi@example.org"]
-    assert_search "hi@example.org", ["hi@example.org"], fields: [{name: :exact}]
-  end
-end

data/test/boost_test.rb

@@ -1,202 +0,0 @@
-require_relative "test_helper"
-
-class BoostTest < Minitest::Test
-  # conversions
-
-  def test_conversions
-    store [
-      {name: "Tomato A", conversions: {"tomato" => 1}},
-      {name: "Tomato B", conversions: {"tomato" => 2}},
-      {name: "Tomato C", conversions: {"tomato" => 3}}
-    ]
-    assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"]
-    assert_equal_scores "tomato", conversions: false
-  end
-
-  def test_multiple_conversions
-    store [
-      {name: "Speaker A", conversions_a: {"speaker" => 1}, conversions_b: {"speaker" => 6}},
-      {name: "Speaker B", conversions_a: {"speaker" => 2}, conversions_b: {"speaker" => 5}},
-      {name: "Speaker C", conversions_a: {"speaker" => 3}, conversions_b: {"speaker" => 4}}
-    ], Speaker
-
-    assert_equal_scores "speaker", {conversions: false}, Speaker
-    assert_equal_scores "speaker", {}, Speaker
-    assert_equal_scores "speaker", {conversions: ["conversions_a", "conversions_b"]}, Speaker
-    assert_equal_scores "speaker", {conversions: ["conversions_b", "conversions_a"]}, Speaker
-    assert_order "speaker", ["Speaker C", "Speaker B", "Speaker A"], {conversions: "conversions_a"}, Speaker
-    assert_order "speaker", ["Speaker A", "Speaker B", "Speaker C"], {conversions: "conversions_b"}, Speaker
-  end
-
-  def test_multiple_conversions_with_boost_term
-    store [
-      {name: "Speaker A", conversions_a: {"speaker" => 4, "speaker_1" => 1}},
-      {name: "Speaker B", conversions_a: {"speaker" => 3, "speaker_1" => 2}},
-      {name: "Speaker C", conversions_a: {"speaker" => 2, "speaker_1" => 3}},
-      {name: "Speaker D", conversions_a: {"speaker" => 1, "speaker_1" => 4}}
-    ], Speaker
-
-    assert_order "speaker", ["Speaker A", "Speaker B", "Speaker C", "Speaker D"], {conversions: "conversions_a"}, Speaker
-    assert_order "speaker", ["Speaker D", "Speaker C", "Speaker B", "Speaker A"], {conversions: "conversions_a", conversions_term: "speaker_1"}, Speaker
-  end
-
-  def test_conversions_stemmed
-    store [
-      {name: "Tomato A", conversions: {"tomato" => 1, "tomatos" => 1, "Tomatoes" => 1}},
-      {name: "Tomato B", conversions: {"tomato" => 2}}
-    ]
-    assert_order "tomato", ["Tomato A", "Tomato B"]
-  end
-
-  # global boost
-
-  def test_boost
-    store [
-      {name: "Tomato A"},
-      {name: "Tomato B", orders_count: 10},
-      {name: "Tomato C", orders_count: 100}
-    ]
-    assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"], boost: "orders_count"
-  end
-
-  def test_boost_zero
-    store [
-      {name: "Zero Boost", orders_count: 0}
-    ]
-    assert_order "zero", ["Zero Boost"], boost: "orders_count"
-  end
-
-  def test_conversions_weight
-    store [
-      {name: "Product Boost", orders_count: 20},
-      {name: "Product Conversions", conversions: {"product" => 10}}
-    ]
-    assert_order "product", ["Product Conversions", "Product Boost"], boost: "orders_count"
-  end
-
-  def test_boost_fields
-    store [
-      {name: "Red", color: "White"},
-      {name: "White", color: "Red Red Red"}
-    ]
-    assert_order "red", ["Red", "White"], fields: ["name^10", "color"]
-  end
-
-  def test_boost_fields_decimal
-    store [
-      {name: "Red", color: "White"},
-      {name: "White", color: "Red Red Red"}
-    ]
-    assert_order "red", ["Red", "White"], fields: ["name^10.5", "color"]
-  end
-
-  def test_boost_fields_word_start
-    store [
-      {name: "Red", color: "White"},
-      {name: "White", color: "Red Red Red"}
-    ]
-    assert_order "red", ["Red", "White"], fields: [{"name^10" => :word_start}, "color"]
-  end
-
-  # for issue #855
-  def test_apostrophes
-    store_names ["Valentine's Day Special"]
-    assert_search "Valentines", ["Valentine's Day Special"], fields: ["name^5"]
-    assert_search "Valentine's", ["Valentine's Day Special"], fields: ["name^5"]
-    assert_search "Valentine", ["Valentine's Day Special"], fields: ["name^5"]
-  end
-
-  def test_boost_by
-    store [
-      {name: "Tomato A"},
-      {name: "Tomato B", orders_count: 10},
-      {name: "Tomato C", orders_count: 100}
-    ]
-    assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"], boost_by: [:orders_count]
-    assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"], boost_by: {orders_count: {factor: 10}}
-  end
-
-  def test_boost_by_missing
-    store [
-      {name: "Tomato A"},
-      {name: "Tomato B", orders_count: 10},
-    ]
-
-    if elasticsearch_below50?
-      assert_raises(ArgumentError) do
-        assert_order "tomato", ["Tomato A", "Tomato B"], boost_by: {orders_count: {missing: 100}}
-      end
-    else
-      assert_order "tomato", ["Tomato A", "Tomato B"], boost_by: {orders_count: {missing: 100}}
-    end
-  end
-
-  def test_boost_by_boost_mode_multiply
-    store [
-      {name: "Tomato A", found_rate: 0.9},
-      {name: "Tomato B"},
-      {name: "Tomato C", found_rate: 0.5}
-    ]
-
-    assert_order "tomato", ["Tomato B", "Tomato A", "Tomato C"], boost_by: {found_rate: {boost_mode: "multiply"}}
-  end
-
-  def test_boost_where
-    store [
-      {name: "Tomato A"},
-      {name: "Tomato B", user_ids: [1, 2]},
-      {name: "Tomato C", user_ids: [3]}
-    ]
-    assert_first "tomato", "Tomato B", boost_where: {user_ids: 2}
-    assert_first "tomato", "Tomato B", boost_where: {user_ids: 1..2}
-    assert_first "tomato", "Tomato B", boost_where: {user_ids: [1, 4]}
-    assert_first "tomato", "Tomato B", boost_where: {user_ids: {value: 2, factor: 10}}
-    assert_first "tomato", "Tomato B", boost_where: {user_ids: {value: [1, 4], factor: 10}}
-    assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"], boost_where: {user_ids: [{value: 1, factor: 10}, {value: 3, factor: 20}]}
-  end
-
-  def test_boost_by_distance
-    store [
-      {name: "San Francisco", latitude: 37.7833, longitude: -122.4167},
-      {name: "San Antonio", latitude: 29.4167, longitude: -98.5000},
-      {name: "San Marino", latitude: 43.9333, longitude: 12.4667}
-    ]
-    assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {field: :location, origin: [37, -122], scale: "1000mi"}
-  end
-
-  def test_boost_by_distance_hash
-    store [
-      {name: "San Francisco", latitude: 37.7833, longitude: -122.4167},
-      {name: "San Antonio", latitude: 29.4167, longitude: -98.5000},
-      {name: "San Marino", latitude: 43.9333, longitude: 12.4667}
-    ]
-    assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {field: :location, origin: {lat: 37, lon: -122}, scale: "1000mi"}
-  end
-
-  def test_boost_by_distance_v2
-    store [
-      {name: "San Francisco", latitude: 37.7833, longitude: -122.4167},
-      {name: "San Antonio", latitude: 29.4167, longitude: -98.5000},
-      {name: "San Marino", latitude: 43.9333, longitude: 12.4667}
-    ]
-    assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {location: {origin: [37, -122], scale: "1000mi"}}
-  end
-
-  def test_boost_by_distance_v2_hash
-    store [
-      {name: "San Francisco", latitude: 37.7833, longitude: -122.4167},
-      {name: "San Antonio", latitude: 29.4167, longitude: -98.5000},
-      {name: "San Marino", latitude: 43.9333, longitude: 12.4667}
-    ]
-    assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {location: {origin: {lat: 37, lon: -122}, scale: "1000mi"}}
-  end
-
-  def test_boost_by_indices
-    skip if cequel?
-
-    store_names ["Rex"], Animal
-    store_names ["Rexx"], Product
-
-    assert_order "Rex", ["Rexx", "Rex"], {index_name: [Animal, Product], indices_boost: {Animal => 1, Product => 200}, fields: [:name]}, Store
-  end
-end