searchkick 2.5.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE.md +7 -0
  3. data/.travis.yml +2 -11
  4. data/CHANGELOG.md +22 -0
  5. data/CONTRIBUTING.md +1 -1
  6. data/Gemfile +3 -3
  7. data/LICENSE.txt +1 -1
  8. data/README.md +68 -141
  9. data/Rakefile +0 -4
  10. data/benchmark/Gemfile +3 -2
  11. data/benchmark/{benchmark.rb → index.rb} +33 -31
  12. data/benchmark/search.rb +48 -0
  13. data/docs/Searchkick-3-Upgrade.md +57 -0
  14. data/lib/searchkick.rb +50 -27
  15. data/lib/searchkick/bulk_indexer.rb +168 -0
  16. data/lib/searchkick/bulk_reindex_job.rb +1 -1
  17. data/lib/searchkick/index.rb +122 -348
  18. data/lib/searchkick/index_options.rb +29 -26
  19. data/lib/searchkick/logging.rb +8 -7
  20. data/lib/searchkick/model.rb +37 -90
  21. data/lib/searchkick/multi_search.rb +6 -7
  22. data/lib/searchkick/query.rb +169 -166
  23. data/lib/searchkick/record_data.rb +133 -0
  24. data/lib/searchkick/record_indexer.rb +55 -0
  25. data/lib/searchkick/reindex_queue.rb +1 -1
  26. data/lib/searchkick/reindex_v2_job.rb +10 -13
  27. data/lib/searchkick/results.rb +14 -25
  28. data/lib/searchkick/tasks.rb +0 -4
  29. data/lib/searchkick/version.rb +1 -1
  30. data/searchkick.gemspec +3 -3
  31. data/test/boost_test.rb +3 -9
  32. data/test/geo_shape_test.rb +0 -4
  33. data/test/highlight_test.rb +28 -12
  34. data/test/index_test.rb +9 -10
  35. data/test/language_test.rb +16 -0
  36. data/test/marshal_test.rb +6 -1
  37. data/test/match_test.rb +9 -4
  38. data/test/model_test.rb +3 -5
  39. data/test/multi_search_test.rb +0 -7
  40. data/test/order_test.rb +1 -7
  41. data/test/pagination_test.rb +1 -1
  42. data/test/reindex_v2_job_test.rb +6 -11
  43. data/test/routing_test.rb +1 -1
  44. data/test/similar_test.rb +2 -2
  45. data/test/sql_test.rb +0 -31
  46. data/test/test_helper.rb +37 -23
  47. metadata +19 -26
  48. data/test/gemfiles/activerecord31.gemfile +0 -7
  49. data/test/gemfiles/activerecord32.gemfile +0 -7
  50. data/test/gemfiles/activerecord40.gemfile +0 -8
  51. data/test/gemfiles/activerecord41.gemfile +0 -8
  52. data/test/gemfiles/mongoid2.gemfile +0 -7
  53. data/test/gemfiles/mongoid3.gemfile +0 -6
  54. data/test/gemfiles/mongoid4.gemfile +0 -7
  55. data/test/records_test.rb +0 -10
data/test/index_test.rb CHANGED
@@ -59,8 +59,10 @@ class IndexTest < Minitest::Test
59
59
  assert_equal ["Dollar Tree"], Store.search(body: {query: {match: {name: "Dollar Tree"}}}, load: false).map(&:name)
60
60
  end
61
61
 
62
- def test_body_warning
63
- assert_output(nil, "The body option replaces the entire body, so the following options are ignored: where\n") { Store.search(body: {query: {match: {name: "dollar"}}}, where: {id: 1}) }
62
+ def test_body_incompatible_options
63
+ assert_raises(ArgumentError) do
64
+ Store.search(body: {query: {match: {name: "dollar"}}}, where: {id: 1})
65
+ end
64
66
  end
65
67
 
66
68
  def test_block
@@ -131,9 +133,10 @@ class IndexTest < Minitest::Test
131
133
  def test_filterable
132
134
  # skip for 5.0 since it throws
133
135
  # Cannot search on field [alt_description] since it is not indexed.
134
- skip unless elasticsearch_below50?
135
136
  store [{name: "Product A", alt_description: "Hello"}]
136
- assert_search "*", [], where: {alt_description: "Hello"}
137
+ assert_raises(Searchkick::InvalidQueryError) do
138
+ assert_search "*", [], where: {alt_description: "Hello"}
139
+ end
137
140
  end
138
141
 
139
142
  def test_filterable_non_string
@@ -147,15 +150,11 @@ class IndexTest < Minitest::Test
147
150
  store [{name: "Product A", text: large_value}], Region
148
151
  assert_search "product", ["Product A"], {}, Region
149
152
  assert_search "hello", ["Product A"], {fields: [:name, :text]}, Region
150
-
151
- # needs fields for ES 6
152
- if elasticsearch_below60?
153
- assert_search "hello", ["Product A"], {}, Region
154
- end
153
+ assert_search "hello", ["Product A"], {}, Region
155
154
  end
156
155
 
157
156
  def test_very_large_value
158
- skip if nobrainer? || elasticsearch_below22?
157
+ skip if nobrainer?
159
158
  large_value = 10000.times.map { "hello" }.join(" ")
160
159
  store [{name: "Product A", text: large_value}], Region
161
160
  assert_search "product", ["Product A"], {}, Region
@@ -0,0 +1,16 @@
1
+ require_relative "test_helper"
2
+
3
+ class LanguageTest < Minitest::Test
4
+ def setup
5
+ Song.destroy_all
6
+ end
7
+
8
+ def test_chinese
9
+ skip unless ENV["CHINESE"]
10
+ Song.reindex
11
+ store_names ["中华人民共和国国歌"], Song
12
+ assert_search "中华人民共和国", ["中华人民共和国国歌"], {}, Song
13
+ assert_search "国歌", ["中华人民共和国国歌"], {}, Song
14
+ assert_search "人", [], {}, Song
15
+ end
16
+ end
data/test/marshal_test.rb CHANGED
@@ -3,6 +3,11 @@ require_relative "test_helper"
3
3
  class MarshalTest < Minitest::Test
4
4
  def test_marshal
5
5
  store_names ["Product A"]
6
- assert Marshal.dump(Product.search("*", load: {dumpable: true}).results)
6
+ assert Marshal.dump(Product.search("*").results)
7
+ end
8
+
9
+ def test_marshal_highlights
10
+ store_names ["Product A"]
11
+ assert Marshal.dump(Product.search("product", highlight: true, load: {dumpable: true}).results)
7
12
  end
8
13
  end
data/test/match_test.rb CHANGED
@@ -121,7 +121,11 @@ class MatchTest < Minitest::Test
121
121
  def test_misspelling_zucchini_transposition
122
122
  store_names ["zucchini"]
123
123
  assert_search "zuccihni", ["zucchini"]
124
- assert_search "zuccihni", [], misspellings: {transpositions: false}
124
+
125
+ # need to specify field
126
+ # as transposition option isn't supported for multi_match queries
127
+ # until Elasticsearch 6.1
128
+ assert_search "zuccihni", [], misspellings: {transpositions: false}, fields: [:name]
125
129
  end
126
130
 
127
131
  def test_misspelling_lasagna
@@ -180,12 +184,12 @@ class MatchTest < Minitest::Test
180
184
 
181
185
  def test_exclude_butter_exact
182
186
  store_names ["Butter Tub", "Peanut Butter Tub"]
183
- assert_search "butter", [], exclude: ["peanut butter"], match: :exact
187
+ assert_search "butter", [], exclude: ["peanut butter"], fields: [{name: :exact}]
184
188
  end
185
189
 
186
190
  def test_exclude_same_exact
187
191
  store_names ["Butter Tub", "Peanut Butter Tub"]
188
- assert_search "Butter Tub", [], exclude: ["Butter Tub"], match: :exact
192
+ assert_search "Butter Tub", ["Butter Tub"], exclude: ["Peanut Butter Tub"], fields: [{name: :exact}]
189
193
  end
190
194
 
191
195
  def test_exclude_egg_word_start
@@ -252,7 +256,7 @@ class MatchTest < Minitest::Test
252
256
 
253
257
  def test_phrase_order
254
258
  store_names ["Wheat Bread", "Whole Wheat Bread"]
255
- assert_order "wheat bread", ["Wheat Bread", "Whole Wheat Bread"], match: :phrase
259
+ assert_order "wheat bread", ["Wheat Bread", "Whole Wheat Bread"], match: :phrase, fields: [:name]
256
260
  end
257
261
 
258
262
  def test_dynamic_fields
@@ -261,6 +265,7 @@ class MatchTest < Minitest::Test
261
265
  end
262
266
 
263
267
  def test_unsearchable
268
+ skip
264
269
  store [
265
270
  {name: "Unsearchable", description: "Almond"}
266
271
  ]
data/test/model_test.rb CHANGED
@@ -4,13 +4,11 @@ class ModelTest < Minitest::Test
4
4
  def test_disable_callbacks_model
5
5
  store_names ["product a"]
6
6
 
7
- Product.disable_search_callbacks
8
- assert !Product.search_callbacks?
9
-
10
- store_names ["product b"]
7
+ Searchkick.callbacks(false) do
8
+ store_names ["product b"]
9
+ end
11
10
  assert_search "product", ["product a"]
12
11
 
13
- Product.enable_search_callbacks
14
12
  Product.reindex
15
13
 
16
14
  assert_search "product", ["product a", "product b"]
@@ -24,13 +24,6 @@ class MultiSearchTest < Minitest::Test
24
24
  store_names ["abc", "abd", "aee"]
25
25
  products = Product.search("abc", misspellings: {below: 2}, execute: false)
26
26
  Searchkick.multi_search([products])
27
- assert_equal ["abc"], products.map(&:name)
28
- end
29
-
30
- def test_misspellings_below_unmet_retry
31
- store_names ["abc", "abd", "aee"]
32
- products = Product.search("abc", misspellings: {below: 2}, execute: false)
33
- Searchkick.multi_search([products], retry_misspellings: true)
34
27
  assert_equal ["abc", "abd"], products.map(&:name)
35
28
  end
36
29
 
data/test/order_test.rb CHANGED
@@ -29,14 +29,8 @@ class OrderTest < Minitest::Test
29
29
  assert_order "product", ["Product A", "Product B", "Product C"], order: {color: :asc, store_id: :desc}
30
30
  end
31
31
 
32
- def test_order_ignore_unmapped
33
- skip unless elasticsearch_below50?
34
- assert_order "product", [], order: {not_mapped: {ignore_unmapped: true}}, conversions: false
35
- end
36
-
37
32
  def test_order_unmapped_type
38
- skip if elasticsearch_below50?
39
- assert_order "product", [], order: {not_mapped: {unmapped_type: "long"}}, conversions: false
33
+ assert_order "product", [], order: {not_mapped: {unmapped_type: "long"}}
40
34
  end
41
35
 
42
36
  def test_order_array
@@ -14,7 +14,7 @@ class PaginationTest < Minitest::Test
14
14
 
15
15
  def test_offset
16
16
  store_names ["Product A", "Product B", "Product C", "Product D"]
17
- assert_order "product", ["Product C", "Product D"], order: {name: :asc}, offset: 2
17
+ assert_order "product", ["Product C", "Product D"], order: {name: :asc}, offset: 2, limit: 100
18
18
  end
19
19
 
20
20
  def test_pagination
@@ -4,29 +4,24 @@ class ReindexV2JobTest < Minitest::Test
4
4
  def setup
5
5
  skip unless defined?(ActiveJob)
6
6
  super
7
- Searchkick.disable_callbacks
8
- end
9
-
10
- def teardown
11
- Searchkick.enable_callbacks
12
7
  end
13
8
 
14
9
  def test_create
15
- product = Product.create!(name: "Boom")
16
- Product.searchkick_index.refresh
10
+ product = Searchkick.callbacks(false) { Product.create!(name: "Boom") }
11
+ Product.search_index.refresh
17
12
  assert_search "*", []
18
13
  Searchkick::ReindexV2Job.perform_later("Product", product.id.to_s)
19
- Product.searchkick_index.refresh
14
+ Product.search_index.refresh
20
15
  assert_search "*", ["Boom"]
21
16
  end
22
17
 
23
18
  def test_destroy
24
- product = Product.create!(name: "Boom")
19
+ product = Searchkick.callbacks(false) { Product.create!(name: "Boom") }
25
20
  Product.reindex
26
21
  assert_search "*", ["Boom"]
27
- product.destroy
22
+ Searchkick.callbacks(false) { product.destroy }
28
23
  Searchkick::ReindexV2Job.perform_later("Product", product.id.to_s)
29
- Product.searchkick_index.refresh
24
+ Product.search_index.refresh
30
25
  assert_search "*", []
31
26
  end
32
27
  end
data/test/routing_test.rb CHANGED
@@ -8,7 +8,7 @@ class RoutingTest < Minitest::Test
8
8
 
9
9
  def test_routing_mappings
10
10
  index_options = Store.searchkick_index.index_options
11
- assert_equal index_options[:mappings][:_default_][:_routing], required: true
11
+ assert_equal index_options[:mappings]["store"][:_routing], required: true
12
12
  end
13
13
 
14
14
  def test_routing_correct_node
data/test/similar_test.rb CHANGED
@@ -3,7 +3,7 @@ require_relative "test_helper"
3
3
  class SimilarTest < Minitest::Test
4
4
  def test_similar
5
5
  store_names ["Annie's Naturals Organic Shiitake & Sesame Dressing"]
6
- assert_search "Annie's Naturals Shiitake & Sesame Vinaigrette", ["Annie's Naturals Organic Shiitake & Sesame Dressing"], similar: true
6
+ assert_search "Annie's Naturals Shiitake & Sesame Vinaigrette", ["Annie's Naturals Organic Shiitake & Sesame Dressing"], similar: true, fields: [:name]
7
7
  end
8
8
 
9
9
  def test_fields
@@ -13,7 +13,7 @@ class SimilarTest < Minitest::Test
13
13
 
14
14
  def test_order
15
15
  store_names ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"]
16
- assert_order "Lucerne Fat Free Chocolate Milk", ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"], similar: true
16
+ assert_order "Lucerne Fat Free Chocolate Milk", ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"], similar: true, fields: [:name]
17
17
  end
18
18
 
19
19
  def test_limit
data/test/sql_test.rb CHANGED
@@ -125,36 +125,7 @@ class SqlTest < Minitest::Test
125
125
  assert_nil hit["_source"]
126
126
  end
127
127
 
128
- def test_select_include
129
- skip unless elasticsearch_below50?
130
- store [{name: "Product A", user_ids: [1, 2]}]
131
- result = Product.search("product", load: false, select: {include: [:name]}).first
132
- assert_equal %w(id name), result.keys.reject { |k| k.start_with?("_") }.sort
133
- assert_equal "Product A", result.name
134
- assert_nil result.store_id
135
- end
136
-
137
- def test_select_exclude
138
- skip unless elasticsearch_below50?
139
- store [{name: "Product A", user_ids: [1, 2], store_id: 1}]
140
- result = Product.search("product", load: false, select: {exclude: [:name]}).first
141
- assert_nil result.name
142
- assert_equal [1, 2], result.user_ids
143
- assert_equal 1, result.store_id
144
- end
145
-
146
- def test_select_include_and_exclude
147
- skip unless elasticsearch_below50?
148
- # let's take this to the next level
149
- store [{name: "Product A", user_ids: [1, 2], store_id: 1}]
150
- result = Product.search("product", load: false, select: {include: [:store_id], exclude: [:name]}).first
151
- assert_equal 1, result.store_id
152
- assert_nil result.name
153
- assert_nil result.user_ids
154
- end
155
-
156
128
  def test_select_includes
157
- skip if elasticsearch_below50?
158
129
  store [{name: "Product A", user_ids: [1, 2]}]
159
130
  result = Product.search("product", load: false, select: {includes: [:name]}).first
160
131
  assert_equal %w(id name), result.keys.reject { |k| k.start_with?("_") }.sort
@@ -163,7 +134,6 @@ class SqlTest < Minitest::Test
163
134
  end
164
135
 
165
136
  def test_select_excludes
166
- skip if elasticsearch_below50?
167
137
  store [{name: "Product A", user_ids: [1, 2], store_id: 1}]
168
138
  result = Product.search("product", load: false, select: {excludes: [:name]}).first
169
139
  assert_nil result.name
@@ -172,7 +142,6 @@ class SqlTest < Minitest::Test
172
142
  end
173
143
 
174
144
  def test_select_include_and_excludes
175
- skip if elasticsearch_below50?
176
145
  # let's take this to the next level
177
146
  store [{name: "Product A", user_ids: [1, 2], store_id: 1}]
178
147
  result = Product.search("product", load: false, select: {includes: [:store_id], excludes: [:name]}).first
data/test/test_helper.rb CHANGED
@@ -38,18 +38,6 @@ end
38
38
 
39
39
  ActiveSupport::LogSubscriber.logger = ActiveSupport::Logger.new(STDOUT) if ENV["NOTIFICATIONS"]
40
40
 
41
- def elasticsearch_below50?
42
- Searchkick.server_below?("5.0.0-alpha1")
43
- end
44
-
45
- def elasticsearch_below60?
46
- Searchkick.server_below?("6.0.0-alpha1")
47
- end
48
-
49
- def elasticsearch_below22?
50
- Searchkick.server_below?("2.2.0")
51
- end
52
-
53
41
  def nobrainer?
54
42
  defined?(NoBrainer)
55
43
  end
@@ -121,6 +109,12 @@ if defined?(Mongoid)
121
109
 
122
110
  field :name
123
111
  end
112
+
113
+ class Song
114
+ include Mongoid::Document
115
+
116
+ field :name
117
+ end
124
118
  elsif defined?(NoBrainer)
125
119
  NoBrainer.configure do |config|
126
120
  config.app_name = :searchkick
@@ -132,7 +126,7 @@ elsif defined?(NoBrainer)
132
126
  include NoBrainer::Document::Timestamps
133
127
 
134
128
  field :id, type: Object
135
- field :name, type: String
129
+ field :name, type: Text
136
130
  field :in_stock, type: Boolean
137
131
  field :backordered, type: Boolean
138
132
  field :orders_count, type: Integer
@@ -188,6 +182,13 @@ elsif defined?(NoBrainer)
188
182
  field :id, type: String
189
183
  field :name, type: String
190
184
  end
185
+
186
+ class Song
187
+ include NoBrainer::Document
188
+
189
+ field :id, type: Object
190
+ field :name, type: String
191
+ end
191
192
  elsif defined?(Cequel)
192
193
  cequel =
193
194
  Cequel.connect(
@@ -276,7 +277,14 @@ elsif defined?(Cequel)
276
277
  column :name, :text
277
278
  end
278
279
 
279
- [Product, Store, Region, Speaker, Animal].each(&:synchronize_schema)
280
+ class Song
281
+ include Cequel::Record
282
+
283
+ key :id, :timeuuid, auto: true
284
+ column :name, :text
285
+ end
286
+
287
+ [Product, Store, Region, Speaker, Animal, Sku, Song].each(&:synchronize_schema)
280
288
  else
281
289
  require "active_record"
282
290
 
@@ -367,6 +375,10 @@ else
367
375
  t.string :name
368
376
  end
369
377
 
378
+ ActiveRecord::Migration.create_table :songs do |t|
379
+ t.string :name
380
+ end
381
+
370
382
  class Product < ActiveRecord::Base
371
383
  belongs_to :store
372
384
  end
@@ -392,6 +404,9 @@ else
392
404
 
393
405
  class Sku < ActiveRecord::Base
394
406
  end
407
+
408
+ class Song < ActiveRecord::Base
409
+ end
395
410
  end
396
411
 
397
412
  class Product
@@ -417,7 +432,6 @@ class Product
417
432
  word_middle: [:name],
418
433
  word_end: [:name],
419
434
  highlight: [:name],
420
- searchable: [:name, :color],
421
435
  filterable: [:name, :color, :description],
422
436
  similarity: "BM25",
423
437
  match: ENV["MATCH"] ? ENV["MATCH"].to_sym : nil
@@ -425,7 +439,7 @@ class Product
425
439
  attr_accessor :conversions, :user_ids, :aisle, :details
426
440
 
427
441
  def search_data
428
- serializable_hash.except("id").merge(
442
+ serializable_hash.except("id", "_id").merge(
429
443
  conversions: conversions,
430
444
  user_ids: user_ids,
431
445
  location: {lat: latitude, lon: longitude},
@@ -448,13 +462,12 @@ end
448
462
 
449
463
  class Store
450
464
  searchkick \
451
- default_fields: elasticsearch_below60? ? nil : [:name],
452
465
  routing: true,
453
466
  merge_mappings: true,
454
467
  mappings: {
455
468
  store: {
456
469
  properties: {
457
- name: elasticsearch_below50? ? {type: "string", analyzer: "keyword"} : {type: "keyword"}
470
+ name: {type: "keyword"}
458
471
  }
459
472
  }
460
473
  }
@@ -470,7 +483,6 @@ end
470
483
 
471
484
  class Region
472
485
  searchkick \
473
- default_fields: elasticsearch_below60? ? nil : [:name],
474
486
  geo_shape: {
475
487
  territory: {tree: "quadtree", precision: "10km"}
476
488
  }
@@ -488,13 +500,12 @@ end
488
500
 
489
501
  class Speaker
490
502
  searchkick \
491
- default_fields: elasticsearch_below60? ? nil : [:name],
492
503
  conversions: ["conversions_a", "conversions_b"]
493
504
 
494
505
  attr_accessor :conversions_a, :conversions_b, :aisle
495
506
 
496
507
  def search_data
497
- serializable_hash.except("id").merge(
508
+ serializable_hash.except("id", "_id").merge(
498
509
  conversions_a: conversions_a,
499
510
  conversions_b: conversions_b,
500
511
  aisle: aisle
@@ -504,8 +515,7 @@ end
504
515
 
505
516
  class Animal
506
517
  searchkick \
507
- default_fields: elasticsearch_below60? ? nil : [:name],
508
- inheritance: !elasticsearch_below60?,
518
+ inheritance: true,
509
519
  text_start: [:name],
510
520
  suggest: [:name],
511
521
  index_name: -> { "#{name.tableize}-#{Date.today.year}#{Searchkick.index_suffix}" },
@@ -517,6 +527,10 @@ class Sku
517
527
  searchkick callbacks: defined?(ActiveJob) ? :async : true
518
528
  end
519
529
 
530
+ class Song
531
+ searchkick language: "chinese"
532
+ end
533
+
520
534
  Product.searchkick_index.delete if Product.searchkick_index.exists?
521
535
  Product.reindex
522
536
  Product.reindex # run twice for both index paths