searchkick 2.3.2 → 5.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (87) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +377 -84
  3. data/LICENSE.txt +1 -1
  4. data/README.md +859 -602
  5. data/lib/searchkick/bulk_reindex_job.rb +13 -9
  6. data/lib/searchkick/controller_runtime.rb +40 -0
  7. data/lib/searchkick/hash_wrapper.rb +12 -0
  8. data/lib/searchkick/index.rb +281 -356
  9. data/lib/searchkick/index_cache.rb +30 -0
  10. data/lib/searchkick/index_options.rb +487 -281
  11. data/lib/searchkick/indexer.rb +15 -8
  12. data/lib/searchkick/log_subscriber.rb +57 -0
  13. data/lib/searchkick/middleware.rb +9 -2
  14. data/lib/searchkick/model.rb +72 -118
  15. data/lib/searchkick/multi_search.rb +9 -10
  16. data/lib/searchkick/process_batch_job.rb +12 -15
  17. data/lib/searchkick/process_queue_job.rb +22 -13
  18. data/lib/searchkick/query.rb +458 -217
  19. data/lib/searchkick/railtie.rb +7 -0
  20. data/lib/searchkick/record_data.rb +128 -0
  21. data/lib/searchkick/record_indexer.rb +164 -0
  22. data/lib/searchkick/reindex_queue.rb +51 -9
  23. data/lib/searchkick/reindex_v2_job.rb +10 -32
  24. data/lib/searchkick/relation.rb +247 -0
  25. data/lib/searchkick/relation_indexer.rb +155 -0
  26. data/lib/searchkick/results.rb +201 -82
  27. data/lib/searchkick/version.rb +1 -1
  28. data/lib/searchkick/where.rb +11 -0
  29. data/lib/searchkick.rb +269 -97
  30. data/lib/tasks/searchkick.rake +37 -0
  31. metadata +24 -178
  32. data/.gitignore +0 -22
  33. data/.travis.yml +0 -39
  34. data/Gemfile +0 -16
  35. data/Rakefile +0 -20
  36. data/benchmark/Gemfile +0 -23
  37. data/benchmark/benchmark.rb +0 -97
  38. data/lib/searchkick/logging.rb +0 -242
  39. data/lib/searchkick/tasks.rb +0 -33
  40. data/searchkick.gemspec +0 -28
  41. data/test/aggs_test.rb +0 -197
  42. data/test/autocomplete_test.rb +0 -75
  43. data/test/boost_test.rb +0 -202
  44. data/test/callbacks_test.rb +0 -59
  45. data/test/ci/before_install.sh +0 -17
  46. data/test/errors_test.rb +0 -19
  47. data/test/gemfiles/activerecord31.gemfile +0 -7
  48. data/test/gemfiles/activerecord32.gemfile +0 -7
  49. data/test/gemfiles/activerecord40.gemfile +0 -8
  50. data/test/gemfiles/activerecord41.gemfile +0 -8
  51. data/test/gemfiles/activerecord42.gemfile +0 -7
  52. data/test/gemfiles/activerecord50.gemfile +0 -7
  53. data/test/gemfiles/apartment.gemfile +0 -8
  54. data/test/gemfiles/cequel.gemfile +0 -8
  55. data/test/gemfiles/mongoid2.gemfile +0 -7
  56. data/test/gemfiles/mongoid3.gemfile +0 -6
  57. data/test/gemfiles/mongoid4.gemfile +0 -7
  58. data/test/gemfiles/mongoid5.gemfile +0 -7
  59. data/test/gemfiles/mongoid6.gemfile +0 -12
  60. data/test/gemfiles/nobrainer.gemfile +0 -8
  61. data/test/gemfiles/parallel_tests.gemfile +0 -8
  62. data/test/geo_shape_test.rb +0 -175
  63. data/test/highlight_test.rb +0 -78
  64. data/test/index_test.rb +0 -166
  65. data/test/inheritance_test.rb +0 -83
  66. data/test/marshal_test.rb +0 -8
  67. data/test/match_test.rb +0 -276
  68. data/test/misspellings_test.rb +0 -56
  69. data/test/model_test.rb +0 -42
  70. data/test/multi_search_test.rb +0 -36
  71. data/test/multi_tenancy_test.rb +0 -22
  72. data/test/order_test.rb +0 -46
  73. data/test/pagination_test.rb +0 -70
  74. data/test/partial_reindex_test.rb +0 -58
  75. data/test/query_test.rb +0 -35
  76. data/test/records_test.rb +0 -10
  77. data/test/reindex_test.rb +0 -64
  78. data/test/reindex_v2_job_test.rb +0 -32
  79. data/test/routing_test.rb +0 -23
  80. data/test/should_index_test.rb +0 -32
  81. data/test/similar_test.rb +0 -28
  82. data/test/sql_test.rb +0 -214
  83. data/test/suggest_test.rb +0 -95
  84. data/test/support/kaminari.yml +0 -21
  85. data/test/synonyms_test.rb +0 -67
  86. data/test/test_helper.rb +0 -567
  87. data/test/where_test.rb +0 -223
data/test/test_helper.rb DELETED
@@ -1,567 +0,0 @@
1
- require "bundler/setup"
2
- Bundler.require(:default)
3
- require "minitest/autorun"
4
- require "minitest/pride"
5
- require "logger"
6
- require "active_support/core_ext" if defined?(NoBrainer)
7
- require "active_support/notifications"
8
-
9
- Searchkick.index_suffix = ENV["TEST_ENV_NUMBER"]
10
-
11
- ENV["RACK_ENV"] = "test"
12
-
13
- Minitest::Test = Minitest::Unit::TestCase unless defined?(Minitest::Test)
14
-
15
- if !defined?(ParallelTests) || ParallelTests.first_process?
16
- File.delete("elasticsearch.log") if File.exist?("elasticsearch.log")
17
- end
18
-
19
- Searchkick.client.transport.logger = Logger.new("elasticsearch.log")
20
- Searchkick.search_timeout = 5
21
-
22
- if defined?(Redis)
23
- if defined?(ConnectionPool)
24
- Searchkick.redis = ConnectionPool.new { Redis.new }
25
- else
26
- Searchkick.redis = Redis.new
27
- end
28
- end
29
-
30
- puts "Running against Elasticsearch #{Searchkick.server_version}"
31
-
32
- I18n.config.enforce_available_locales = true
33
-
34
- if defined?(ActiveJob)
35
- ActiveJob::Base.logger = nil
36
- ActiveJob::Base.queue_adapter = :inline
37
- end
38
-
39
- ActiveSupport::LogSubscriber.logger = ActiveSupport::Logger.new(STDOUT) if ENV["NOTIFICATIONS"]
40
-
41
- def elasticsearch_below50?
42
- Searchkick.server_below?("5.0.0-alpha1")
43
- end
44
-
45
- def elasticsearch_below60?
46
- Searchkick.server_below?("6.0.0-alpha1")
47
- end
48
-
49
- def elasticsearch_below22?
50
- Searchkick.server_below?("2.2.0")
51
- end
52
-
53
- def nobrainer?
54
- defined?(NoBrainer)
55
- end
56
-
57
- def cequel?
58
- defined?(Cequel)
59
- end
60
-
61
- if defined?(Mongoid)
62
- Mongoid.logger.level = Logger::INFO
63
- Mongo::Logger.logger.level = Logger::INFO if defined?(Mongo::Logger)
64
-
65
- Mongoid.configure do |config|
66
- config.connect_to "searchkick_test"
67
- end
68
-
69
- class Product
70
- include Mongoid::Document
71
- include Mongoid::Timestamps
72
-
73
- field :name
74
- field :store_id, type: Integer
75
- field :in_stock, type: Boolean
76
- field :backordered, type: Boolean
77
- field :orders_count, type: Integer
78
- field :found_rate, type: BigDecimal
79
- field :price, type: Integer
80
- field :color
81
- field :latitude, type: BigDecimal
82
- field :longitude, type: BigDecimal
83
- field :description
84
- field :alt_description
85
- end
86
-
87
- class Store
88
- include Mongoid::Document
89
- has_many :products
90
-
91
- field :name
92
- end
93
-
94
- class Region
95
- include Mongoid::Document
96
-
97
- field :name
98
- field :text
99
- end
100
-
101
- class Speaker
102
- include Mongoid::Document
103
-
104
- field :name
105
- end
106
-
107
- class Animal
108
- include Mongoid::Document
109
-
110
- field :name
111
- end
112
-
113
- class Dog < Animal
114
- end
115
-
116
- class Cat < Animal
117
- end
118
-
119
- class Sku
120
- include Mongoid::Document
121
-
122
- field :name
123
- end
124
- elsif defined?(NoBrainer)
125
- NoBrainer.configure do |config|
126
- config.app_name = :searchkick
127
- config.environment = :test
128
- end
129
-
130
- class Product
131
- include NoBrainer::Document
132
- include NoBrainer::Document::Timestamps
133
-
134
- field :id, type: Object
135
- field :name, type: String
136
- field :in_stock, type: Boolean
137
- field :backordered, type: Boolean
138
- field :orders_count, type: Integer
139
- field :found_rate
140
- field :price, type: Integer
141
- field :color, type: String
142
- field :latitude
143
- field :longitude
144
- field :description, type: String
145
- field :alt_description, type: String
146
-
147
- belongs_to :store, validates: false
148
- end
149
-
150
- class Store
151
- include NoBrainer::Document
152
-
153
- field :id, type: Object
154
- field :name, type: String
155
- end
156
-
157
- class Region
158
- include NoBrainer::Document
159
-
160
- field :id, type: Object
161
- field :name, type: String
162
- field :text, type: Text
163
- end
164
-
165
- class Speaker
166
- include NoBrainer::Document
167
-
168
- field :id, type: Object
169
- field :name, type: String
170
- end
171
-
172
- class Animal
173
- include NoBrainer::Document
174
-
175
- field :id, type: Object
176
- field :name, type: String
177
- end
178
-
179
- class Dog < Animal
180
- end
181
-
182
- class Cat < Animal
183
- end
184
-
185
- class Sku
186
- include NoBrainer::Document
187
-
188
- field :id, type: String
189
- field :name, type: String
190
- end
191
- elsif defined?(Cequel)
192
- cequel =
193
- Cequel.connect(
194
- host: "127.0.0.1",
195
- port: 9042,
196
- keyspace: "searchkick_test",
197
- default_consistency: :all
198
- )
199
- # cequel.logger = ActiveSupport::Logger.new(STDOUT)
200
- cequel.schema.drop! if cequel.schema.exists?
201
- cequel.schema.create!
202
- Cequel::Record.connection = cequel
203
-
204
- class Product
205
- include Cequel::Record
206
-
207
- key :id, :uuid, auto: true
208
- column :name, :text, index: true
209
- column :store_id, :int
210
- column :in_stock, :boolean
211
- column :backordered, :boolean
212
- column :orders_count, :int
213
- column :found_rate, :decimal
214
- column :price, :int
215
- column :color, :text
216
- column :latitude, :decimal
217
- column :longitude, :decimal
218
- column :description, :text
219
- column :alt_description, :text
220
- column :created_at, :timestamp
221
- end
222
-
223
- class Store
224
- include Cequel::Record
225
-
226
- key :id, :timeuuid, auto: true
227
- column :name, :text
228
-
229
- # has issue with id serialization
230
- def search_data
231
- {
232
- name: name
233
- }
234
- end
235
- end
236
-
237
- class Region
238
- include Cequel::Record
239
-
240
- key :id, :timeuuid, auto: true
241
- column :name, :text
242
- column :text, :text
243
- end
244
-
245
- class Speaker
246
- include Cequel::Record
247
-
248
- key :id, :timeuuid, auto: true
249
- column :name, :text
250
- end
251
-
252
- class Animal
253
- include Cequel::Record
254
-
255
- key :id, :timeuuid, auto: true
256
- column :name, :text
257
-
258
- # has issue with id serialization
259
- def search_data
260
- {
261
- name: name
262
- }
263
- end
264
- end
265
-
266
- class Dog < Animal
267
- end
268
-
269
- class Cat < Animal
270
- end
271
-
272
- class Sku
273
- include Cequel::Record
274
-
275
- key :id, :uuid
276
- column :name, :text
277
- end
278
-
279
- [Product, Store, Region, Speaker, Animal].each(&:synchronize_schema)
280
- else
281
- require "active_record"
282
-
283
- # for debugging
284
- # ActiveRecord::Base.logger = Logger.new(STDOUT)
285
-
286
- # rails does this in activerecord/lib/active_record/railtie.rb
287
- ActiveRecord::Base.default_timezone = :utc
288
- ActiveRecord::Base.time_zone_aware_attributes = true
289
-
290
- # migrations
291
- ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
292
-
293
- ActiveRecord::Base.raise_in_transactional_callbacks = true if ActiveRecord::VERSION::STRING.start_with?("4.2.")
294
-
295
- if defined?(Apartment)
296
- class Rails
297
- def self.env
298
- ENV["RACK_ENV"]
299
- end
300
- end
301
-
302
- tenants = ["tenant1", "tenant2"]
303
- Apartment.configure do |config|
304
- config.tenant_names = tenants
305
- config.database_schema_file = false
306
- config.excluded_models = ["Product", "Store", "Animal", "Dog", "Cat"]
307
- end
308
-
309
- class Tenant < ActiveRecord::Base
310
- searchkick index_prefix: -> { Apartment::Tenant.current }
311
- end
312
-
313
- tenants.each do |tenant|
314
- begin
315
- Apartment::Tenant.create(tenant)
316
- rescue Apartment::TenantExists
317
- # do nothing
318
- end
319
- Apartment::Tenant.switch!(tenant)
320
-
321
- ActiveRecord::Migration.create_table :tenants, force: true do |t|
322
- t.string :name
323
- t.timestamps null: true
324
- end
325
-
326
- Tenant.reindex
327
- end
328
-
329
- Apartment::Tenant.reset
330
- end
331
-
332
- ActiveRecord::Migration.create_table :products do |t|
333
- t.string :name
334
- t.integer :store_id
335
- t.boolean :in_stock
336
- t.boolean :backordered
337
- t.integer :orders_count
338
- t.decimal :found_rate
339
- t.integer :price
340
- t.string :color
341
- t.decimal :latitude, precision: 10, scale: 7
342
- t.decimal :longitude, precision: 10, scale: 7
343
- t.text :description
344
- t.text :alt_description
345
- t.timestamps null: true
346
- end
347
-
348
- ActiveRecord::Migration.create_table :stores do |t|
349
- t.string :name
350
- end
351
-
352
- ActiveRecord::Migration.create_table :regions do |t|
353
- t.string :name
354
- t.text :text
355
- end
356
-
357
- ActiveRecord::Migration.create_table :speakers do |t|
358
- t.string :name
359
- end
360
-
361
- ActiveRecord::Migration.create_table :animals do |t|
362
- t.string :name
363
- t.string :type
364
- end
365
-
366
- ActiveRecord::Migration.create_table :skus, id: :uuid do |t|
367
- t.string :name
368
- end
369
-
370
- class Product < ActiveRecord::Base
371
- belongs_to :store
372
- end
373
-
374
- class Store < ActiveRecord::Base
375
- has_many :products
376
- end
377
-
378
- class Region < ActiveRecord::Base
379
- end
380
-
381
- class Speaker < ActiveRecord::Base
382
- end
383
-
384
- class Animal < ActiveRecord::Base
385
- end
386
-
387
- class Dog < Animal
388
- end
389
-
390
- class Cat < Animal
391
- end
392
-
393
- class Sku < ActiveRecord::Base
394
- end
395
- end
396
-
397
- class Product
398
- searchkick \
399
- synonyms: [
400
- ["clorox", "bleach"],
401
- ["scallion", "greenonion"],
402
- ["saranwrap", "plasticwrap"],
403
- ["qtip", "cottonswab"],
404
- ["burger", "hamburger"],
405
- ["bandaid", "bandag"],
406
- ["UPPERCASE", "lowercase"],
407
- "lightbulb => led,lightbulb",
408
- "lightbulb => halogenlamp"
409
- ],
410
- suggest: [:name, :color],
411
- conversions: [:conversions],
412
- locations: [:location, :multiple_locations],
413
- text_start: [:name],
414
- text_middle: [:name],
415
- text_end: [:name],
416
- word_start: [:name],
417
- word_middle: [:name],
418
- word_end: [:name],
419
- highlight: [:name],
420
- searchable: [:name, :color],
421
- filterable: [:name, :color, :description],
422
- similarity: "BM25",
423
- match: ENV["MATCH"] ? ENV["MATCH"].to_sym : nil
424
-
425
- attr_accessor :conversions, :user_ids, :aisle, :details
426
-
427
- def search_data
428
- serializable_hash.except("id").merge(
429
- conversions: conversions,
430
- user_ids: user_ids,
431
- location: {lat: latitude, lon: longitude},
432
- multiple_locations: [{lat: latitude, lon: longitude}, {lat: 0, lon: 0}],
433
- aisle: aisle,
434
- details: details
435
- )
436
- end
437
-
438
- def should_index?
439
- name != "DO NOT INDEX"
440
- end
441
-
442
- def search_name
443
- {
444
- name: name
445
- }
446
- end
447
- end
448
-
449
- class Store
450
- searchkick \
451
- default_fields: elasticsearch_below60? ? nil : [:name],
452
- routing: true,
453
- merge_mappings: true,
454
- mappings: {
455
- store: {
456
- properties: {
457
- name: elasticsearch_below50? ? {type: "string", analyzer: "keyword"} : {type: "keyword"}
458
- }
459
- }
460
- }
461
-
462
- def search_document_id
463
- id
464
- end
465
-
466
- def search_routing
467
- name
468
- end
469
- end
470
-
471
- class Region
472
- searchkick \
473
- default_fields: elasticsearch_below60? ? nil : [:name],
474
- geo_shape: {
475
- territory: {tree: "quadtree", precision: "10km"}
476
- }
477
-
478
- attr_accessor :territory
479
-
480
- def search_data
481
- {
482
- name: name,
483
- text: text,
484
- territory: territory
485
- }
486
- end
487
- end
488
-
489
- class Speaker
490
- searchkick \
491
- default_fields: elasticsearch_below60? ? nil : [:name],
492
- conversions: ["conversions_a", "conversions_b"]
493
-
494
- attr_accessor :conversions_a, :conversions_b, :aisle
495
-
496
- def search_data
497
- serializable_hash.except("id").merge(
498
- conversions_a: conversions_a,
499
- conversions_b: conversions_b,
500
- aisle: aisle
501
- )
502
- end
503
- end
504
-
505
- class Animal
506
- searchkick \
507
- default_fields: elasticsearch_below60? ? nil : [:name],
508
- text_start: [:name],
509
- suggest: [:name],
510
- index_name: -> { "#{name.tableize}-#{Date.today.year}#{Searchkick.index_suffix}" },
511
- callbacks: defined?(ActiveJob) ? :async : true
512
- # wordnet: true
513
- end
514
-
515
- class Sku
516
- searchkick callbacks: defined?(ActiveJob) ? :async : true
517
- end
518
-
519
- Product.searchkick_index.delete if Product.searchkick_index.exists?
520
- Product.reindex
521
- Product.reindex # run twice for both index paths
522
- Product.create!(name: "Set mapping")
523
-
524
- Store.reindex
525
- Animal.reindex
526
- Speaker.reindex
527
- Region.reindex
528
-
529
- class Minitest::Test
530
- def setup
531
- Product.destroy_all
532
- Store.destroy_all
533
- Animal.destroy_all
534
- Speaker.destroy_all
535
- Sku.destroy_all
536
- end
537
-
538
- protected
539
-
540
- def store(documents, klass = Product)
541
- documents.shuffle.each do |document|
542
- klass.create!(document)
543
- end
544
- klass.searchkick_index.refresh
545
- end
546
-
547
- def store_names(names, klass = Product)
548
- store names.map { |name| {name: name} }, klass
549
- end
550
-
551
- # no order
552
- def assert_search(term, expected, options = {}, klass = Product)
553
- assert_equal expected.sort, klass.search(term, options).map(&:name).sort
554
- end
555
-
556
- def assert_order(term, expected, options = {}, klass = Product)
557
- assert_equal expected, klass.search(term, options).map(&:name)
558
- end
559
-
560
- def assert_equal_scores(term, options = {}, klass = Product)
561
- assert_equal 1, klass.search(term, options).hits.map { |a| a["_score"] }.uniq.size
562
- end
563
-
564
- def assert_first(term, expected, options = {}, klass = Product)
565
- assert_equal expected, klass.search(term, options).map(&:name).first
566
- end
567
- end