searchkick 1.1.1 → 1.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +20 -7
- data/lib/searchkick.rb +71 -12
- data/lib/searchkick/index.rb +16 -29
- data/lib/searchkick/middleware.rb +12 -0
- data/lib/searchkick/model.rb +6 -8
- data/lib/searchkick/query.rb +1 -1
- data/lib/searchkick/version.rb +1 -1
- data/test/boost_test.rb +9 -0
- data/test/callbacks_test.rb +27 -0
- data/test/test_helper.rb +1 -0
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 01f20120d276745b8434ffaa0529c328f8870080
|
4
|
+
data.tar.gz: 18756878052ac0bd566d025e7f87569992f903aa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 51e077ad2d600a823fea927c02a9c76299423a923286bf483ac9d0caffd32632de2592b9116fb5e34bd7558216e171d8ce8cff78a43bf2900a394830580cbf6c
|
7
|
+
data.tar.gz: 106cca0902d1a59b1de01380b6da09972c626075520e54a29e879740b90ff0ecf42f69d8fe16467308ed8c013526b51295a9e1e848ac1e0cec945aa524b309c0
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
## 1.1.2
|
2
|
+
|
3
|
+
- Added bulk updates with `callbacks` method
|
4
|
+
- Added `bulk_delete` method
|
5
|
+
- Added `search_timeout` option
|
6
|
+
- Fixed bug with new location format for `boost_by_distance`
|
7
|
+
|
1
8
|
## 1.1.1
|
2
9
|
|
3
10
|
- Added support for `{lat: lat, lon: lon}` as preferred format for locations
|
data/README.md
CHANGED
@@ -406,6 +406,22 @@ There are three strategies for keeping the index synced with your database.
|
|
406
406
|
end
|
407
407
|
```
|
408
408
|
|
409
|
+
You can also do bulk updates.
|
410
|
+
|
411
|
+
```ruby
|
412
|
+
Searchkick.callbacks(:bulk) do
|
413
|
+
User.find_each(&:update_fields)
|
414
|
+
end
|
415
|
+
```
|
416
|
+
|
417
|
+
Or temporarily skip updates.
|
418
|
+
|
419
|
+
```ruby
|
420
|
+
Searchkick.callbacks(false) do
|
421
|
+
User.find_each(&:update_fields)
|
422
|
+
end
|
423
|
+
```
|
424
|
+
|
409
425
|
#### Associations
|
410
426
|
|
411
427
|
Data is **not** automatically synced when an association is updated. If this is desired, add a callback to reindex:
|
@@ -1129,19 +1145,16 @@ class Product < ActiveRecord::Base
|
|
1129
1145
|
end
|
1130
1146
|
```
|
1131
1147
|
|
1132
|
-
|
1148
|
+
Change timeout
|
1133
1149
|
|
1134
1150
|
```ruby
|
1135
|
-
|
1136
|
-
ExpensiveProductsTask.execute
|
1137
|
-
Product.enable_search_callbacks # or use Searchkick.enable_callbacks for all models
|
1138
|
-
Product.reindex
|
1151
|
+
Searchkick.timeout = 15 # defaults to 10
|
1139
1152
|
```
|
1140
1153
|
|
1141
|
-
|
1154
|
+
Set a lower timeout for searches
|
1142
1155
|
|
1143
1156
|
```ruby
|
1144
|
-
Searchkick.
|
1157
|
+
Searchkick.search_timeout = 3
|
1145
1158
|
```
|
1146
1159
|
|
1147
1160
|
Change the search method name in `config/initializers/searchkick.rb`
|
data/lib/searchkick.rb
CHANGED
@@ -8,6 +8,7 @@ require "searchkick/query"
|
|
8
8
|
require "searchkick/reindex_job"
|
9
9
|
require "searchkick/model"
|
10
10
|
require "searchkick/tasks"
|
11
|
+
require "searchkick/middleware"
|
11
12
|
require "searchkick/logging" if defined?(Rails)
|
12
13
|
|
13
14
|
# background jobs
|
@@ -27,11 +28,8 @@ module Searchkick
|
|
27
28
|
class ImportError < Error; end
|
28
29
|
|
29
30
|
class << self
|
30
|
-
attr_accessor :search_method_name
|
31
|
-
|
32
|
-
attr_accessor :timeout
|
33
|
-
attr_accessor :models
|
34
|
-
attr_writer :env
|
31
|
+
attr_accessor :search_method_name, :wordnet_path, :timeout, :models
|
32
|
+
attr_writer :client, :env, :search_timeout
|
35
33
|
end
|
36
34
|
self.search_method_name = :search
|
37
35
|
self.wordnet_path = "/var/lib/wn_s.pl"
|
@@ -43,11 +41,17 @@ module Searchkick
|
|
43
41
|
Elasticsearch::Client.new(
|
44
42
|
url: ENV["ELASTICSEARCH_URL"],
|
45
43
|
transport_options: {request: {timeout: timeout}}
|
46
|
-
)
|
44
|
+
) do |f|
|
45
|
+
f.use Searchkick::Middleware
|
46
|
+
end
|
47
47
|
end
|
48
48
|
|
49
|
-
|
50
|
-
|
49
|
+
def self.env
|
50
|
+
@env ||= ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.search_timeout
|
54
|
+
@search_timeout || timeout
|
51
55
|
end
|
52
56
|
|
53
57
|
def self.server_version
|
@@ -55,19 +59,74 @@ module Searchkick
|
|
55
59
|
end
|
56
60
|
|
57
61
|
def self.enable_callbacks
|
58
|
-
|
62
|
+
self.callbacks_value = nil
|
59
63
|
end
|
60
64
|
|
61
65
|
def self.disable_callbacks
|
62
|
-
|
66
|
+
self.callbacks_value = false
|
63
67
|
end
|
64
68
|
|
65
69
|
def self.callbacks?
|
66
70
|
Thread.current[:searchkick_callbacks_enabled].nil? || Thread.current[:searchkick_callbacks_enabled]
|
67
71
|
end
|
68
72
|
|
69
|
-
def self.
|
70
|
-
|
73
|
+
def self.callbacks(value)
|
74
|
+
if block_given?
|
75
|
+
previous_value = callbacks_value
|
76
|
+
begin
|
77
|
+
self.callbacks_value = value
|
78
|
+
yield
|
79
|
+
perform_bulk if callbacks_value == :bulk
|
80
|
+
ensure
|
81
|
+
self.callbacks_value = previous_value
|
82
|
+
end
|
83
|
+
else
|
84
|
+
self.callbacks_value = value
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# private
|
89
|
+
def self.queue_items(items)
|
90
|
+
queued_items.concat(items)
|
91
|
+
perform_bulk unless callbacks_value == :bulk
|
92
|
+
end
|
93
|
+
|
94
|
+
# private
|
95
|
+
def self.perform_bulk
|
96
|
+
items = queued_items
|
97
|
+
clear_queued_items
|
98
|
+
perform_items(items)
|
99
|
+
end
|
100
|
+
|
101
|
+
# private
|
102
|
+
def self.perform_items(items)
|
103
|
+
if items.any?
|
104
|
+
response = client.bulk(body: items)
|
105
|
+
if response["errors"]
|
106
|
+
first_item = response["items"].first
|
107
|
+
raise Searchkick::ImportError, (first_item["index"] || first_item["delete"])["error"]
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
# private
|
113
|
+
def self.queued_items
|
114
|
+
Thread.current[:searchkick_queued_items] ||= []
|
115
|
+
end
|
116
|
+
|
117
|
+
# private
|
118
|
+
def self.clear_queued_items
|
119
|
+
Thread.current[:searchkick_queued_items] = []
|
120
|
+
end
|
121
|
+
|
122
|
+
# private
|
123
|
+
def self.callbacks_value
|
124
|
+
Thread.current[:searchkick_callbacks_enabled]
|
125
|
+
end
|
126
|
+
|
127
|
+
# private
|
128
|
+
def self.callbacks_value=(value)
|
129
|
+
Thread.current[:searchkick_callbacks_enabled] = value
|
71
130
|
end
|
72
131
|
end
|
73
132
|
|
data/lib/searchkick/index.rb
CHANGED
@@ -45,38 +45,21 @@ module Searchkick
|
|
45
45
|
# record based
|
46
46
|
|
47
47
|
def store(record)
|
48
|
-
|
49
|
-
index: name,
|
50
|
-
type: document_type(record),
|
51
|
-
id: search_id(record),
|
52
|
-
body: search_data(record)
|
53
|
-
)
|
48
|
+
bulk_index([record])
|
54
49
|
end
|
55
50
|
|
56
51
|
def remove(record)
|
57
|
-
|
58
|
-
unless id.blank?
|
59
|
-
client.delete(
|
60
|
-
index: name,
|
61
|
-
type: document_type(record),
|
62
|
-
id: id
|
63
|
-
)
|
64
|
-
end
|
52
|
+
bulk_delete([record])
|
65
53
|
end
|
66
54
|
|
67
|
-
def
|
68
|
-
records.
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
body: batch.map { |r| {index: {_id: search_id(r), data: search_data(r)}} }
|
74
|
-
)
|
75
|
-
if response["errors"]
|
76
|
-
raise Searchkick::ImportError, response["items"].first["index"]["error"]
|
77
|
-
end
|
78
|
-
end
|
55
|
+
def bulk_delete(records)
|
56
|
+
Searchkick.queue_items(records.reject { |r| r.id.blank? }.map { |r| {delete: {_index: name, _type: document_type(r), _id: search_id(r)}} })
|
57
|
+
end
|
58
|
+
|
59
|
+
def bulk_index(records)
|
60
|
+
Searchkick.queue_items(records.map { |r| {index: {_index: name, _type: document_type(r), _id: search_id(r), data: search_data(r)}} })
|
79
61
|
end
|
62
|
+
alias_method :import, :bulk_index
|
80
63
|
|
81
64
|
def retrieve(record)
|
82
65
|
client.get(
|
@@ -99,10 +82,14 @@ module Searchkick
|
|
99
82
|
end
|
100
83
|
|
101
84
|
def reindex_record_async(record)
|
102
|
-
if
|
103
|
-
Searchkick::ReindexV2Job
|
85
|
+
if Searchkick.callbacks_value.nil?
|
86
|
+
if defined?(Searchkick::ReindexV2Job)
|
87
|
+
Searchkick::ReindexV2Job.perform_later(record.class.name, record.id.to_s)
|
88
|
+
else
|
89
|
+
Delayed::Job.enqueue Searchkick::ReindexJob.new(record.class.name, record.id.to_s)
|
90
|
+
end
|
104
91
|
else
|
105
|
-
|
92
|
+
reindex_record(record)
|
106
93
|
end
|
107
94
|
end
|
108
95
|
|
@@ -0,0 +1,12 @@
|
|
1
|
+
require "faraday/middleware"
|
2
|
+
|
3
|
+
module Searchkick
|
4
|
+
class Middleware < Faraday::Middleware
|
5
|
+
def call(env)
|
6
|
+
if env.method == :get && env.url.path.to_s.end_with?("/_search")
|
7
|
+
env.request.timeout = Searchkick.search_timeout
|
8
|
+
end
|
9
|
+
@app.call(env)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
data/lib/searchkick/model.rb
CHANGED
@@ -70,14 +70,12 @@ module Searchkick
|
|
70
70
|
end
|
71
71
|
extend Searchkick::Reindex # legacy for Searchjoy
|
72
72
|
|
73
|
-
|
74
|
-
|
75
|
-
if
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
after_destroy callback_name, if: proc { self.class.search_callbacks? }
|
80
|
-
end
|
73
|
+
callback_name = callbacks == :async ? :reindex_async : :reindex
|
74
|
+
if respond_to?(:after_commit)
|
75
|
+
after_commit callback_name, if: proc { self.class.search_callbacks? }
|
76
|
+
else
|
77
|
+
after_save callback_name, if: proc { self.class.search_callbacks? }
|
78
|
+
after_destroy callback_name, if: proc { self.class.search_callbacks? }
|
81
79
|
end
|
82
80
|
|
83
81
|
def reindex
|
data/lib/searchkick/query.rb
CHANGED
@@ -335,7 +335,7 @@ module Searchkick
|
|
335
335
|
raise ArgumentError, "boost_by_distance requires :field and :origin"
|
336
336
|
end
|
337
337
|
function_params = boost_by_distance.select { |k, _| [:origin, :scale, :offset, :decay].include?(k) }
|
338
|
-
function_params[:origin] = function_params[:origin]
|
338
|
+
function_params[:origin] = location_value(function_params[:origin])
|
339
339
|
custom_filters << {
|
340
340
|
boost_by_distance[:function] => {
|
341
341
|
boost_by_distance[:field] => function_params
|
data/lib/searchkick/version.rb
CHANGED
data/test/boost_test.rb
CHANGED
@@ -132,4 +132,13 @@ class BoostTest < Minitest::Test
|
|
132
132
|
]
|
133
133
|
assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {field: :location, origin: [37, -122], scale: "1000mi"}
|
134
134
|
end
|
135
|
+
|
136
|
+
def test_boost_by_distance_hash
|
137
|
+
store [
|
138
|
+
{name: "San Francisco", latitude: 37.7833, longitude: -122.4167},
|
139
|
+
{name: "San Antonio", latitude: 29.4167, longitude: -98.5000},
|
140
|
+
{name: "San Marino", latitude: 43.9333, longitude: 12.4667}
|
141
|
+
]
|
142
|
+
assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {field: :location, origin: {lat: 37, lon: -122}, scale: "1000mi"}
|
143
|
+
end
|
135
144
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require_relative "test_helper"
|
2
|
+
|
3
|
+
class CallbacksTest < Minitest::Test
|
4
|
+
def test_true_create
|
5
|
+
Searchkick.callbacks(true) do
|
6
|
+
store_names ["Product A", "Product B"]
|
7
|
+
end
|
8
|
+
Product.searchkick_index.refresh
|
9
|
+
assert_search "product", ["Product A", "Product B"]
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_false_create
|
13
|
+
Searchkick.callbacks(false) do
|
14
|
+
store_names ["Product A", "Product B"]
|
15
|
+
end
|
16
|
+
Product.searchkick_index.refresh
|
17
|
+
assert_search "product", []
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_bulk_create
|
21
|
+
Searchkick.callbacks(:bulk) do
|
22
|
+
store_names ["Product A", "Product B"]
|
23
|
+
end
|
24
|
+
Product.searchkick_index.refresh
|
25
|
+
assert_search "product", ["Product A", "Product B"]
|
26
|
+
end
|
27
|
+
end
|
data/test/test_helper.rb
CHANGED
@@ -11,6 +11,7 @@ Minitest::Test = Minitest::Unit::TestCase unless defined?(Minitest::Test)
|
|
11
11
|
|
12
12
|
File.delete("elasticsearch.log") if File.exist?("elasticsearch.log")
|
13
13
|
Searchkick.client.transport.logger = Logger.new("elasticsearch.log")
|
14
|
+
Searchkick.search_timeout = 5
|
14
15
|
|
15
16
|
puts "Running against Elasticsearch #{Searchkick.server_version}"
|
16
17
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: searchkick
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-12-
|
11
|
+
date: 2015-12-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activemodel
|
@@ -111,6 +111,7 @@ files:
|
|
111
111
|
- lib/searchkick.rb
|
112
112
|
- lib/searchkick/index.rb
|
113
113
|
- lib/searchkick/logging.rb
|
114
|
+
- lib/searchkick/middleware.rb
|
114
115
|
- lib/searchkick/model.rb
|
115
116
|
- lib/searchkick/query.rb
|
116
117
|
- lib/searchkick/reindex_job.rb
|
@@ -122,6 +123,7 @@ files:
|
|
122
123
|
- test/aggs_test.rb
|
123
124
|
- test/autocomplete_test.rb
|
124
125
|
- test/boost_test.rb
|
126
|
+
- test/callbacks_test.rb
|
125
127
|
- test/ci/before_install.sh
|
126
128
|
- test/dangerous_reindex_test.rb
|
127
129
|
- test/facets_test.rb
|
@@ -184,6 +186,7 @@ test_files:
|
|
184
186
|
- test/aggs_test.rb
|
185
187
|
- test/autocomplete_test.rb
|
186
188
|
- test/boost_test.rb
|
189
|
+
- test/callbacks_test.rb
|
187
190
|
- test/ci/before_install.sh
|
188
191
|
- test/dangerous_reindex_test.rb
|
189
192
|
- test/facets_test.rb
|