searchkick 4.6.3 → 5.2.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,79 +1,163 @@
1
1
  module Searchkick
2
2
  class RecordIndexer
3
- attr_reader :record, :index
3
+ attr_reader :index
4
4
 
5
- def initialize(record)
6
- @record = record
7
- @index = record.class.searchkick_index
5
+ def initialize(index)
6
+ @index = index
8
7
  end
9
8
 
10
- def reindex(method_name = nil, refresh: false, mode: nil)
11
- unless [:inline, true, nil, :async, :queue].include?(mode)
12
- raise ArgumentError, "Invalid value for mode"
13
- end
14
-
15
- mode ||= Searchkick.callbacks_value || index.options[:callbacks] || true
9
+ def reindex(records, mode:, method_name:, full: false, single: false)
10
+ # prevents exists? check if records is a relation
11
+ records = records.to_a
12
+ return if records.empty?
16
13
 
17
14
  case mode
15
+ when :async
16
+ unless defined?(ActiveJob)
17
+ raise Error, "Active Job not found"
18
+ end
19
+
20
+ # we could likely combine ReindexV2Job, BulkReindexJob, and ProcessBatchJob
21
+ # but keep them separate for now
22
+ if single
23
+ record = records.first
24
+
25
+ # always pass routing in case record is deleted
26
+ # before the async job runs
27
+ if record.respond_to?(:search_routing)
28
+ routing = record.search_routing
29
+ end
30
+
31
+ Searchkick::ReindexV2Job.perform_later(
32
+ record.class.name,
33
+ record.id.to_s,
34
+ method_name ? method_name.to_s : nil,
35
+ routing: routing,
36
+ index_name: index.name
37
+ )
38
+ else
39
+ Searchkick::BulkReindexJob.perform_later(
40
+ class_name: records.first.class.searchkick_options[:class_name],
41
+ record_ids: records.map { |r| r.id.to_s },
42
+ index_name: index.name,
43
+ method_name: method_name ? method_name.to_s : nil
44
+ )
45
+ end
18
46
  when :queue
19
47
  if method_name
20
- raise Searchkick::Error, "Partial reindex not supported with queue option"
48
+ raise Error, "Partial reindex not supported with queue option"
21
49
  end
22
50
 
23
- # always pass routing in case record is deleted
24
- # before the queue job runs
25
- if record.respond_to?(:search_routing)
26
- routing = record.search_routing
27
- end
51
+ index.reindex_queue.push_records(records)
52
+ when true, :inline
53
+ index_records, other_records = records.partition { |r| index_record?(r) }
54
+ import_inline(index_records, !full ? other_records : [], method_name: method_name, single: single)
55
+ else
56
+ raise ArgumentError, "Invalid value for mode"
57
+ end
28
58
 
29
- # escape pipe with double pipe
30
- value = queue_escape(record.id.to_s)
31
- value = "#{value}|#{queue_escape(routing)}" if routing
32
- index.reindex_queue.push(value)
33
- when :async
34
- unless defined?(ActiveJob)
35
- raise Searchkick::Error, "Active Job not found"
36
- end
59
+ # return true like model and relation reindex for now
60
+ true
61
+ end
37
62
 
38
- # always pass routing in case record is deleted
39
- # before the async job runs
40
- if record.respond_to?(:search_routing)
41
- routing = record.search_routing
42
- end
63
+ def reindex_items(klass, items, method_name:, single: false)
64
+ routing = items.to_h { |r| [r[:id], r[:routing]] }
65
+ record_ids = routing.keys
43
66
 
44
- Searchkick::ReindexV2Job.perform_later(
45
- record.class.name,
46
- record.id.to_s,
47
- method_name ? method_name.to_s : nil,
48
- routing: routing
49
- )
50
- else # bulk, inline/true/nil
51
- reindex_record(method_name)
67
+ relation = Searchkick.load_records(klass, record_ids)
68
+ # call search_import even for single records for nested associations
69
+ relation = relation.search_import if relation.respond_to?(:search_import)
70
+ records = relation.select(&:should_index?)
52
71
 
53
- index.refresh if refresh
54
- end
72
+ # determine which records to delete
73
+ delete_ids = record_ids - records.map { |r| r.id.to_s }
74
+ delete_records =
75
+ delete_ids.map do |id|
76
+ construct_record(klass, id, routing[id])
77
+ end
78
+
79
+ import_inline(records, delete_records, method_name: method_name, single: single)
55
80
  end
56
81
 
57
82
  private
58
83
 
59
- def queue_escape(value)
60
- value.gsub("|", "||")
84
+ def index_record?(record)
85
+ record.persisted? && !record.destroyed? && record.should_index?
61
86
  end
62
87
 
63
- def reindex_record(method_name)
64
- if record.destroyed? || !record.persisted? || !record.should_index?
65
- begin
66
- index.remove(record)
67
- rescue => e
68
- raise e unless Searchkick.not_found_error?(e)
69
- # do nothing if not found
88
+ # import in single request with retries
89
+ def import_inline(index_records, delete_records, method_name:, single:)
90
+ return if index_records.empty? && delete_records.empty?
91
+
92
+ maybe_bulk(index_records, delete_records, method_name, single) do
93
+ if index_records.any?
94
+ if method_name
95
+ index.bulk_update(index_records, method_name)
96
+ else
97
+ index.bulk_index(index_records)
98
+ end
70
99
  end
100
+
101
+ if delete_records.any?
102
+ index.bulk_delete(delete_records)
103
+ end
104
+ end
105
+ end
106
+
107
+ def maybe_bulk(index_records, delete_records, method_name, single)
108
+ if Searchkick.callbacks_value == :bulk
109
+ yield
71
110
  else
72
- if method_name
73
- index.update_record(record, method_name)
74
- else
75
- index.store(record)
111
+ # set action and data
112
+ action =
113
+ if single && index_records.empty?
114
+ "Remove"
115
+ elsif method_name
116
+ "Update"
117
+ else
118
+ single ? "Store" : "Import"
119
+ end
120
+ record = index_records.first || delete_records.first
121
+ name = record.class.searchkick_klass.name
122
+ message = lambda do |event|
123
+ event[:name] = "#{name} #{action}"
124
+ if single
125
+ event[:id] = index.search_id(record)
126
+ else
127
+ event[:count] = index_records.size + delete_records.size
128
+ end
129
+ end
130
+
131
+ with_retries do
132
+ Searchkick.callbacks(:bulk, message: message) do
133
+ yield
134
+ end
135
+ end
136
+ end
137
+ end
138
+
139
+ def construct_record(klass, id, routing)
140
+ record = klass.new
141
+ record.id = id
142
+ if routing
143
+ record.define_singleton_method(:search_routing) do
144
+ routing
145
+ end
146
+ end
147
+ record
148
+ end
149
+
150
+ def with_retries
151
+ retries = 0
152
+
153
+ begin
154
+ yield
155
+ rescue Faraday::ClientError => e
156
+ if retries < 1
157
+ retries += 1
158
+ retry
76
159
  end
160
+ raise e
77
161
  end
78
162
  end
79
163
  end
@@ -5,21 +5,40 @@ module Searchkick
5
5
  def initialize(name)
6
6
  @name = name
7
7
 
8
- raise Searchkick::Error, "Searchkick.redis not set" unless Searchkick.redis
8
+ raise Error, "Searchkick.redis not set" unless Searchkick.redis
9
9
  end
10
10
 
11
- def push(record_id)
12
- Searchkick.with_redis { |r| r.lpush(redis_key, record_id) }
11
+ # supports single and multiple ids
12
+ def push(record_ids)
13
+ Searchkick.with_redis { |r| r.call("LPUSH", redis_key, record_ids) }
14
+ end
15
+
16
+ def push_records(records)
17
+ record_ids =
18
+ records.map do |record|
19
+ # always pass routing in case record is deleted
20
+ # before the queue job runs
21
+ if record.respond_to?(:search_routing)
22
+ routing = record.search_routing
23
+ end
24
+
25
+ # escape pipe with double pipe
26
+ value = escape(record.id.to_s)
27
+ value = "#{value}|#{escape(routing)}" if routing
28
+ value
29
+ end
30
+
31
+ push(record_ids)
13
32
  end
14
33
 
15
34
  # TODO use reliable queuing
16
35
  def reserve(limit: 1000)
17
36
  if supports_rpop_with_count?
18
- Searchkick.with_redis { |r| r.call("rpop", redis_key, limit) }.to_a
37
+ Searchkick.with_redis { |r| r.call("RPOP", redis_key, limit) }.to_a
19
38
  else
20
39
  record_ids = []
21
40
  Searchkick.with_redis do |r|
22
- while record_ids.size < limit && (record_id = r.rpop(redis_key))
41
+ while record_ids.size < limit && (record_id = r.call("RPOP", redis_key))
23
42
  record_ids << record_id
24
43
  end
25
44
  end
@@ -28,11 +47,11 @@ module Searchkick
28
47
  end
29
48
 
30
49
  def clear
31
- Searchkick.with_redis { |r| r.del(redis_key) }
50
+ Searchkick.with_redis { |r| r.call("DEL", redis_key) }
32
51
  end
33
52
 
34
53
  def length
35
- Searchkick.with_redis { |r| r.llen(redis_key) }
54
+ Searchkick.with_redis { |r| r.call("LLEN", redis_key) }
36
55
  end
37
56
 
38
57
  private
@@ -46,7 +65,16 @@ module Searchkick
46
65
  end
47
66
 
48
67
  def redis_version
49
- @redis_version ||= Searchkick.with_redis { |r| Gem::Version.new(r.info["redis_version"]) }
68
+ @redis_version ||=
69
+ Searchkick.with_redis do |r|
70
+ info = r.call("INFO")
71
+ matches = /redis_version:(\S+)/.match(info)
72
+ Gem::Version.new(matches[1])
73
+ end
74
+ end
75
+
76
+ def escape(value)
77
+ value.gsub("|", "||")
50
78
  end
51
79
  end
52
80
  end
@@ -1,41 +1,17 @@
1
1
  module Searchkick
2
2
  class ReindexV2Job < ActiveJob::Base
3
- RECORD_NOT_FOUND_CLASSES = [
4
- "ActiveRecord::RecordNotFound",
5
- "Mongoid::Errors::DocumentNotFound",
6
- "NoBrainer::Error::DocumentNotFound",
7
- "Cequel::Record::RecordNotFound"
8
- ]
9
-
10
3
  queue_as { Searchkick.queue_name }
11
4
 
12
- def perform(klass, id, method_name = nil, routing: nil)
13
- model = klass.constantize
14
- record =
15
- begin
16
- if model.respond_to?(:unscoped)
17
- model.unscoped.find(id)
18
- else
19
- model.find(id)
20
- end
21
- rescue => e
22
- # check by name rather than rescue directly so we don't need
23
- # to determine which classes are defined
24
- raise e unless RECORD_NOT_FOUND_CLASSES.include?(e.class.name)
25
- nil
26
- end
27
-
28
- unless record
29
- record = model.new
30
- record.id = id
31
- if routing
32
- record.define_singleton_method(:search_routing) do
33
- routing
34
- end
35
- end
36
- end
37
-
38
- RecordIndexer.new(record).reindex(method_name, mode: :inline)
5
+ def perform(class_name, id, method_name = nil, routing: nil, index_name: nil)
6
+ model = Searchkick.load_model(class_name, allow_child: true)
7
+ index = model.searchkick_index(name: index_name)
8
+ # use should_index? to decide whether to index (not default scope)
9
+ # just like saving inline
10
+ # could use Searchkick.scope() in future
11
+ # but keep for now for backwards compatibility
12
+ model = model.unscoped if model.respond_to?(:unscoped)
13
+ items = [{id: id, routing: routing}]
14
+ RecordIndexer.new(index).reindex_items(model, items, method_name: method_name, single: true)
39
15
  end
40
16
  end
41
17
  end
@@ -0,0 +1,247 @@
1
+ module Searchkick
2
+ class Relation
3
+ NO_DEFAULT_VALUE = Object.new
4
+
5
+ # note: modifying body directly is not supported
6
+ # and has no impact on query after being executed
7
+ # TODO freeze body object?
8
+ delegate :body, :params, to: :query
9
+ delegate_missing_to :private_execute
10
+
11
+ attr_reader :model
12
+ alias_method :klass, :model
13
+
14
+ def initialize(model, term = "*", **options)
15
+ @model = model
16
+ @term = term
17
+ @options = options
18
+
19
+ # generate query to validate options
20
+ query
21
+ end
22
+
23
+ # same as Active Record
24
+ def inspect
25
+ entries = results.first(11).map!(&:inspect)
26
+ entries[10] = "..." if entries.size == 11
27
+ "#<#{self.class.name} [#{entries.join(', ')}]>"
28
+ end
29
+
30
+ def execute
31
+ Searchkick.warn("The execute method is no longer needed")
32
+ load
33
+ end
34
+
35
+ # experimental
36
+ def limit(value)
37
+ clone.limit!(value)
38
+ end
39
+
40
+ # experimental
41
+ def limit!(value)
42
+ check_loaded
43
+ @options[:limit] = value
44
+ self
45
+ end
46
+
47
+ # experimental
48
+ def offset(value = NO_DEFAULT_VALUE)
49
+ # TODO remove in Searchkick 6
50
+ if value == NO_DEFAULT_VALUE
51
+ private_execute.offset
52
+ else
53
+ clone.offset!(value)
54
+ end
55
+ end
56
+
57
+ # experimental
58
+ def offset!(value)
59
+ check_loaded
60
+ @options[:offset] = value
61
+ self
62
+ end
63
+
64
+ # experimental
65
+ def page(value)
66
+ clone.page!(value)
67
+ end
68
+
69
+ # experimental
70
+ def page!(value)
71
+ check_loaded
72
+ @options[:page] = value
73
+ self
74
+ end
75
+
76
+ # experimental
77
+ def per_page(value = NO_DEFAULT_VALUE)
78
+ # TODO remove in Searchkick 6
79
+ if value == NO_DEFAULT_VALUE
80
+ private_execute.per_page
81
+ else
82
+ clone.per_page!(value)
83
+ end
84
+ end
85
+
86
+ # experimental
87
+ def per_page!(value)
88
+ check_loaded
89
+ @options[:per_page] = value
90
+ self
91
+ end
92
+
93
+ # experimental
94
+ def where(value = NO_DEFAULT_VALUE)
95
+ if value == NO_DEFAULT_VALUE
96
+ Where.new(self)
97
+ else
98
+ clone.where!(value)
99
+ end
100
+ end
101
+
102
+ # experimental
103
+ def where!(value)
104
+ check_loaded
105
+ if @options[:where]
106
+ @options[:where] = {_and: [@options[:where], ensure_permitted(value)]}
107
+ else
108
+ @options[:where] = ensure_permitted(value)
109
+ end
110
+ self
111
+ end
112
+
113
+ # experimental
114
+ def rewhere(value)
115
+ clone.rewhere!(value)
116
+ end
117
+
118
+ # experimental
119
+ def rewhere!(value)
120
+ check_loaded
121
+ @options[:where] = ensure_permitted(value)
122
+ self
123
+ end
124
+
125
+ # experimental
126
+ def order(*values)
127
+ clone.order!(*values)
128
+ end
129
+
130
+ # experimental
131
+ def order!(*values)
132
+ values = values.first if values.size == 1 && values.first.is_a?(Array)
133
+ check_loaded
134
+ (@options[:order] ||= []).concat(values)
135
+ self
136
+ end
137
+
138
+ # experimental
139
+ def reorder(*values)
140
+ clone.reorder!(*values)
141
+ end
142
+
143
+ # experimental
144
+ def reorder!(*values)
145
+ check_loaded
146
+ @options[:order] = values
147
+ self
148
+ end
149
+
150
+ # experimental
151
+ def select(*values, &block)
152
+ if block_given?
153
+ private_execute.select(*values, &block)
154
+ else
155
+ clone.select!(*values)
156
+ end
157
+ end
158
+
159
+ # experimental
160
+ def select!(*values)
161
+ check_loaded
162
+ (@options[:select] ||= []).concat(values)
163
+ self
164
+ end
165
+
166
+ # experimental
167
+ def reselect(*values)
168
+ clone.reselect!(*values)
169
+ end
170
+
171
+ # experimental
172
+ def reselect!(*values)
173
+ check_loaded
174
+ @options[:select] = values
175
+ self
176
+ end
177
+
178
+ # experimental
179
+ def includes(*values)
180
+ clone.includes!(*values)
181
+ end
182
+
183
+ # experimental
184
+ def includes!(*values)
185
+ check_loaded
186
+ (@options[:includes] ||= []).concat(values)
187
+ self
188
+ end
189
+
190
+ # experimental
191
+ def only(*keys)
192
+ Relation.new(@model, @term, **@options.slice(*keys))
193
+ end
194
+
195
+ # experimental
196
+ def except(*keys)
197
+ Relation.new(@model, @term, **@options.except(*keys))
198
+ end
199
+
200
+ # experimental
201
+ def load
202
+ private_execute
203
+ self
204
+ end
205
+
206
+ def loaded?
207
+ !@execute.nil?
208
+ end
209
+
210
+ def respond_to_missing?(method_name, include_all)
211
+ Results.new(nil, nil, nil).respond_to?(method_name, include_all) || super
212
+ end
213
+
214
+ # TODO uncomment in 6.0
215
+ # def to_yaml
216
+ # private_execute.to_a.to_yaml
217
+ # end
218
+
219
+ private
220
+
221
+ def private_execute
222
+ @execute ||= query.execute
223
+ end
224
+
225
+ def query
226
+ @query ||= Query.new(@model, @term, **@options)
227
+ end
228
+
229
+ def check_loaded
230
+ raise Error, "Relation loaded" if loaded?
231
+
232
+ # reset query since options will change
233
+ @query = nil
234
+ end
235
+
236
+ # provides *very* basic protection from unfiltered parameters
237
+ # this is not meant to be comprehensive and may be expanded in the future
238
+ def ensure_permitted(obj)
239
+ obj.to_h
240
+ end
241
+
242
+ def initialize_copy(other)
243
+ super
244
+ @execute = nil
245
+ end
246
+ end
247
+ end