spree-typesense 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +203 -0
- data/LICENSE +9 -0
- data/README.md +311 -0
- data/Rakefile +6 -0
- data/app/models/spree/search_provider/typesense/facets.rb +147 -0
- data/app/models/spree/search_provider/typesense/index_layout.rb +91 -0
- data/app/models/spree/search_provider/typesense/query_builder.rb +161 -0
- data/app/models/spree/search_provider/typesense/schema.rb +128 -0
- data/app/models/spree/search_provider/typesense.rb +901 -0
- data/app/presenters/spree/typesense/product_presenter.rb +252 -0
- data/lib/spree/typesense/circuit_breaker.rb +137 -0
- data/lib/spree/typesense/engine.rb +15 -0
- data/lib/spree/typesense/version.rb +6 -0
- data/lib/spree/typesense.rb +42 -0
- data/lib/spree-typesense.rb +6 -0
- metadata +115 -0
|
@@ -0,0 +1,901 @@
|
|
|
1
|
+
module Spree
|
|
2
|
+
module SearchProvider
|
|
3
|
+
class Typesense < Base
|
|
4
|
+
# Typesense rejected some documents of an import, which it reports per
|
|
5
|
+
# document rather than as an HTTP error.
|
|
6
|
+
class ImportError < StandardError
|
|
7
|
+
# The message names this many ids; #failures keeps them all.
|
|
8
|
+
MAX_IDS_IN_MESSAGE = 50
|
|
9
|
+
|
|
10
|
+
# @return [Array<Hash>] `{ id:, error: }` per rejected document
|
|
11
|
+
attr_reader :failures
|
|
12
|
+
|
|
13
|
+
# @param collection_name [String]
|
|
14
|
+
# @param failures [Array<Hash>] `{ id:, error: }` per rejected document
|
|
15
|
+
def initialize(collection_name, failures)
|
|
16
|
+
@failures = failures
|
|
17
|
+
ids = failures.map { |f| f[:id] }
|
|
18
|
+
shown = ids.first(MAX_IDS_IN_MESSAGE).join(', ')
|
|
19
|
+
more = ids.size > MAX_IDS_IN_MESSAGE ? " and #{ids.size - MAX_IDS_IN_MESSAGE} more" : ''
|
|
20
|
+
super("#{ids.size} #{ids.size == 1 ? 'document' : 'documents'} rejected by Typesense " \
|
|
21
|
+
"collection '#{collection_name}': #{shown}#{more}")
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# A search Typesense answered with an error body instead of an HTTP
|
|
26
|
+
# error, as multi_search does per search.
|
|
27
|
+
class SearchError < ::Typesense::Error; end
|
|
28
|
+
|
|
29
|
+
# The network errors typesense-ruby 5.0's ApiCall retries on and, once
|
|
30
|
+
# its retries are spent, raises as they are.
|
|
31
|
+
NETWORK_ERRORS = [
|
|
32
|
+
Errno::EINVAL, Errno::ENETDOWN, Errno::ENETUNREACH, Errno::ENETRESET, Errno::ECONNABORTED,
|
|
33
|
+
Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTDOWN, Errno::EHOSTUNREACH
|
|
34
|
+
].freeze
|
|
35
|
+
|
|
36
|
+
# What the Typesense client raises when it cannot answer: its own errors
|
|
37
|
+
# (HTTP 4xx/5xx, timeouts, missing configuration, SearchError), Faraday's,
|
|
38
|
+
# the network errors above, and a response body that is not JSON. The
|
|
39
|
+
# public search entry points answer from the Database provider on any of
|
|
40
|
+
# these. The base class is ::Typesense::Error; there is no
|
|
41
|
+
# ::Typesense::Error::Error.
|
|
42
|
+
UNAVAILABLE_ERRORS = [::Typesense::Error, Faraday::Error, *NETWORK_ERRORS, JSON::ParserError].freeze
|
|
43
|
+
|
|
44
|
+
# The failures that pass by themselves: timeouts, 5xx, lost connections.
|
|
45
|
+
# These are reported as warnings. The rest (a rejected query from schema
|
|
46
|
+
# drift, a missing collection, a bad API key, missing configuration) will
|
|
47
|
+
# not pass without someone acting, and are reported as errors, so a broken
|
|
48
|
+
# index does not go unnoticed while the Database provider answers.
|
|
49
|
+
TRANSIENT_ERRORS = [
|
|
50
|
+
::Typesense::Error::TimeoutError, ::Typesense::Error::ServerError, ::Typesense::Error::HTTPStatus0Error,
|
|
51
|
+
Faraday::Error, *NETWORK_ERRORS, JSON::ParserError
|
|
52
|
+
].freeze
|
|
53
|
+
|
|
54
|
+
# Raised by #request_typesense in place of a failure in UNAVAILABLE_ERRORS,
|
|
55
|
+
# so the search entry points fall back on Typesense failures only, and
|
|
56
|
+
# never on an error raised around the request (the Database provider's,
|
|
57
|
+
# or the authorized scope's), whatever its class.
|
|
58
|
+
class Unavailable < StandardError
|
|
59
|
+
attr_reader :error
|
|
60
|
+
|
|
61
|
+
def initialize(error)
|
|
62
|
+
@error = error
|
|
63
|
+
super("#{error.class}: #{error.message}")
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
private_constant :Unavailable
|
|
67
|
+
|
|
68
|
+
# Reported once when the circuit breaker opens: Typesense has failed
|
|
69
|
+
# transiently several times in a row, and searches skip it for a while.
|
|
70
|
+
class CircuitOpened < StandardError; end
|
|
71
|
+
|
|
72
|
+
# Set once missing configuration has been logged; see #report_unavailable.
|
|
73
|
+
UNCONFIGURED_LOGGED = Concurrent::AtomicBoolean.new(false)
|
|
74
|
+
|
|
75
|
+
# 'manual' is the merchant's own order, as the Database provider offers
|
|
76
|
+
# it (Taxon/Collection::SORT_ORDERS). On Spree 6 a manually sorted
|
|
77
|
+
# category or collection listing sorts by the product's position in it
|
|
78
|
+
# (QueryBuilder.sort_by).
|
|
79
|
+
AVAILABLE_SORT_OPTIONS = %w[manual price -price name -name -available_on available_on best_selling].freeze
|
|
80
|
+
|
|
81
|
+
# Spree's Spree::Api::V3::ProductFilterSortOptionSerializer declares
|
|
82
|
+
# `attributes :id, :label` and raises KeyError when :label is absent.
|
|
83
|
+
#
|
|
84
|
+
# NOTE: core also appends custom-field sort options
|
|
85
|
+
# (`custom_field_schema.sort_options`). We do not yet surface those, so a
|
|
86
|
+
# store using custom-field sorts will not see them under this provider.
|
|
87
|
+
def sort_options_payload
|
|
88
|
+
AVAILABLE_SORT_OPTIONS.map { |id| { id: id, label: nil } }
|
|
89
|
+
end
|
|
90
|
+
DEFAULT_SORT = 'manual'.freeze
|
|
91
|
+
|
|
92
|
+
# Typesense defaults to 10 facet values per field and truncates silently.
|
|
93
|
+
# Needed by option/category facets; the price range reads stats instead.
|
|
94
|
+
MAX_FACET_VALUES = 100
|
|
95
|
+
|
|
96
|
+
# See #schema_client.
|
|
97
|
+
SCHEMA_CHANGE_TIMEOUT_SECONDS = 120
|
|
98
|
+
|
|
99
|
+
# See #search_client. TYPESENSE_SEARCH_TIMEOUT overrides it.
|
|
100
|
+
SEARCH_TIMEOUT_SECONDS = 2
|
|
101
|
+
|
|
102
|
+
def self.indexing_required?
|
|
103
|
+
true
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Brings the collection the live name serves from up to
|
|
107
|
+
# Schema.definition in place, adding the fields it lacks. With no index
|
|
108
|
+
# at all it does nothing: only #reindex creates one, so a store never
|
|
109
|
+
# serves an index that holds part of its catalogue without falling back.
|
|
110
|
+
def ensure_index_settings!
|
|
111
|
+
name = live_collection_name
|
|
112
|
+
update_collection_schema!(name) if name
|
|
113
|
+
nil
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Mirrors meilisearch.rb#reindex: default scope is store.products (not
|
|
117
|
+
# Spree::Product.where(store_id:)), batch size 500, deterministic order,
|
|
118
|
+
# lazy-preloaded associations. Returns the number of DOCUMENTS indexed
|
|
119
|
+
# (not products — one product can emit N locale x currency documents).
|
|
120
|
+
#
|
|
121
|
+
# Builds a fresh versioned collection from Schema.definition, marked
|
|
122
|
+
# `__building` while it fills so single-product writes reach it too,
|
|
123
|
+
# then points the live alias at it and drops what it replaced: the
|
|
124
|
+
# older versioned collections and a real collection holding the live
|
|
125
|
+
# name (0.2.0's, or one a rollback to 0.2.0 created). Searches answer
|
|
126
|
+
# from the previous collection until the swap, and documents of
|
|
127
|
+
# products that are gone or out of scope go with the old collection.
|
|
128
|
+
#
|
|
129
|
+
# The scope becomes the whole index: pass every product the store
|
|
130
|
+
# should find (the rake task passes store.products). To refresh some
|
|
131
|
+
# products, use #index for each.
|
|
132
|
+
#
|
|
133
|
+
# A batch with rejected documents does not stop the reindex, and those
|
|
134
|
+
# documents are not counted. Once every batch is sent, an ImportError
|
|
135
|
+
# names them all, so `rake spree:search:reindex` fails visibly; the new
|
|
136
|
+
# collection is then dropped and the live alias left where it was, so
|
|
137
|
+
# no product goes missing from search. Any other error does the same.
|
|
138
|
+
def reindex(scope = nil)
|
|
139
|
+
scope ||= store.products
|
|
140
|
+
layout = index_layout
|
|
141
|
+
building = layout.new_version_name(current_time_ms)
|
|
142
|
+
client.collections.create(Schema.definition(building))
|
|
143
|
+
layout.point!(layout.building_name, building)
|
|
144
|
+
|
|
145
|
+
begin
|
|
146
|
+
indexed, failures = index_scope(scope, into: building)
|
|
147
|
+
rescue StandardError
|
|
148
|
+
abandon_build(layout, building)
|
|
149
|
+
raise
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
Rails.logger.info do
|
|
153
|
+
"[spree_typesense] reindex complete: #{indexed} documents indexed, #{failures.size} failed"
|
|
154
|
+
end
|
|
155
|
+
if failures.any?
|
|
156
|
+
abandon_build(layout, building)
|
|
157
|
+
raise ImportError.new(building, failures)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
swap_live_alias(layout, building)
|
|
161
|
+
indexed
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Upserts the product's documents, then deletes those of its documents
|
|
165
|
+
# it no longer produces: one product is a document per locale x currency,
|
|
166
|
+
# and a product that lost a currency's price, say, must not stay
|
|
167
|
+
# searchable in that market. A product that produces none loses them all.
|
|
168
|
+
#
|
|
169
|
+
# Every document is stamped `indexed_at` with the moment the product was
|
|
170
|
+
# read, and only documents stamped earlier are deleted, so when two jobs
|
|
171
|
+
# for one product overlap, the older one finishing last cannot drop what
|
|
172
|
+
# the newer one wrote (#9).
|
|
173
|
+
#
|
|
174
|
+
# Writes to the live collection and, while a reindex is filling one, to
|
|
175
|
+
# that one too, so the change survives the swap. A collection being
|
|
176
|
+
# built that has since gone (its reindex died) is skipped.
|
|
177
|
+
#
|
|
178
|
+
# Raises ImportError (after the deletion) when Typesense rejects a
|
|
179
|
+
# document, so the index job retries it.
|
|
180
|
+
def index(product)
|
|
181
|
+
read_at = current_time_ms
|
|
182
|
+
documents = Array(presenter_class.new(product, store).call)
|
|
183
|
+
|
|
184
|
+
results = nil
|
|
185
|
+
rejected = nil
|
|
186
|
+
each_write_target do |target, live|
|
|
187
|
+
begin
|
|
188
|
+
batch_results = index_batch(documents, into: target, indexed_at: read_at)
|
|
189
|
+
results = batch_results if live
|
|
190
|
+
rescue ImportError => e
|
|
191
|
+
rejected ||= e
|
|
192
|
+
end
|
|
193
|
+
remove_stale_documents(product.prefixed_id, documents, from: target, written_before: read_at)
|
|
194
|
+
end
|
|
195
|
+
raise rejected if rejected
|
|
196
|
+
|
|
197
|
+
results
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Typesense answers an import with HTTP 200 and one result per document,
|
|
201
|
+
# in order, so a rejected document raises nothing by itself. Each
|
|
202
|
+
# rejection is logged with its document id, and once the whole batch is
|
|
203
|
+
# sent an ImportError names them all.
|
|
204
|
+
#
|
|
205
|
+
# @param documents [Array<Hash>]
|
|
206
|
+
# @param into [String] the collection or alias to import into
|
|
207
|
+
# @param indexed_at [Integer] when the products were read, epoch ms
|
|
208
|
+
# @return [Array<Hash>, nil] Typesense's per-document results
|
|
209
|
+
# @raise [ImportError] when Typesense rejected any document
|
|
210
|
+
def index_batch(documents, into: collection_name, indexed_at: current_time_ms)
|
|
211
|
+
return if documents.blank?
|
|
212
|
+
|
|
213
|
+
documents = documents.map { |d| derive_fields(d, indexed_at) }
|
|
214
|
+
results = client.collections[into].documents.import(documents, action: 'upsert')
|
|
215
|
+
|
|
216
|
+
failures = import_failures(documents, results, into)
|
|
217
|
+
raise ImportError.new(into, failures) if failures.any?
|
|
218
|
+
|
|
219
|
+
results
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
def remove(product)
|
|
223
|
+
remove_by_id(product.prefixed_id)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# Public entry point. Answers from Typesense, or from the Database
|
|
227
|
+
# provider when Typesense cannot answer: unreachable, timed out, a 5xx,
|
|
228
|
+
# a rejected query (a 400 from a field the collection lacks, a missing
|
|
229
|
+
# collection), or not configured. Correct results, more slowly, beat
|
|
230
|
+
# an empty page. Only the Typesense request is covered: errors raised
|
|
231
|
+
# around it, by the Database provider or the authorized scope, still
|
|
232
|
+
# raise, as do errors that are not Typesense failing.
|
|
233
|
+
def search_and_filter(scope:, query: nil, filters: {}, sort: nil, page: 1, limit: 25)
|
|
234
|
+
fallback = lambda do
|
|
235
|
+
fallback_provider.search_and_filter(
|
|
236
|
+
scope: scope, query: query, filters: filters, sort: sort, page: page, limit: limit
|
|
237
|
+
)
|
|
238
|
+
end
|
|
239
|
+
answer_through_breaker(query, fallback) do
|
|
240
|
+
search_with_typesense(scope: scope, query: query, filters: filters, sort: sort, page: page, limit: limit)
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Raises Unavailable when Typesense cannot answer; #search_and_filter
|
|
245
|
+
# falls back. The intersection with the authorized scope runs outside
|
|
246
|
+
# #request_typesense, so its errors are never taken for Typesense's.
|
|
247
|
+
def search_with_typesense(scope:, query: nil, filters: {}, sort: nil, page: 1, limit: 25)
|
|
248
|
+
page = [page.to_i, 1].max
|
|
249
|
+
limit = limit.to_i.clamp(1, 100)
|
|
250
|
+
|
|
251
|
+
base, grouped = build_conditions(filters)
|
|
252
|
+
option_conditions, exact = QueryBuilder.option_conditions(grouped)
|
|
253
|
+
result = request_typesense do
|
|
254
|
+
execute_search(query: query, conditions: base + option_conditions, sort: sort, page: page, limit: limit,
|
|
255
|
+
filters: filters)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
product_ids = result['hits'].map { |h| h['document']['product_id'] }.uniq
|
|
259
|
+
raw_ids = product_ids.filter_map { |pid| Spree::Product.decode_prefixed_id(pid) }
|
|
260
|
+
|
|
261
|
+
# Values of two or more option types must sit on ONE variant, as core's
|
|
262
|
+
# `Product.with_option_value_ids` decides. Documents carry combination
|
|
263
|
+
# tokens that answer it exactly. Where they cannot (more types than the
|
|
264
|
+
# tokens cover), the index only pre-filters and core's scope gives the
|
|
265
|
+
# exact answer, so a page can come back short of `limit` and the total
|
|
266
|
+
# over-count.
|
|
267
|
+
scope = scope.with_option_value_ids(grouped.values.flatten) unless exact
|
|
268
|
+
|
|
269
|
+
# SECURITY: intersect with the authorized scope, preserving engine order.
|
|
270
|
+
products = if raw_ids.any?
|
|
271
|
+
records = scope.where(id: raw_ids).reorder(nil).index_by(&:id)
|
|
272
|
+
raw_ids.filter_map { |id| records[id] }
|
|
273
|
+
else
|
|
274
|
+
scope.none
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
SearchResult.new(products: products, total_count: result['found'] || 0,
|
|
278
|
+
pagy: build_pagy(result['found'] || 0, page, limit))
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
# Facets from Typesense, or, when Typesense cannot answer, from the
|
|
282
|
+
# Database provider, which computes the same ones from the database.
|
|
283
|
+
# As in #search_and_filter, only the Typesense requests are covered.
|
|
284
|
+
def filters(scope:, query: nil, filters: {})
|
|
285
|
+
fallback = -> { fallback_provider.filters(scope: scope, query: query, filters: filters) }
|
|
286
|
+
answer_through_breaker(query, fallback) do
|
|
287
|
+
result, distribution, stats = execute_faceted_search(query: query, filters: filters)
|
|
288
|
+
|
|
289
|
+
FiltersResult.new(
|
|
290
|
+
filters: Facets.build(distribution, currency: currency, stats: stats),
|
|
291
|
+
sort_options: sort_options_payload,
|
|
292
|
+
default_sort: default_sort_for(filters),
|
|
293
|
+
total_count: result['found'] || 0
|
|
294
|
+
)
|
|
295
|
+
end
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
# Composite ids mean one product maps to N documents (locale × currency),
|
|
299
|
+
# so delete by filter on product_id rather than by document id. From the
|
|
300
|
+
# live collection and any collection a reindex is filling.
|
|
301
|
+
def remove_by_id(prefixed_id)
|
|
302
|
+
each_write_target do |target, _live|
|
|
303
|
+
delete_product_documents(prefixed_id, from: target)
|
|
304
|
+
end
|
|
305
|
+
nil
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# The client indexing and removal go through: background jobs, which can
|
|
309
|
+
# afford to wait, and retry once on a node that fails.
|
|
310
|
+
def client
|
|
311
|
+
@client ||= build_client(connection_timeout_seconds: 5)
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
# The client searches and facets go through. A shopper waits on these,
|
|
315
|
+
# so a hung node must not hold the request: a short timeout and no retry
|
|
316
|
+
# (the indexing client's 5s and one retry cost about 10s), after which
|
|
317
|
+
# the search is answered by the Database provider. The timeout is
|
|
318
|
+
# TYPESENSE_SEARCH_TIMEOUT seconds when that is a positive number.
|
|
319
|
+
def search_client
|
|
320
|
+
@search_client ||= build_client(connection_timeout_seconds: search_timeout_seconds, num_retries: 0)
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# The live name searches and writes go through: since 0.3.0 an alias
|
|
324
|
+
# over the versioned collection the last reindex built (see
|
|
325
|
+
# IndexLayout).
|
|
326
|
+
def collection_name
|
|
327
|
+
@collection_name ||= "products_#{store.code}"
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
private
|
|
331
|
+
|
|
332
|
+
# Where this store's collections and aliases live.
|
|
333
|
+
def index_layout
|
|
334
|
+
@index_layout ||= IndexLayout.new(client, store.code)
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
def current_time_ms = Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond)
|
|
338
|
+
|
|
339
|
+
# The collection the live name serves from: a real collection holding
|
|
340
|
+
# the name, which takes requests ahead of an alias, else the alias's
|
|
341
|
+
# target. Nil when there is no index.
|
|
342
|
+
def live_collection_name
|
|
343
|
+
layout = index_layout
|
|
344
|
+
layout.legacy_collection? ? layout.live_name : layout.alias_target(layout.live_name)
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# Yields each collection a single product's write goes to, and whether
|
|
348
|
+
# it is the live one. A collection being built that is gone is skipped:
|
|
349
|
+
# its reindex died, and the next one replaces it.
|
|
350
|
+
def each_write_target
|
|
351
|
+
index_layout.write_targets.each_with_index do |target, position|
|
|
352
|
+
live = position.zero?
|
|
353
|
+
begin
|
|
354
|
+
yield target, live
|
|
355
|
+
rescue ::Typesense::Error::ObjectNotFound
|
|
356
|
+
raise if live
|
|
357
|
+
|
|
358
|
+
Rails.logger.warn { "[spree_typesense] skipped '#{target}': the collection a reindex was building is gone" }
|
|
359
|
+
end
|
|
360
|
+
end
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# Indexes the scope in batches into `into`.
|
|
364
|
+
# @return [Array(Integer, Array<Hash>)] documents accepted, and the rejected ones
|
|
365
|
+
def index_scope(scope, into:)
|
|
366
|
+
indexed = 0
|
|
367
|
+
failures = []
|
|
368
|
+
scope.reorder(id: :asc).preload_associations_lazily.find_in_batches(batch_size: 500) do |batch|
|
|
369
|
+
read_at = current_time_ms
|
|
370
|
+
documents = batch.flat_map { |product| presenter_class.new(product, store).call }
|
|
371
|
+
next if documents.empty?
|
|
372
|
+
|
|
373
|
+
accepted = documents.size
|
|
374
|
+
begin
|
|
375
|
+
index_batch(documents, into: into, indexed_at: read_at)
|
|
376
|
+
rescue ImportError => e
|
|
377
|
+
failures.concat(e.failures)
|
|
378
|
+
accepted -= e.failures.size
|
|
379
|
+
end
|
|
380
|
+
indexed += accepted
|
|
381
|
+
Rails.logger.info { "[spree_typesense] indexed #{accepted} (#{indexed} total) into #{into}" }
|
|
382
|
+
end
|
|
383
|
+
[indexed, failures]
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
# Forgets a reindex's collection: removes the marker and drops it,
|
|
387
|
+
# leaving the live alias as it was. Best effort, so the error that
|
|
388
|
+
# stopped the reindex is the one raised.
|
|
389
|
+
def abandon_build(layout, building)
|
|
390
|
+
layout.unpoint!(layout.building_name)
|
|
391
|
+
layout.drop!(building)
|
|
392
|
+
rescue StandardError => e
|
|
393
|
+
Rails.logger.error { "[spree_typesense] could not clean up '#{building}': #{e.class}: #{e.message}" }
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
# Points the live alias at the collection just built, then drops what
|
|
397
|
+
# it replaced. Typesense refuses an alias whose name a collection holds,
|
|
398
|
+
# so a real collection holding the live name is dropped first; searches
|
|
399
|
+
# in that moment get a 404 and answer from the Database provider.
|
|
400
|
+
# Pointing the alias is retried once: after that drop, failing it would
|
|
401
|
+
# leave the store with no index. Once it points at the new collection
|
|
402
|
+
# the reindex has succeeded, so the clean-up after it only logs what it
|
|
403
|
+
# could not do. Older versioned collections go, newer ones stay: a newer
|
|
404
|
+
# one belongs to a reindex still running, which will drop this one when
|
|
405
|
+
# it swaps.
|
|
406
|
+
def swap_live_alias(layout, building)
|
|
407
|
+
previous = layout.alias_target(layout.live_name)
|
|
408
|
+
legacy = layout.legacy_collection?
|
|
409
|
+
layout.drop!(layout.live_name) if legacy
|
|
410
|
+
point_live_alias(layout, building)
|
|
411
|
+
|
|
412
|
+
after_swap("remove the #{layout.building_name} alias") { layout.unpoint!(layout.building_name) }
|
|
413
|
+
replaced = after_swap('list the collections it replaced') do
|
|
414
|
+
layout.versioned_collections.select { |name| name < building }
|
|
415
|
+
end || []
|
|
416
|
+
dropped = replaced.count { |name| after_swap("drop #{name}") { layout.drop!(name) } }
|
|
417
|
+
|
|
418
|
+
Rails.logger.info do
|
|
419
|
+
was = [(layout.live_name if legacy), previous].compact.uniq.join(', ')
|
|
420
|
+
"[spree_typesense] #{layout.live_name} now serves #{building}#{" (was #{was})" unless was.empty?}; " \
|
|
421
|
+
"dropped #{dropped} older collection(s)#{' and the real collection holding the live name' if legacy}"
|
|
422
|
+
end
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
def point_live_alias(layout, building)
|
|
426
|
+
attempts = 0
|
|
427
|
+
begin
|
|
428
|
+
layout.point!(layout.live_name, building)
|
|
429
|
+
rescue StandardError => e
|
|
430
|
+
attempts += 1
|
|
431
|
+
raise if attempts > 1
|
|
432
|
+
|
|
433
|
+
Rails.logger.warn { "[spree_typesense] pointing #{layout.live_name} at #{building} failed (#{e.message}); retrying" }
|
|
434
|
+
retry
|
|
435
|
+
end
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
# Clean-up once the live alias points at the new collection: the
|
|
439
|
+
# reindex has succeeded, so a failure here is logged for the next
|
|
440
|
+
# reindex to finish, not raised. Returns the block's value (truthy for
|
|
441
|
+
# a drop), or nil when it failed.
|
|
442
|
+
def after_swap(what)
|
|
443
|
+
yield || true
|
|
444
|
+
rescue StandardError => e
|
|
445
|
+
Rails.logger.warn do
|
|
446
|
+
"[spree_typesense] the swap succeeded, but could not #{what} (#{e.class}: #{e.message}); " \
|
|
447
|
+
'the next reindex will'
|
|
448
|
+
end
|
|
449
|
+
nil
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
# The rejected documents of an import, logged one by one. A result names
|
|
453
|
+
# its document only when the import asked for that, so fall back to the
|
|
454
|
+
# position: Typesense answers one result per document, in order.
|
|
455
|
+
def import_failures(documents, results, into = collection_name)
|
|
456
|
+
Array(results).each_with_index.filter_map do |result, index|
|
|
457
|
+
next if result['success']
|
|
458
|
+
|
|
459
|
+
failure = { id: result['id'] || documents.dig(index, 'id'), error: result['error'] }
|
|
460
|
+
Rails.logger.error do
|
|
461
|
+
"[spree_typesense] Typesense rejected document #{failure[:id]} in collection " \
|
|
462
|
+
"'#{into}': #{failure[:error]}"
|
|
463
|
+
end
|
|
464
|
+
failure
|
|
465
|
+
end
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
# Deletes the product's documents in `from` other than `documents`; all
|
|
469
|
+
# of them when `documents` is empty. Only documents written before
|
|
470
|
+
# `written_before` go: a later job's are its own business. Ids go into
|
|
471
|
+
# the filter quoted in backticks, Typesense's string literal, because
|
|
472
|
+
# locales such as `en-US` put a hyphen in them. An id that itself holds a
|
|
473
|
+
# backtick cannot be quoted, so nothing is deleted then rather than risk
|
|
474
|
+
# deleting the wrong documents.
|
|
475
|
+
def remove_stale_documents(prefixed_id, documents, from:, written_before:)
|
|
476
|
+
kept_ids = documents.map { |d| (d[:id] || d['id']).to_s }
|
|
477
|
+
return delete_product_documents(prefixed_id, from: from, written_before: written_before) if kept_ids.empty?
|
|
478
|
+
|
|
479
|
+
if kept_ids.any? { |id| id.include?('`') }
|
|
480
|
+
Rails.logger.warn { "[spree_typesense] not removing stale documents of #{prefixed_id}: an id holds a backtick" }
|
|
481
|
+
return
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
client.collections[from].documents.delete(
|
|
485
|
+
filter_by: "product_id:=#{sanitize_prefixed_id(prefixed_id)} && " \
|
|
486
|
+
"id:!=[#{kept_ids.map { |id| "`#{id}`" }.join(',')}] && indexed_at:<#{written_before.to_i}"
|
|
487
|
+
)
|
|
488
|
+
rescue ::Typesense::Error::ObjectNotFound
|
|
489
|
+
nil
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
# Nothing to remove from a collection that does not exist. With
|
|
493
|
+
# `written_before`, only documents stamped earlier.
|
|
494
|
+
def delete_product_documents(prefixed_id, from:, written_before: nil)
|
|
495
|
+
filter = "product_id:=#{sanitize_prefixed_id(prefixed_id)}"
|
|
496
|
+
filter += " && indexed_at:<#{written_before.to_i}" if written_before
|
|
497
|
+
client.collections[from].documents.delete(filter_by: filter)
|
|
498
|
+
rescue ::Typesense::Error::ObjectNotFound
|
|
499
|
+
nil
|
|
500
|
+
end
|
|
501
|
+
|
|
502
|
+
# The Typesense request; callers wrap it in #request_typesense. Sent as
|
|
503
|
+
# a one-search multi_search, a POST: a GET search carries its filter in
|
|
504
|
+
# the query string, which Typesense caps at 4000 characters, and a
|
|
505
|
+
# selection across option types can make a far longer filter.
|
|
506
|
+
def execute_search(query:, conditions:, sort:, page:, limit:, facet_by: nil, filters: {})
|
|
507
|
+
perform_searches(
|
|
508
|
+
[search_params(query: query, conditions: conditions, sort: sort, page: page, limit: limit,
|
|
509
|
+
facet_by: facet_by, filters: filters)]
|
|
510
|
+
).first
|
|
511
|
+
end
|
|
512
|
+
|
|
513
|
+
# Faceted read path. Returns [main_result, distribution, stats], and
|
|
514
|
+
# raises Unavailable when Typesense cannot answer.
|
|
515
|
+
# `stats` always comes from the MAIN query — it covers numeric fields
|
|
516
|
+
# (price), which disjunctive relaxation never touches.
|
|
517
|
+
#
|
|
518
|
+
# DISJUNCTIVE FACETING: with option values selected, the counts for each
|
|
519
|
+
# selected option type must be computed with THAT type's own filter
|
|
520
|
+
# removed — otherwise picking "Red" reports every other colour as zero
|
|
521
|
+
# and the shopper can never widen their choice. That needs one extra
|
|
522
|
+
# query per selected type, so it goes through multi_search. With nothing
|
|
523
|
+
# selected there is nothing to relax, so we stay on the single-query path
|
|
524
|
+
# rather than pay for a multi_search on every request.
|
|
525
|
+
def execute_faceted_search(query:, filters:)
|
|
526
|
+
base, grouped = build_conditions(filters)
|
|
527
|
+
facet_by = Schema::FACET_FIELDS.join(',')
|
|
528
|
+
|
|
529
|
+
if grouped.empty?
|
|
530
|
+
result = request_typesense do
|
|
531
|
+
execute_search(query: query, conditions: base, sort: nil, page: 1, limit: 0, facet_by: facet_by)
|
|
532
|
+
end
|
|
533
|
+
return [result, Facets.normalize(result['facet_counts']), Facets.normalize_stats(result['facet_counts'])]
|
|
534
|
+
end
|
|
535
|
+
|
|
536
|
+
option_type_ids = grouped.keys
|
|
537
|
+
searches = [
|
|
538
|
+
search_params(query: query, conditions: base + QueryBuilder.option_conditions(grouped).first,
|
|
539
|
+
sort: nil, page: 1, limit: 0, facet_by: facet_by)
|
|
540
|
+
]
|
|
541
|
+
option_type_ids.each do |option_type_id|
|
|
542
|
+
relaxed = QueryBuilder.option_conditions(grouped.except(option_type_id)).first
|
|
543
|
+
searches << search_params(query: query, conditions: base + relaxed,
|
|
544
|
+
sort: nil, page: 1, limit: 0, facet_by: 'option_value_ids')
|
|
545
|
+
end
|
|
546
|
+
|
|
547
|
+
results = request_typesense { execute_multi_search(searches) }
|
|
548
|
+
[results.first,
|
|
549
|
+
merge_disjunctive_facets(results.first, results.drop(1), option_type_ids),
|
|
550
|
+
Facets.normalize_stats(results.first['facet_counts'])]
|
|
551
|
+
end
|
|
552
|
+
|
|
553
|
+
# The disjunctive facet searches, in one request.
|
|
554
|
+
def execute_multi_search(searches) = perform_searches(searches)
|
|
555
|
+
|
|
556
|
+
# Typesense answers multi_search with HTTP 200 even when an individual
|
|
557
|
+
# search failed — the failure is a per-result {error:, code:} body, NOT an
|
|
558
|
+
# exception. Raise it here as the error a search would have raised for
|
|
559
|
+
# that status, so a broken filter cannot silently serve wrong results or
|
|
560
|
+
# counts, and a 5xx counts as transient.
|
|
561
|
+
def perform_searches(searches)
|
|
562
|
+
response = search_client.multi_search.perform(
|
|
563
|
+
{ searches: searches.map { |s| s.merge(collection: collection_name) } }
|
|
564
|
+
)
|
|
565
|
+
results = response['results']
|
|
566
|
+
|
|
567
|
+
failed = Array(results).find { |r| r['error'] }
|
|
568
|
+
raise search_error_for(failed) if failed
|
|
569
|
+
|
|
570
|
+
results
|
|
571
|
+
end
|
|
572
|
+
|
|
573
|
+
PER_RESULT_ERRORS = {
|
|
574
|
+
400 => ::Typesense::Error::RequestMalformed,
|
|
575
|
+
401 => ::Typesense::Error::RequestUnauthorized,
|
|
576
|
+
403 => ::Typesense::Error::RequestUnauthorized,
|
|
577
|
+
404 => ::Typesense::Error::ObjectNotFound,
|
|
578
|
+
422 => ::Typesense::Error::ObjectUnprocessable
|
|
579
|
+
}.freeze
|
|
580
|
+
private_constant :PER_RESULT_ERRORS
|
|
581
|
+
|
|
582
|
+
def search_error_for(failed)
|
|
583
|
+
code = failed['code'].to_i
|
|
584
|
+
klass = PER_RESULT_ERRORS.fetch(code) { (500..599).cover?(code) ? ::Typesense::Error::ServerError : SearchError }
|
|
585
|
+
klass.new("multi_search failed: #{failed['code']} #{failed['error']}")
|
|
586
|
+
end
|
|
587
|
+
|
|
588
|
+
# Start from the main (fully filtered) distribution, then for each option
|
|
589
|
+
# type overwrite the counts of the values BELONGING TO THAT TYPE with the
|
|
590
|
+
# counts from that type's own relaxed query. Values of other types keep
|
|
591
|
+
# the main query's counts. Mapping a value to its type needs the database,
|
|
592
|
+
# same lookup as +group_option_values_by_type+.
|
|
593
|
+
def merge_disjunctive_facets(main_result, relaxed_results, option_type_ids)
|
|
594
|
+
distribution = Facets.normalize(main_result['facet_counts'])
|
|
595
|
+
merged = (distribution['option_value_ids'] || {}).dup
|
|
596
|
+
|
|
597
|
+
per_type = {}
|
|
598
|
+
relaxed_results.each_with_index do |result, i|
|
|
599
|
+
per_type[option_type_ids[i]] = Facets.normalize(result['facet_counts'])['option_value_ids'] || {}
|
|
600
|
+
end
|
|
601
|
+
|
|
602
|
+
type_of = option_type_of(per_type.values.flat_map(&:keys).uniq)
|
|
603
|
+
per_type.each do |option_type_id, relaxed_distribution|
|
|
604
|
+
relaxed_distribution.each do |prefixed_id, count|
|
|
605
|
+
merged[prefixed_id] = count if type_of[prefixed_id] == option_type_id
|
|
606
|
+
end
|
|
607
|
+
end
|
|
608
|
+
|
|
609
|
+
distribution.merge('option_value_ids' => merged)
|
|
610
|
+
end
|
|
611
|
+
|
|
612
|
+
# { 'ov_red' => option_type_id, ... } in one query.
|
|
613
|
+
def option_type_of(prefixed_ids)
|
|
614
|
+
raw_ids = prefixed_ids.filter_map { |pid| Spree::OptionValue.decode_prefixed_id(pid) }
|
|
615
|
+
by_raw_id = Spree::OptionValue.where(id: raw_ids).pluck(:id, :option_type_id).to_h
|
|
616
|
+
|
|
617
|
+
prefixed_ids.each_with_object({}) do |prefixed_id, acc|
|
|
618
|
+
raw_id = Spree::OptionValue.decode_prefixed_id(prefixed_id)
|
|
619
|
+
acc[prefixed_id] = by_raw_id[raw_id] if raw_id
|
|
620
|
+
end
|
|
621
|
+
end
|
|
622
|
+
|
|
623
|
+
# Returns [base_conditions, grouped_option_values]. Kept separate because
|
|
624
|
+
# disjunctive faceting has to rebuild the option half per query while the
|
|
625
|
+
# base half (system scoping + user filters) stays fixed.
|
|
626
|
+
def build_conditions(filters)
|
|
627
|
+
f = normalize_filters(filters)
|
|
628
|
+
option_ids = f.delete('with_option_value_ids')
|
|
629
|
+
|
|
630
|
+
base = QueryBuilder.system_conditions(store: store, locale: locale, currency: currency,
|
|
631
|
+
channel_id: current_channel&.id)
|
|
632
|
+
base += QueryBuilder.user_conditions(f)
|
|
633
|
+
|
|
634
|
+
[base, group_option_values_by_type(Array(option_ids))]
|
|
635
|
+
end
|
|
636
|
+
|
|
637
|
+
# The channel the request is served on, read as core's storefront scope
|
|
638
|
+
# and the Meilisearch provider read it on every supported version. Nil
|
|
639
|
+
# outside a channel context, and then nothing is narrowed.
|
|
640
|
+
def current_channel
|
|
641
|
+
Spree::Current.channel
|
|
642
|
+
end
|
|
643
|
+
|
|
644
|
+
# A string-keyed copy of the filters, whichever shape they arrive in:
|
|
645
|
+
# Parameters, a symbol- or string-keyed Hash, or nil.
|
|
646
|
+
def normalize_filters(filters)
|
|
647
|
+
f = filters.respond_to?(:to_unsafe_h) ? filters.to_unsafe_h : (filters || {}).to_h
|
|
648
|
+
f.transform_keys(&:to_s)
|
|
649
|
+
end
|
|
650
|
+
|
|
651
|
+
# The sort a listing starts on, as core's FiltersAggregator derives it:
|
|
652
|
+
# the collection's own sort order (categories are manual only), given in
|
|
653
|
+
# the API's form ('price desc' -> '-price'), and 'manual' otherwise.
|
|
654
|
+
def default_sort_for(filters)
|
|
655
|
+
collection = normalize_filters(filters)['_collection']
|
|
656
|
+
sort_order = collection.sort_order.presence if collection.respond_to?(:sort_order)
|
|
657
|
+
field, direction = (sort_order || DEFAULT_SORT).to_s.split(' ', 2)
|
|
658
|
+
direction == 'desc' ? "-#{field}" : field
|
|
659
|
+
end
|
|
660
|
+
|
|
661
|
+
def search_params(query:, conditions:, sort:, page:, limit:, facet_by: nil, filters: {})
|
|
662
|
+
params = {
|
|
663
|
+
q: query.presence || '*',
|
|
664
|
+
query_by: Schema::SEARCHABLE_FIELDS.join(','),
|
|
665
|
+
filter_by: conditions.join(' && '),
|
|
666
|
+
page: page,
|
|
667
|
+
per_page: limit
|
|
668
|
+
}
|
|
669
|
+
sort_by = QueryBuilder.sort_by(sort, normalize_filters(filters))
|
|
670
|
+
params[:sort_by] = sort_by if sort_by
|
|
671
|
+
if facet_by
|
|
672
|
+
params[:facet_by] = facet_by
|
|
673
|
+
# Typesense caps facet values at 10 per field by default and truncates
|
|
674
|
+
# SILENTLY. The price range no longer depends on this (it reads
|
|
675
|
+
# stats.min/max, which are exact at any cardinality), but option and
|
|
676
|
+
# category facets DO — their counts only exist as returned values, and
|
|
677
|
+
# string facets get no stats block. Meilisearch's equivalent default is
|
|
678
|
+
# 100; match it. Residual risk: an option type with more than 100
|
|
679
|
+
# values still loses its tail.
|
|
680
|
+
params[:max_facet_values] = MAX_FACET_VALUES
|
|
681
|
+
end
|
|
682
|
+
params
|
|
683
|
+
end
|
|
684
|
+
|
|
685
|
+
# Answers through Typesense unless the circuit breaker says it is
|
|
686
|
+
# failing, in which case `fallback` (the Database provider) answers at
|
|
687
|
+
# once, with nothing logged. A transient failure counts towards opening
|
|
688
|
+
# it, and the one that opens it is announced once; anything else
|
|
689
|
+
# Typesense answered, even with an error, counts as it answering.
|
|
690
|
+
# Missing configuration made no request and counts as neither. Errors
|
|
691
|
+
# that are not Typesense's propagate, and free the probe if this was it.
|
|
692
|
+
def answer_through_breaker(query, fallback)
|
|
693
|
+
breaker = circuit_breaker
|
|
694
|
+
return fallback.call unless breaker.allow_request?
|
|
695
|
+
|
|
696
|
+
begin
|
|
697
|
+
answer = yield
|
|
698
|
+
rescue Unavailable => e
|
|
699
|
+
record_unavailable(breaker, e.error)
|
|
700
|
+
report_unavailable(e.error, query)
|
|
701
|
+
return fallback.call
|
|
702
|
+
rescue StandardError
|
|
703
|
+
breaker.release_probe
|
|
704
|
+
raise
|
|
705
|
+
end
|
|
706
|
+
|
|
707
|
+
Rails.logger.info { '[spree_typesense] Typesense is answering again' } if breaker.record_success == :closed
|
|
708
|
+
answer
|
|
709
|
+
end
|
|
710
|
+
|
|
711
|
+
def record_unavailable(breaker, error)
|
|
712
|
+
if error.is_a?(::Typesense::Error::MissingConfiguration)
|
|
713
|
+
breaker.release_probe
|
|
714
|
+
elsif TRANSIENT_ERRORS.any? { |klass| error.is_a?(klass) }
|
|
715
|
+
announce_circuit_opened(breaker, error) if breaker.record_failure == :opened
|
|
716
|
+
elsif breaker.record_success == :closed
|
|
717
|
+
Rails.logger.info { '[spree_typesense] Typesense is answering again' }
|
|
718
|
+
end
|
|
719
|
+
end
|
|
720
|
+
|
|
721
|
+
def announce_circuit_opened(breaker, error)
|
|
722
|
+
message = "Typesense failed #{breaker.threshold} times in a row (last: #{error.class}: #{error.message}); " \
|
|
723
|
+
"skipping it for #{breaker.open_seconds.to_s.delete_suffix('.0')} seconds and answering searches " \
|
|
724
|
+
'from the Database provider'
|
|
725
|
+
Rails.logger.error { "[spree_typesense] #{message}" }
|
|
726
|
+
Rails.error.report(CircuitOpened.new(message), handled: true, severity: :warning,
|
|
727
|
+
context: { collection: collection_name })
|
|
728
|
+
end
|
|
729
|
+
|
|
730
|
+
def circuit_breaker = Spree::Typesense::CircuitBreaker.for(ENV.fetch('TYPESENSE_URL', ''))
|
|
731
|
+
|
|
732
|
+
# Runs a Typesense request, raising Unavailable in place of any failure
|
|
733
|
+
# in UNAVAILABLE_ERRORS. Wrap only the request itself: whatever the block
|
|
734
|
+
# raises in that list is taken for Typesense being unavailable.
|
|
735
|
+
def request_typesense
|
|
736
|
+
yield
|
|
737
|
+
rescue *UNAVAILABLE_ERRORS => e
|
|
738
|
+
raise Unavailable, e
|
|
739
|
+
end
|
|
740
|
+
|
|
741
|
+
# Logs and reports a failure the caller is about to answer from the
|
|
742
|
+
# Database provider: as a warning when it is transient, and as an error
|
|
743
|
+
# when it needs someone to act (see TRANSIENT_ERRORS). Missing
|
|
744
|
+
# configuration is logged and reported once per process: it fails every
|
|
745
|
+
# search the same way until someone sets it.
|
|
746
|
+
#
|
|
747
|
+
# A manual listing of a grouping no indexed document has a position in
|
|
748
|
+
# yet (before the first reindex that writes them, or a grouping with no
|
|
749
|
+
# indexed product) is refused by Typesense for want of the field. That
|
|
750
|
+
# is expected and needs nobody: it is logged at info, not reported, and
|
|
751
|
+
# the Database provider, which sorts by position too, answers it.
|
|
752
|
+
def report_unavailable(error, query)
|
|
753
|
+
if error.is_a?(::Typesense::Error::MissingConfiguration)
|
|
754
|
+
return unless UNCONFIGURED_LOGGED.make_true
|
|
755
|
+
end
|
|
756
|
+
if unindexed_position_sort?(error)
|
|
757
|
+
Rails.logger.info do
|
|
758
|
+
"[spree_typesense] no indexed document has a position here yet (#{error.message}); answering this " \
|
|
759
|
+
'manual listing from the Database provider'
|
|
760
|
+
end
|
|
761
|
+
return
|
|
762
|
+
end
|
|
763
|
+
|
|
764
|
+
Rails.logger.error do
|
|
765
|
+
"[spree_typesense] Typesense could not answer (#{error.class}: #{error.message}); answering from the " \
|
|
766
|
+
'Database provider. If the collection is missing or out of date, run `rake spree:search:reindex`.'
|
|
767
|
+
end
|
|
768
|
+
severity = TRANSIENT_ERRORS.any? { |klass| error.is_a?(klass) } ? :warning : :error
|
|
769
|
+
Rails.error.report(error, handled: true, severity: severity,
|
|
770
|
+
context: { collection: collection_name, query: query })
|
|
771
|
+
end
|
|
772
|
+
|
|
773
|
+
def unindexed_position_sort?(error)
|
|
774
|
+
error.is_a?(::Typesense::Error::ObjectNotFound) && error.message.to_s.include?('field named `pos_')
|
|
775
|
+
end
|
|
776
|
+
|
|
777
|
+
# Typesense alters a live collection in place: PATCH /collections/:name
|
|
778
|
+
# with the fields to add. Documents already stored keep their values and
|
|
779
|
+
# gain the new fields when next indexed. So a field a release adds
|
|
780
|
+
# reaches the collection without dropping it, and the store keeps
|
|
781
|
+
# serving searches throughout.
|
|
782
|
+
#
|
|
783
|
+
# What Typesense cannot do in place is change a field's type. Never drop
|
|
784
|
+
# the collection here, which would empty a live index: name the field
|
|
785
|
+
# and the remedy instead. A reindex builds a fresh collection, which has
|
|
786
|
+
# the declared type.
|
|
787
|
+
#
|
|
788
|
+
# A failed PATCH raises, so the caller stops rather than carrying on
|
|
789
|
+
# against a collection that still cannot serve the new filters.
|
|
790
|
+
#
|
|
791
|
+
# @param name [String] a real collection's name, not an alias
|
|
792
|
+
def update_collection_schema!(name)
|
|
793
|
+
live_fields = client.collections[name].retrieve['fields']
|
|
794
|
+
|
|
795
|
+
missing = Schema.missing_fields(live_fields)
|
|
796
|
+
if missing.any?
|
|
797
|
+
schema_client.collections[name].update(fields: missing)
|
|
798
|
+
Rails.logger.info do
|
|
799
|
+
"[spree_typesense] added #{missing.map { |f| f[:name] }.join(', ')} to collection '#{name}'"
|
|
800
|
+
end
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
Schema.type_mismatches(live_fields).each do |mismatch|
|
|
804
|
+
Rails.logger.warn do
|
|
805
|
+
"[spree_typesense] collection '#{name}' declares '#{mismatch[:name]}' as " \
|
|
806
|
+
"#{mismatch[:live_type]}, but this release declares it #{mismatch[:type]}. Typesense cannot change " \
|
|
807
|
+
"a field's type in place: reindex (`rake spree:search:reindex`), which builds a fresh collection."
|
|
808
|
+
end
|
|
809
|
+
end
|
|
810
|
+
end
|
|
811
|
+
|
|
812
|
+
# Schema changes block writes to the collection until Typesense finishes
|
|
813
|
+
# them, and the response comes only then, so they need a far longer
|
|
814
|
+
# timeout than indexing. No retries: Typesense runs one alter at a time,
|
|
815
|
+
# and would refuse a retry while the first is still running.
|
|
816
|
+
def schema_client
|
|
817
|
+
@schema_client ||= build_client(connection_timeout_seconds: SCHEMA_CHANGE_TIMEOUT_SECONDS, num_retries: 0)
|
|
818
|
+
end
|
|
819
|
+
|
|
820
|
+
# Raises Typesense::Error::MissingConfiguration, rather than KeyError,
|
|
821
|
+
# when TYPESENSE_URL or TYPESENSE_API_KEY is unset or the URL does not
|
|
822
|
+
# parse, so a search treats it as Typesense being unavailable.
|
|
823
|
+
def build_client(**options)
|
|
824
|
+
url = ENV.fetch('TYPESENSE_URL', nil).presence
|
|
825
|
+
api_key = ENV.fetch('TYPESENSE_API_KEY', nil).presence
|
|
826
|
+
unless url && api_key
|
|
827
|
+
missing = { 'TYPESENSE_URL' => url, 'TYPESENSE_API_KEY' => api_key }.select { |_, v| v.nil? }.keys
|
|
828
|
+
raise ::Typesense::Error::MissingConfiguration, "#{missing.join(' and ')} not set"
|
|
829
|
+
end
|
|
830
|
+
|
|
831
|
+
uri = URI.parse(url)
|
|
832
|
+
::Typesense::Client.new(
|
|
833
|
+
nodes: [{ host: uri.host, port: uri.port, protocol: uri.scheme }],
|
|
834
|
+
api_key: api_key,
|
|
835
|
+
**options
|
|
836
|
+
)
|
|
837
|
+
rescue URI::InvalidURIError => e
|
|
838
|
+
raise ::Typesense::Error::MissingConfiguration, "TYPESENSE_URL is not a valid URL: #{e.message}"
|
|
839
|
+
end
|
|
840
|
+
|
|
841
|
+
def search_timeout_seconds
|
|
842
|
+
configured = Float(ENV.fetch('TYPESENSE_SEARCH_TIMEOUT', nil), exception: false)
|
|
843
|
+
configured&.positive? ? configured : SEARCH_TIMEOUT_SECONDS
|
|
844
|
+
end
|
|
845
|
+
|
|
846
|
+
# Mirrors meilisearch.rb#group_option_values_by_type. Hits the DB.
|
|
847
|
+
def group_option_values_by_type(prefixed_ids)
|
|
848
|
+
ids = prefixed_ids.flatten.compact.select { |id| QueryBuilder.valid_id?(id) }
|
|
849
|
+
return {} if ids.empty?
|
|
850
|
+
|
|
851
|
+
raw = ids.filter_map { |id| Spree::OptionValue.decode_prefixed_id(id) }
|
|
852
|
+
Spree::OptionValue.where(id: raw).group_by(&:option_type_id)
|
|
853
|
+
.transform_values { |ovs| ovs.map(&:prefixed_id) }
|
|
854
|
+
end
|
|
855
|
+
|
|
856
|
+
# The presenter emits Spree's canonical document; Typesense needs fields
|
|
857
|
+
# Spree does not provide. Derive them here rather than forking the presenter.
|
|
858
|
+
def derive_fields(document, indexed_at = nil)
|
|
859
|
+
doc = document.transform_keys(&:to_s)
|
|
860
|
+
doc['indexed_at'] = indexed_at.to_i if indexed_at
|
|
861
|
+
raw = doc['available_on']
|
|
862
|
+
doc['available_on_i'] = raw.present? ? Time.parse(raw.to_s).to_i : 0
|
|
863
|
+
doc
|
|
864
|
+
rescue ArgumentError
|
|
865
|
+
doc['available_on_i'] = 0
|
|
866
|
+
doc
|
|
867
|
+
end
|
|
868
|
+
|
|
869
|
+
def build_pagy(count, page, limit)
|
|
870
|
+
Pagy::Offset.new(count: count, page: page, limit: limit)
|
|
871
|
+
end
|
|
872
|
+
|
|
873
|
+
# The Database provider takes the same `audience:` as this one, and uses
|
|
874
|
+
# it for its query and ransack authorisation, so a back-office search
|
|
875
|
+
# (audience: nil) must not degrade into a storefront one.
|
|
876
|
+
def fallback_provider
|
|
877
|
+
Spree::SearchProvider::Database.new(store, audience: audience)
|
|
878
|
+
end
|
|
879
|
+
|
|
880
|
+
# The gem's own presenter, unless the application named a subclass of
|
|
881
|
+
# it. Core configures none (`search_product_presenter: nil`); the engine
|
|
882
|
+
# registers ours, which counts as overridden.
|
|
883
|
+
#
|
|
884
|
+
# The setting is shared by every installed search gem, so anything that is
|
|
885
|
+
# not ours is ignored: with spree_meilisearch installed it may hold
|
|
886
|
+
# Meilisearch's presenter, whose documents this schema does not describe.
|
|
887
|
+
def presenter_class
|
|
888
|
+
configured = Spree::Dependencies.search_product_presenter_class if Spree::Dependencies.overridden?(:search_product_presenter)
|
|
889
|
+
configured.is_a?(Class) && configured <= Spree::Typesense::ProductPresenter ? configured : Spree::Typesense::ProductPresenter
|
|
890
|
+
end
|
|
891
|
+
|
|
892
|
+
# `remove_by_id` is public and reached via Spree's RemoveJob (ActiveJob
|
|
893
|
+
# serialisation, not a guaranteed well-formed id), so strip anything
|
|
894
|
+
# that isn't part of a legitimate prefixed id before it lands in a
|
|
895
|
+
# Typesense filter expression — mirrors the Meilisearch provider's guard.
|
|
896
|
+
def sanitize_prefixed_id(value)
|
|
897
|
+
value.to_s.gsub(/[^a-zA-Z0-9_]/, '')
|
|
898
|
+
end
|
|
899
|
+
end
|
|
900
|
+
end
|
|
901
|
+
end
|